From e50ea55064d891ce8c9239de7a533a420f42713e Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 4 Mar 2026 23:32:46 +0100 Subject: [PATCH] =?UTF-8?q?feat:=20create=20services/api/=20=E2=80=94=20un?= =?UTF-8?q?ified=20gRPC=20gateway=20with=20tonic-web?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copied from api_gateway, removed REST handlers (port 8080), added tonic-web + CORS for grpc-web browser access. Binary renamed: api-gateway → api Changes: - Package name: api-gateway → api - Deleted src/handlers/ (REST ML endpoints on port 8080) - Added tonic-web 0.13 + tower-http CORS layer - Server::builder().accept_http1(true) for grpc-web - CORS_ORIGINS env var (default http://localhost:5173) - Metrics server on port 9091 (axum) preserved - All 95 lib tests pass, 0 clippy warnings - Added services/api to workspace members Co-Authored-By: Claude Opus 4.6 --- Cargo.lock | 162 +- Cargo.toml | 1 + ...53eb03952985b7cc1b23e219c444eed0159fb.json | 14 + ...c2ce019140db89e57baddc1942b977ab3a431.json | 18 + ...9638d68a9cf64a78160e9e1ea5c9b2a0bbbb0.json | 58 + ...bf319b34186b30f241359d00440d8816c8772.json | 34 + ...43bfd0a233ea5223af844e5b2c402938db416.json | 17 + ...ced5b5057a1e7cc7a4bf8a4200d4b5777a73b.json | 82 + ...9bfe047c0151433c2359e51ad3e3c5e878f6a.json | 14 + ...da7c398662e8081e3a7929132a2031555dc5a.json | 14 + ...3b4e2d2647303c3e7a480ccea2df6359dc943.json | 18 + ...06361647f52ab989fd8a7fedcbb2a66748355.json | 14 + ...bd55d93c3ee12244ac0aaf9489f41c35c9189.json | 41 + services/api/Cargo.toml | 169 ++ services/api/Dockerfile | 82 + services/api/README.md | 22 + services/api/benches/auth_overhead.rs | 384 +++ .../api/benches/authz_dashmap_benchmark.rs | 341 +++ services/api/benches/cache_performance.rs | 408 ++++ .../api/benches/dashmap_rate_limiter_bench.rs | 334 +++ services/api/benches/proxy_latency.rs | 413 ++++ services/api/benches/rate_limiter_bench.rs | 143 ++ services/api/benches/rate_limiting_perf.rs | 320 +++ services/api/benches/revocation_cache_perf.rs | 382 +++ services/api/benches/routing_latency.rs | 294 +++ services/api/benches/throughput.rs | 448 ++++ services/api/build.rs | 194 ++ services/api/src/auth/interceptor.rs | 1071 ++++++++ services/api/src/auth/jwt/endpoints.rs | 318 +++ services/api/src/auth/jwt/mod.rs | 29 + services/api/src/auth/jwt/revocation.rs | 594 +++++ services/api/src/auth/jwt/service.rs | 508 ++++ services/api/src/auth/mfa/backup_codes.rs | 353 +++ services/api/src/auth/mfa/enrollment.rs | 214 ++ services/api/src/auth/mfa/mod.rs | 579 +++++ services/api/src/auth/mfa/qr_code.rs | 185 ++ services/api/src/auth/mfa/totp.rs | 431 ++++ services/api/src/auth/mfa/verification.rs | 180 ++ services/api/src/auth/mod.rs | 33 + services/api/src/auth/mtls/mod.rs | 77 + services/api/src/auth/mtls/revocation.rs | 791 ++++++ services/api/src/auth/mtls/tls_config.rs | 288 +++ services/api/src/auth/mtls/validator.rs | 555 +++++ services/api/src/config/authz.rs | 381 +++ services/api/src/config/endpoints.rs | 291 +++ services/api/src/config/manager.rs | 322 +++ services/api/src/config/mod.rs | 13 + services/api/src/config/validator.rs | 347 +++ services/api/src/error.rs | 31 + services/api/src/grpc/backtesting_proxy.rs | 582 +++++ .../api/src/grpc/backtesting_proxy_bench.rs | 41 + services/api/src/grpc/broker_gateway_proxy.rs | 197 ++ services/api/src/grpc/config_proxy.rs | 220 ++ .../api/src/grpc/data_acquisition_proxy.rs | 138 ++ services/api/src/grpc/ml_proxy.rs | 230 ++ services/api/src/grpc/ml_trading_proxy.rs | 483 ++++ services/api/src/grpc/ml_training_proxy.rs | 632 +++++ services/api/src/grpc/mod.rs | 40 + services/api/src/grpc/monitoring_handler.rs | 1386 +++++++++++ services/api/src/grpc/pods_handler.rs | 114 + services/api/src/grpc/risk_proxy.rs | 230 ++ services/api/src/grpc/server.rs | 443 ++++ services/api/src/grpc/trading_agent_proxy.rs | 551 +++++ services/api/src/grpc/trading_direct_proxy.rs | 336 +++ services/api/src/grpc/trading_proxy.rs | 2158 +++++++++++++++++ services/api/src/health.rs | 133 + services/api/src/health_router.rs | 285 +++ services/api/src/lib.rs | 129 + services/api/src/main.rs | 773 ++++++ services/api/src/metrics/auth_metrics.rs | 412 ++++ services/api/src/metrics/config_metrics.rs | 231 ++ services/api/src/metrics/exporter.rs | 136 ++ services/api/src/metrics/mod.rs | 74 + services/api/src/metrics/proxy_metrics.rs | 346 +++ services/api/src/routing/mod.rs | 12 + services/api/src/routing/rate_limiter.rs | 446 ++++ services/api/tests/auth_edge_cases.rs | 1289 ++++++++++ services/api/tests/auth_flow_tests.rs | 519 ++++ services/api/tests/common/mod.rs | 211 ++ services/api/tests/docker-compose.yml | 36 + services/api/tests/e2e_tests.rs | 879 +++++++ services/api/tests/grpc_error_handling.rs | 627 +++++ .../api/tests/grpc_error_handling_tests.rs | 24 + services/api/tests/health_check_tests.rs | 672 +++++ services/api/tests/integration_tests.rs | 13 + services/api/tests/jwt_service_edge_cases.rs | 662 +++++ .../api/tests/metrics_integration_test.rs | 298 +++ services/api/tests/mfa_comprehensive.rs | 1300 ++++++++++ .../tests/mfa_enrollment_integration_test.rs | 449 ++++ .../api/tests/ml_trading_integration_tests.rs | 922 +++++++ services/api/tests/proxy_latency_test.rs | 263 ++ .../api/tests/rate_limiter_advanced_tests.rs | 654 +++++ .../api/tests/rate_limiter_stress_test.rs | 480 ++++ .../api/tests/rate_limiting_comprehensive.rs | 1133 +++++++++ services/api/tests/rate_limiting_tests.rs | 344 +++ .../tests/real_backend_integration_test.rs | 637 +++++ services/api/tests/regime_endpoint_tests.rs | 51 + .../tests/regime_routing_integration_test.rs | 606 +++++ services/api/tests/routing_edge_cases.rs | 595 +++++ services/api/tests/service_proxy_tests.rs | 317 +++ 100 files changed, 35727 insertions(+), 24 deletions(-) create mode 100644 services/api/.sqlx/query-040b9e27fe399c9f581f93966d753eb03952985b7cc1b23e219c444eed0159fb.json create mode 100644 services/api/.sqlx/query-1368d36645c2548f0e4fb545b0cc2ce019140db89e57baddc1942b977ab3a431.json create mode 100644 services/api/.sqlx/query-3f7b62f1896d9d19ba56adde7119638d68a9cf64a78160e9e1ea5c9b2a0bbbb0.json create mode 100644 services/api/.sqlx/query-6b774bb2e56924adfbd961d335fbf319b34186b30f241359d00440d8816c8772.json create mode 100644 services/api/.sqlx/query-aea06694ee3e20e90795f0b3a9c43bfd0a233ea5223af844e5b2c402938db416.json create mode 100644 services/api/.sqlx/query-c00246b33b871002aa0a1a1a098ced5b5057a1e7cc7a4bf8a4200d4b5777a73b.json create mode 100644 services/api/.sqlx/query-dc585d296075abbee6d32a08ca49bfe047c0151433c2359e51ad3e3c5e878f6a.json create mode 100644 services/api/.sqlx/query-e19dc7e9161ae2510850e709c7fda7c398662e8081e3a7929132a2031555dc5a.json create mode 100644 services/api/.sqlx/query-ea9264a98b2bf6034a0ecaf22903b4e2d2647303c3e7a480ccea2df6359dc943.json create mode 100644 services/api/.sqlx/query-ed947b6e0201c32cd49d191293906361647f52ab989fd8a7fedcbb2a66748355.json create mode 100644 services/api/.sqlx/query-fa8446a8e2163a642e241866e04bd55d93c3ee12244ac0aaf9489f41c35c9189.json create mode 100644 services/api/Cargo.toml create mode 100644 services/api/Dockerfile create mode 100644 services/api/README.md create mode 100644 services/api/benches/auth_overhead.rs create mode 100644 services/api/benches/authz_dashmap_benchmark.rs create mode 100644 services/api/benches/cache_performance.rs create mode 100644 services/api/benches/dashmap_rate_limiter_bench.rs create mode 100644 services/api/benches/proxy_latency.rs create mode 100644 services/api/benches/rate_limiter_bench.rs create mode 100644 services/api/benches/rate_limiting_perf.rs create mode 100644 services/api/benches/revocation_cache_perf.rs create mode 100644 services/api/benches/routing_latency.rs create mode 100644 services/api/benches/throughput.rs create mode 100644 services/api/build.rs create mode 100644 services/api/src/auth/interceptor.rs create mode 100644 services/api/src/auth/jwt/endpoints.rs create mode 100644 services/api/src/auth/jwt/mod.rs create mode 100644 services/api/src/auth/jwt/revocation.rs create mode 100644 services/api/src/auth/jwt/service.rs create mode 100644 services/api/src/auth/mfa/backup_codes.rs create mode 100644 services/api/src/auth/mfa/enrollment.rs create mode 100644 services/api/src/auth/mfa/mod.rs create mode 100644 services/api/src/auth/mfa/qr_code.rs create mode 100644 services/api/src/auth/mfa/totp.rs create mode 100644 services/api/src/auth/mfa/verification.rs create mode 100644 services/api/src/auth/mod.rs create mode 100644 services/api/src/auth/mtls/mod.rs create mode 100644 services/api/src/auth/mtls/revocation.rs create mode 100644 services/api/src/auth/mtls/tls_config.rs create mode 100644 services/api/src/auth/mtls/validator.rs create mode 100644 services/api/src/config/authz.rs create mode 100644 services/api/src/config/endpoints.rs create mode 100644 services/api/src/config/manager.rs create mode 100644 services/api/src/config/mod.rs create mode 100644 services/api/src/config/validator.rs create mode 100644 services/api/src/error.rs create mode 100644 services/api/src/grpc/backtesting_proxy.rs create mode 100644 services/api/src/grpc/backtesting_proxy_bench.rs create mode 100644 services/api/src/grpc/broker_gateway_proxy.rs create mode 100644 services/api/src/grpc/config_proxy.rs create mode 100644 services/api/src/grpc/data_acquisition_proxy.rs create mode 100644 services/api/src/grpc/ml_proxy.rs create mode 100644 services/api/src/grpc/ml_trading_proxy.rs create mode 100644 services/api/src/grpc/ml_training_proxy.rs create mode 100644 services/api/src/grpc/mod.rs create mode 100644 services/api/src/grpc/monitoring_handler.rs create mode 100644 services/api/src/grpc/pods_handler.rs create mode 100644 services/api/src/grpc/risk_proxy.rs create mode 100644 services/api/src/grpc/server.rs create mode 100644 services/api/src/grpc/trading_agent_proxy.rs create mode 100644 services/api/src/grpc/trading_direct_proxy.rs create mode 100644 services/api/src/grpc/trading_proxy.rs create mode 100644 services/api/src/health.rs create mode 100644 services/api/src/health_router.rs create mode 100644 services/api/src/lib.rs create mode 100644 services/api/src/main.rs create mode 100644 services/api/src/metrics/auth_metrics.rs create mode 100644 services/api/src/metrics/config_metrics.rs create mode 100644 services/api/src/metrics/exporter.rs create mode 100644 services/api/src/metrics/mod.rs create mode 100644 services/api/src/metrics/proxy_metrics.rs create mode 100644 services/api/src/routing/mod.rs create mode 100644 services/api/src/routing/rate_limiter.rs create mode 100644 services/api/tests/auth_edge_cases.rs create mode 100644 services/api/tests/auth_flow_tests.rs create mode 100644 services/api/tests/common/mod.rs create mode 100644 services/api/tests/docker-compose.yml create mode 100644 services/api/tests/e2e_tests.rs create mode 100644 services/api/tests/grpc_error_handling.rs create mode 100644 services/api/tests/grpc_error_handling_tests.rs create mode 100644 services/api/tests/health_check_tests.rs create mode 100644 services/api/tests/integration_tests.rs create mode 100644 services/api/tests/jwt_service_edge_cases.rs create mode 100644 services/api/tests/metrics_integration_test.rs create mode 100644 services/api/tests/mfa_comprehensive.rs create mode 100644 services/api/tests/mfa_enrollment_integration_test.rs create mode 100644 services/api/tests/ml_trading_integration_tests.rs create mode 100644 services/api/tests/proxy_latency_test.rs create mode 100644 services/api/tests/rate_limiter_advanced_tests.rs create mode 100644 services/api/tests/rate_limiter_stress_test.rs create mode 100644 services/api/tests/rate_limiting_comprehensive.rs create mode 100644 services/api/tests/rate_limiting_tests.rs create mode 100644 services/api/tests/real_backend_integration_test.rs create mode 100644 services/api/tests/regime_endpoint_tests.rs create mode 100644 services/api/tests/regime_routing_integration_test.rs create mode 100644 services/api/tests/routing_edge_cases.rs create mode 100644 services/api/tests/service_proxy_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 6743a2ba7..282e78f22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -172,6 +172,83 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +[[package]] +name = "api" +version = "1.0.0" +dependencies = [ + "anyhow", + "async-stream", + "async-trait", + "axum 0.7.9", + "base32", + "base64 0.22.1", + "bytes", + "chrono", + "clap", + "common", + "config", + "const-oid", + "criterion", + "dashmap 6.1.0", + "futures", + "fxt", + "governor", + "hdrhistogram", + "hex", + "hmac", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.7.0", + "hyper-util", + "image 0.25.8", + "jsonwebtoken", + "k8s-openapi 0.27.0", + "kube 3.0.1", + "lru", + "num-traits", + "ocsp", + "once_cell", + "prometheus", + "prost 0.14.1", + "prost-build", + "qrcode", + "rand 0.8.5", + "redis", + "regex", + "reqwest 0.12.23", + "rust_decimal", + "rustls 0.23.32", + "secrecy 0.8.0", + "serde", + "serde_json", + "sha1", + "sha2", + "sqlx", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tokio-test", + "tonic 0.14.2", + "tonic-health", + "tonic-prost", + "tonic-prost-build", + "tonic-reflection", + "tonic-web", + "totp-rs", + "tower 0.4.13", + "tower-http 0.5.2", + "tower-layer", + "tower-service", + "tracing", + "tracing-subscriber", + "trading_engine", + "urlencoding", + "uuid", + "x509-parser", + "zeroize", +] + [[package]] name = "api-gateway" version = "1.0.0" @@ -229,7 +306,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-test", - "tonic", + "tonic 0.14.2", "tonic-health", "tonic-prost", "tonic-prost-build", @@ -273,7 +350,7 @@ dependencies = [ "tokio", "tokio-stream", "toml", - "tonic", + "tonic 0.14.2", "tonic-prost", "tracing", "tracing-subscriber", @@ -1568,7 +1645,7 @@ dependencies = [ "tokio-stream", "tokio-test", "tokio-util", - "tonic", + "tonic 0.14.2", "tonic-health", "tonic-prost", "tonic-prost-build", @@ -1825,7 +1902,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-test", - "tonic", + "tonic 0.14.2", "tonic-health", "tonic-prost", "tonic-prost-build", @@ -2329,7 +2406,7 @@ dependencies = [ "tokio", "tokio-test", "toml", - "tonic", + "tonic 0.14.2", "tower-layer", "tower-service", "tracing", @@ -3039,7 +3116,7 @@ dependencies = [ "tokio-retry", "tokio-stream", "tokio-util", - "tonic", + "tonic 0.14.2", "tonic-health", "tonic-prost", "tonic-prost-build", @@ -3929,7 +4006,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tokio-stream", - "tonic", + "tonic 0.14.2", "tracing", "tracing-subscriber", "trading_engine", @@ -3970,7 +4047,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-test", - "tonic", + "tonic 0.14.2", "tonic-prost", "tonic-prost-build", "tracing", @@ -4182,7 +4259,7 @@ dependencies = [ "tokio-test", "tokio-util", "toml", - "tonic", + "tonic 0.14.2", "tonic-prost", "tonic-prost-build", "tracing", @@ -5255,7 +5332,7 @@ dependencies = [ "prost-build", "reqwest 0.12.23", "tokio", - "tonic", + "tonic 0.14.2", "tonic-prost", "tonic-prost-build", "uuid", @@ -5281,7 +5358,7 @@ dependencies = [ "serial_test", "thiserror 1.0.69", "tokio", - "tonic", + "tonic 0.14.2", "tonic-prost", "tonic-prost-build", "tracing", @@ -6389,7 +6466,7 @@ dependencies = [ "tokio-retry", "tokio-stream", "tokio-util", - "tonic", + "tonic 0.14.2", "tonic-health", "tonic-prost", "tonic-prost-build", @@ -6958,7 +7035,7 @@ dependencies = [ "prost 0.14.1", "thiserror 2.0.17", "tokio", - "tonic", + "tonic 0.14.2", ] [[package]] @@ -6970,7 +7047,7 @@ dependencies = [ "opentelemetry", "opentelemetry_sdk", "prost 0.14.1", - "tonic", + "tonic 0.14.2", "tonic-prost", ] @@ -10056,7 +10133,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "tokio", - "tonic", + "tonic 0.14.2", "tonic-prost", "tracing", "tracing-subscriber", @@ -10444,7 +10521,7 @@ dependencies = [ "tokio-stream", "tokio-test", "toml", - "tonic", + "tonic 0.14.2", "tonic-health", "tracing", "tracing-subscriber", @@ -10847,6 +10924,25 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tonic" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "base64 0.22.1", + "bytes", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project", + "tokio-stream", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tonic" version = "0.14.2" @@ -10899,7 +10995,7 @@ dependencies = [ "prost 0.14.1", "tokio", "tokio-stream", - "tonic", + "tonic 0.14.2", "tonic-prost", ] @@ -10911,7 +11007,7 @@ checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67" dependencies = [ "bytes", "prost 0.14.1", - "tonic", + "tonic 0.14.2", ] [[package]] @@ -10940,10 +11036,28 @@ dependencies = [ "prost-types", "tokio", "tokio-stream", - "tonic", + "tonic 0.14.2", "tonic-prost", ] +[[package]] +name = "tonic-web" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774cad0f35370f81b6c59e3a1f5d0c3188bdb4a2a1b8b7f0921c860bfbd3aec6" +dependencies = [ + "base64 0.22.1", + "bytes", + "http 1.3.1", + "http-body 1.0.1", + "pin-project", + "tokio-stream", + "tonic 0.13.1", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "totp-rs" version = "5.7.0" @@ -11208,7 +11322,7 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tokio-stream", - "tonic", + "tonic 0.14.2", "tonic-health", "tonic-prost", "tonic-prost-build", @@ -11297,7 +11411,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-tungstenite", - "tonic", + "tonic 0.14.2", "tonic-health", "tonic-prost", "tonic-prost-build", @@ -11397,7 +11511,7 @@ dependencies = [ "tokio", "tokio-stream", "toml", - "tonic", + "tonic 0.14.2", "tonic-prost", "tonic-prost-build", "tracing", @@ -11419,7 +11533,7 @@ dependencies = [ "serde", "serde_json", "tokio", - "tonic", + "tonic 0.14.2", "tonic-prost", "tonic-prost-build", "tracing", @@ -11904,7 +12018,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-test", - "tonic", + "tonic 0.14.2", "tonic-prost", "tonic-prost-build", "tower 0.4.13", diff --git a/Cargo.toml b/Cargo.toml index e0c460988..66b51a328 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,6 +132,7 @@ members = [ "services/data_acquisition_service", "services/trading_agent_service", "services/api_gateway", + "services/api", # Testing "testing/integration", "testing/e2e", diff --git a/services/api/.sqlx/query-040b9e27fe399c9f581f93966d753eb03952985b7cc1b23e219c444eed0159fb.json b/services/api/.sqlx/query-040b9e27fe399c9f581f93966d753eb03952985b7cc1b23e219c444eed0159fb.json new file mode 100644 index 000000000..be64b4d45 --- /dev/null +++ b/services/api/.sqlx/query-040b9e27fe399c9f581f93966d753eb03952985b7cc1b23e219c444eed0159fb.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE mfa_backup_codes SET is_used = true WHERE user_id = $1 AND is_used = false", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "040b9e27fe399c9f581f93966d753eb03952985b7cc1b23e219c444eed0159fb" +} diff --git a/services/api/.sqlx/query-1368d36645c2548f0e4fb545b0cc2ce019140db89e57baddc1942b977ab3a431.json b/services/api/.sqlx/query-1368d36645c2548f0e4fb545b0cc2ce019140db89e57baddc1942b977ab3a431.json new file mode 100644 index 000000000..3dc4533f3 --- /dev/null +++ b/services/api/.sqlx/query-1368d36645c2548f0e4fb545b0cc2ce019140db89e57baddc1942b977ab3a431.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO mfa_backup_codes (\n id, user_id, code_hash, code_hint, expires_at\n ) VALUES ($1, $2, $3, $4, $5)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid", + "Varchar", + "Varchar", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "1368d36645c2548f0e4fb545b0cc2ce019140db89e57baddc1942b977ab3a431" +} diff --git a/services/api/.sqlx/query-3f7b62f1896d9d19ba56adde7119638d68a9cf64a78160e9e1ea5c9b2a0bbbb0.json b/services/api/.sqlx/query-3f7b62f1896d9d19ba56adde7119638d68a9cf64a78160e9e1ea5c9b2a0bbbb0.json new file mode 100644 index 000000000..22a2a6fca --- /dev/null +++ b/services/api/.sqlx/query-3f7b62f1896d9d19ba56adde7119638d68a9cf64a78160e9e1ea5c9b2a0bbbb0.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n id, code_hint as hint, is_used, used_at,\n used_from_ip::text as \"used_from_ip_str\", expires_at, created_at\n FROM mfa_backup_codes\n WHERE user_id = $1\n ORDER BY created_at DESC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "hint", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "is_used", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "used_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "used_from_ip_str", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "expires_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + false, + true, + null, + false, + false + ] + }, + "hash": "3f7b62f1896d9d19ba56adde7119638d68a9cf64a78160e9e1ea5c9b2a0bbbb0" +} diff --git a/services/api/.sqlx/query-6b774bb2e56924adfbd961d335fbf319b34186b30f241359d00440d8816c8772.json b/services/api/.sqlx/query-6b774bb2e56924adfbd961d335fbf319b34186b30f241359d00440d8816c8772.json new file mode 100644 index 000000000..656fc99e1 --- /dev/null +++ b/services/api/.sqlx/query-6b774bb2e56924adfbd961d335fbf319b34186b30f241359d00440d8816c8772.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n COUNT(*) FILTER (WHERE is_used = false) as \"remaining!\",\n COUNT(*) FILTER (WHERE is_used = true) as \"used!\",\n MIN(expires_at) FILTER (WHERE is_used = false) as \"earliest_expiry: chrono::DateTime\"\n FROM mfa_backup_codes\n WHERE user_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "remaining!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "used!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "earliest_expiry: chrono::DateTime", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "6b774bb2e56924adfbd961d335fbf319b34186b30f241359d00440d8816c8772" +} diff --git a/services/api/.sqlx/query-aea06694ee3e20e90795f0b3a9c43bfd0a233ea5223af844e5b2c402938db416.json b/services/api/.sqlx/query-aea06694ee3e20e90795f0b3a9c43bfd0a233ea5223af844e5b2c402938db416.json new file mode 100644 index 000000000..ef5f0b022 --- /dev/null +++ b/services/api/.sqlx/query-aea06694ee3e20e90795f0b3a9c43bfd0a233ea5223af844e5b2c402938db416.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO mfa_config (\n id, user_id, totp_secret_encrypted, totp_algorithm, totp_digits, totp_period,\n is_enabled, is_verified, enrolled_at, verified_at, backup_codes_remaining\n ) VALUES ($1, $2, $3, 'SHA1', 6, 30, true, true, $4, $4, 10)\n ON CONFLICT (user_id) DO UPDATE SET\n totp_secret_encrypted = EXCLUDED.totp_secret_encrypted,\n is_enabled = true,\n is_verified = true,\n verified_at = $4,\n backup_codes_remaining = 10\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid", + "Bytea", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "aea06694ee3e20e90795f0b3a9c43bfd0a233ea5223af844e5b2c402938db416" +} diff --git a/services/api/.sqlx/query-c00246b33b871002aa0a1a1a098ced5b5057a1e7cc7a4bf8a4200d4b5777a73b.json b/services/api/.sqlx/query-c00246b33b871002aa0a1a1a098ced5b5057a1e7cc7a4bf8a4200d4b5777a73b.json new file mode 100644 index 000000000..18639aa84 --- /dev/null +++ b/services/api/.sqlx/query-c00246b33b871002aa0a1a1a098ced5b5057a1e7cc7a4bf8a4200d4b5777a73b.json @@ -0,0 +1,82 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n id, user_id, is_enabled, is_verified,\n enrolled_at as \"enrolled_at: chrono::DateTime\",\n verified_at as \"verified_at: chrono::DateTime\",\n last_used_at as \"last_used_at: chrono::DateTime\",\n backup_codes_remaining, failed_verification_attempts,\n last_failed_attempt_at as \"last_failed_attempt_at: chrono::DateTime\",\n locked_until as \"locked_until: chrono::DateTime\"\n FROM mfa_config\n WHERE user_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "user_id", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "is_enabled", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "is_verified", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "enrolled_at: chrono::DateTime", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "verified_at: chrono::DateTime", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "last_used_at: chrono::DateTime", + "type_info": "Timestamptz" + }, + { + "ordinal": 7, + "name": "backup_codes_remaining", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "failed_verification_attempts", + "type_info": "Int4" + }, + { + "ordinal": 9, + "name": "last_failed_attempt_at: chrono::DateTime", + "type_info": "Timestamptz" + }, + { + "ordinal": 10, + "name": "locked_until: chrono::DateTime", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + true, + true, + false, + false, + true, + true + ] + }, + "hash": "c00246b33b871002aa0a1a1a098ced5b5057a1e7cc7a4bf8a4200d4b5777a73b" +} diff --git a/services/api/.sqlx/query-dc585d296075abbee6d32a08ca49bfe047c0151433c2359e51ad3e3c5e878f6a.json b/services/api/.sqlx/query-dc585d296075abbee6d32a08ca49bfe047c0151433c2359e51ad3e3c5e878f6a.json new file mode 100644 index 000000000..b9dd1eb50 --- /dev/null +++ b/services/api/.sqlx/query-dc585d296075abbee6d32a08ca49bfe047c0151433c2359e51ad3e3c5e878f6a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE mfa_enrollment_sessions SET is_active = false, completed_at = NOW() WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "dc585d296075abbee6d32a08ca49bfe047c0151433c2359e51ad3e3c5e878f6a" +} diff --git a/services/api/.sqlx/query-e19dc7e9161ae2510850e709c7fda7c398662e8081e3a7929132a2031555dc5a.json b/services/api/.sqlx/query-e19dc7e9161ae2510850e709c7fda7c398662e8081e3a7929132a2031555dc5a.json new file mode 100644 index 000000000..d19fdf0c8 --- /dev/null +++ b/services/api/.sqlx/query-e19dc7e9161ae2510850e709c7fda7c398662e8081e3a7929132a2031555dc5a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE mfa_enrollment_sessions SET verification_attempts = verification_attempts + 1 WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "e19dc7e9161ae2510850e709c7fda7c398662e8081e3a7929132a2031555dc5a" +} diff --git a/services/api/.sqlx/query-ea9264a98b2bf6034a0ecaf22903b4e2d2647303c3e7a480ccea2df6359dc943.json b/services/api/.sqlx/query-ea9264a98b2bf6034a0ecaf22903b4e2d2647303c3e7a480ccea2df6359dc943.json new file mode 100644 index 000000000..b1d7b7b50 --- /dev/null +++ b/services/api/.sqlx/query-ea9264a98b2bf6034a0ecaf22903b4e2d2647303c3e7a480ccea2df6359dc943.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO mfa_enrollment_sessions (\n id, user_id, temp_totp_secret_encrypted, qr_code_data,\n is_active, expires_at\n ) VALUES ($1, $2, $3, $4, true, $5)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid", + "Bytea", + "Text", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "ea9264a98b2bf6034a0ecaf22903b4e2d2647303c3e7a480ccea2df6359dc943" +} diff --git a/services/api/.sqlx/query-ed947b6e0201c32cd49d191293906361647f52ab989fd8a7fedcbb2a66748355.json b/services/api/.sqlx/query-ed947b6e0201c32cd49d191293906361647f52ab989fd8a7fedcbb2a66748355.json new file mode 100644 index 000000000..957a69a86 --- /dev/null +++ b/services/api/.sqlx/query-ed947b6e0201c32cd49d191293906361647f52ab989fd8a7fedcbb2a66748355.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE mfa_config SET is_enabled = false, updated_at = NOW() WHERE user_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "ed947b6e0201c32cd49d191293906361647f52ab989fd8a7fedcbb2a66748355" +} diff --git a/services/api/.sqlx/query-fa8446a8e2163a642e241866e04bd55d93c3ee12244ac0aaf9489f41c35c9189.json b/services/api/.sqlx/query-fa8446a8e2163a642e241866e04bd55d93c3ee12244ac0aaf9489f41c35c9189.json new file mode 100644 index 000000000..b824db34f --- /dev/null +++ b/services/api/.sqlx/query-fa8446a8e2163a642e241866e04bd55d93c3ee12244ac0aaf9489f41c35c9189.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n temp_totp_secret_encrypted, \n is_active, \n expires_at as \"expires_at: chrono::DateTime\", \n verification_attempts\n FROM mfa_enrollment_sessions\n WHERE id = $1 AND user_id = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "temp_totp_secret_encrypted", + "type_info": "Bytea" + }, + { + "ordinal": 1, + "name": "is_active", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "expires_at: chrono::DateTime", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "verification_attempts", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "fa8446a8e2163a642e241866e04bd55d93c3ee12244ac0aaf9489f41c35c9189" +} diff --git a/services/api/Cargo.toml b/services/api/Cargo.toml new file mode 100644 index 000000000..5d793d69b --- /dev/null +++ b/services/api/Cargo.toml @@ -0,0 +1,169 @@ +[package] +name = "api" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +description = "Foxhunt API Service — unified gRPC gateway with tonic-web for browser access" + +[[bin]] +name = "api" +path = "src/main.rs" + +[dependencies] +# Core async and utilities +tokio = { workspace = true, features = ["sync", "time"] } +anyhow.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +serde.workspace = true +serde_json.workspace = true +once_cell.workspace = true +clap.workspace = true + +# gRPC and networking (Tonic 0.14) +tonic = { workspace = true, features = ["transport", "server", "tls-ring", "tls-webpki-roots"] } +tonic-prost.workspace = true +tonic-reflection.workspace = true +tonic-health.workspace = true +prost.workspace = true +tower = { workspace = true, features = ["util"] } +tower-layer.workspace = true +tower-service.workspace = true +hyper.workspace = true +http-body.workspace = true +http-body-util.workspace = true +hyper-util.workspace = true +bytes.workspace = true + +# Async streams and futures +tokio-stream.workspace = true +async-stream.workspace = true +futures.workspace = true +async-trait.workspace = true + +# Performance monitoring +hdrhistogram.workspace = true +prometheus = { workspace = true, features = ["process"] } + +# Cryptography and security +sha2.workspace = true +x509-parser = { version = "0.16", features = ["verify"] } +reqwest = { version = "0.12", features = ["rustls-tls"], default-features = false } +base64.workspace = true +jsonwebtoken.workspace = true +chrono.workspace = true +ocsp = "0.4" +lru = "0.12" +hex = "0.4" +const-oid = "0.9" + +# MFA/TOTP dependencies (optional, enabled by default via "mfa" feature) +totp-rs = { version = "5.6", optional = true } +qrcode = { version = "0.14", optional = true } +image = { version = "0.25", optional = true } +base32 = { version = "0.5", optional = true } +hmac = { version = "0.12", optional = true } +sha1 = "0.10" +urlencoding = { version = "2.1", optional = true } +secrecy = { version = "0.8", features = ["serde"] } +zeroize.workspace = true + +thiserror.workspace = true +uuid.workspace = true +sqlx = { workspace = true, features = ["postgres", "chrono", "uuid", "json", "macros"] } + +num-traits.workspace = true +rust_decimal.workspace = true +rand.workspace = true + +# Internal workspace crates +trading_engine.workspace = true +common = { workspace = true, features = ["database"] } +config = { workspace = true, features = ["postgres"] } + +# Redis for JWT revocation and rate limiting +redis = { workspace = true, features = ["tokio-comp", "connection-manager"] } + +# Kubernetes API client for pod listing +kube = { version = "3.0", features = ["client", "runtime", "derive"] } +k8s-openapi = { version = "0.27", features = ["latest"] } + +# Rate limiting +governor = "0.6" +dashmap = "6.0" + +# HTTP and networking +http = "1.1" + +# Regex for validation +regex = "1.11" + +# Circuit breaker for fault tolerance (using tower's timeout + buffer for now) +# Note: Full circuit breaker can be implemented using tower-layer + +# TLS crypto provider for kube-rs in-cluster client +rustls = { version = "0.23", default-features = false, features = ["ring"] } + +# HTTP server for Prometheus metrics endpoint +axum = "0.7" + +# gRPC-Web support for browser access +tonic-web = "0.13" +tower-http = { workspace = true, features = ["cors"] } + +[build-dependencies] +tonic-prost-build.workspace = true +prost-build.workspace = true + +[dev-dependencies] +criterion = { version = "0.5", features = ["html_reports"] } +tokio-test = "0.4" +fxt.workspace = true # Required for proto definitions in tests + +[features] +default = ["minimal", "mfa"] +minimal = [] +database = [] +mfa = ["dep:totp-rs", "dep:qrcode", "dep:image", "dep:base32", "dep:hmac", "dep:urlencoding"] + +[[bench]] +name = "rate_limiter_bench" +harness = false + +[[bench]] +name = "auth_overhead" +harness = false + +[[bench]] +name = "routing_latency" +harness = false + +[[bench]] +name = "rate_limiting_perf" +harness = false + +[[bench]] +name = "cache_performance" +harness = false + +[[bench]] +name = "throughput" +harness = false + +[[bench]] +name = "revocation_cache_perf" +harness = false + +[[bench]] +name = "authz_dashmap_benchmark" +harness = false + +[[bench]] +name = "dashmap_rate_limiter_bench" +harness = false + +[[bench]] +name = "proxy_latency" +harness = false diff --git a/services/api/Dockerfile b/services/api/Dockerfile new file mode 100644 index 000000000..bda096129 --- /dev/null +++ b/services/api/Dockerfile @@ -0,0 +1,82 @@ +# Multi-stage build for Foxhunt API Service +# Production-optimized Docker image + +# ============================================================================= +# Builder Stage +# ============================================================================= +FROM rust:1.89-slim-bookworm AS builder + +# Install build dependencies +RUN apt-get update && apt-get install -y \ + pkg-config \ + libssl-dev \ + protobuf-compiler \ + perl \ + make \ + && rm -rf /var/lib/apt/lists/* + +# Set working directory +WORKDIR /build + +# Copy workspace manifests +COPY Cargo.toml Cargo.lock ./ + +# Copy sqlx offline cache for compile-time query verification +COPY .sqlx ./.sqlx + +# Copy workspace members to satisfy manifest dependencies +COPY crates ./crates +COPY bin/fxt ./bin/fxt +COPY services ./services +COPY testing ./testing + +# Enable sqlx offline mode to use cached query metadata +ENV SQLX_OFFLINE=true + +# Build the application +RUN cargo build --release -p api + +# ============================================================================= +# Runtime Stage +# ============================================================================= +FROM debian:bookworm-slim + +# Install runtime dependencies +RUN apt-get update && apt-get install -y \ + ca-certificates \ + libssl3 \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Download and install grpc_health_probe for health checks +RUN curl -sSL https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.4.25/grpc_health_probe-linux-amd64 \ + -o /usr/local/bin/grpc_health_probe && \ + chmod +x /usr/local/bin/grpc_health_probe + +# Create non-root user +RUN groupadd --system --gid 1000 foxhunt && \ + useradd --system --uid 1000 --gid foxhunt --shell /bin/bash foxhunt + +# Create application directories +RUN mkdir -p /app/config /app/logs && \ + chown -R foxhunt:foxhunt /app + +# Set working directory +WORKDIR /app + +# Copy binary from builder +COPY --from=builder /build/target/release/api ./api +RUN chmod +x ./api + +# Switch to non-root user +USER foxhunt + +# Expose gRPC and metrics ports +EXPOSE 50051 9091 + +# Health check using grpc_health_probe +HEALTHCHECK --interval=10s --timeout=5s --start-period=30s --retries=3 \ + CMD /usr/local/bin/grpc_health_probe -addr=localhost:50051 || exit 1 + +# Run the application +ENTRYPOINT ["./api"] diff --git a/services/api/README.md b/services/api/README.md new file mode 100644 index 000000000..a5ff704b5 --- /dev/null +++ b/services/api/README.md @@ -0,0 +1,22 @@ +# api + +Foxhunt API Service -- unified gRPC gateway with tonic-web for browser access, 6-layer authentication, RBAC, rate limiting, and gRPC proxy routing. + +## Key Types + +- `TradingServiceProxy` -- main gRPC proxy implementation +- `RoleBasedAccessControl` -- RBAC authorization +- `RateLimiter` -- 3-tier rate limiting (auth/trading/compute) +- `CircuitBreaker` -- upstream service protection + +## Configuration + +- `JWT_SECRET` -- JWT signing secret (min 32 chars) +- `CORS_ORIGINS` -- comma-separated allowed origins for gRPC-Web (default: http://localhost:5173) +- `TRADING_SERVICE_URL` -- trading service gRPC endpoint +- `ML_TRAINING_SERVICE_URL` -- ML training service gRPC endpoint + +## Features + +- `mfa` (default) -- multi-factor authentication support +- Build without: `cargo check -p api --no-default-features --features minimal` diff --git a/services/api/benches/auth_overhead.rs b/services/api/benches/auth_overhead.rs new file mode 100644 index 000000000..a4f20a80e --- /dev/null +++ b/services/api/benches/auth_overhead.rs @@ -0,0 +1,384 @@ +//! Auth Overhead Benchmark - 8-Layer Authentication Pipeline +//! +//! Measures performance of each authentication layer: +//! 1. JWT Extraction (<100ns target) +//! 2. JWT Signature Validation (<1μs target) +//! 3. JWT Revocation Check (<500ns target) +//! 4. RBAC Permission Check (<100ns target) +//! 5. Rate Limiting (<50ns target) +//! 6. Audit Logging (non-blocking) +//! 7. User Context Injection (<50ns target) +//! 8. Metrics Recording (<20ns target) +//! +//! Total Target: <10μs end-to-end + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use jsonwebtoken::{decode, encode, Algorithm, DecodingKey, EncodingKey, Header, Validation}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// JWT Claims structure +#[derive(Debug, Serialize, Deserialize, Clone)] +struct JwtClaims { + sub: String, + jti: String, + exp: u64, + nbf: u64, + iat: u64, + iss: String, + aud: String, + roles: Vec, + permissions: Vec, +} + +/// Mock revocation cache (simulates Redis) +struct RevocationCache { + blacklist: HashMap, +} + +impl RevocationCache { + fn new() -> Self { + Self { + blacklist: HashMap::new(), + } + } + + fn is_revoked(&self, jti: &str) -> bool { + self.blacklist.get(jti).copied().unwrap_or(false) + } +} + +/// Mock RBAC cache +struct RbacCache { + permissions: HashMap>, +} + +impl RbacCache { + fn new() -> Self { + let mut permissions = HashMap::new(); + permissions.insert( + "user123".to_string(), + vec![ + "trade:read".to_string(), + "trade:write".to_string(), + "portfolio:read".to_string(), + ], + ); + Self { permissions } + } + + fn has_permission(&self, user_id: &str, permission: &str) -> bool { + self.permissions + .get(user_id) + .map(|perms| perms.iter().any(|p| p == permission)) + .unwrap_or(false) + } +} + +/// Rate limiter with atomic counters +struct RateLimiter { + counter: Arc, + limit: u64, +} + +impl RateLimiter { + fn new(limit: u64) -> Self { + Self { + counter: Arc::new(AtomicU64::new(0)), + limit, + } + } + + fn check_limit(&self, _user_id: &str) -> bool { + let count = self.counter.fetch_add(1, Ordering::Relaxed); + count < self.limit + } +} + +/// Helper function to create test JWT +fn create_test_jwt(secret: &str) -> String { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + + let claims = JwtClaims { + sub: "user123".to_string(), + jti: "token-id-12345".to_string(), + exp: now + 3600, + nbf: now, + iat: now, + iss: "foxhunt-api".to_string(), + aud: "trading-service".to_string(), + roles: vec!["trader".to_string()], + permissions: vec![ + "trade:read".to_string(), + "trade:write".to_string(), + "portfolio:read".to_string(), + ], + }; + + encode( + &Header::new(Algorithm::HS256), + &claims, + &EncodingKey::from_secret(secret.as_bytes()), + ) + .unwrap() +} + +/// Benchmark 1: JWT Extraction from Authorization Header +fn bench_jwt_extraction(c: &mut Criterion) { + let token = create_test_jwt("test-secret-key-32-bytes-long!"); + let auth_header = format!("Bearer {}", token); + + c.bench_function("jwt_extraction", |b| { + b.iter(|| { + let header = black_box(&auth_header); + let extracted = header.strip_prefix("Bearer ").unwrap_or(""); + black_box(extracted); + }); + }); +} + +/// Benchmark 2: JWT Signature Validation (TARGET: <1μs) +fn bench_jwt_validation(c: &mut Criterion) { + let secret = "test-secret-key-32-bytes-long!"; + let token = create_test_jwt(secret); + let decoding_key = DecodingKey::from_secret(secret.as_bytes()); + + let mut validation = Validation::new(Algorithm::HS256); + validation.set_issuer(&["foxhunt-api"]); + validation.set_audience(&["trading-service"]); + + c.bench_function("jwt_signature_validation", |b| { + b.iter(|| { + let result = decode::(black_box(&token), &decoding_key, &validation); + black_box(result); + }); + }); +} + +/// Benchmark 3: JWT Revocation Check (TARGET: <500ns) +fn bench_revocation_check(c: &mut Criterion) { + let cache = RevocationCache::new(); + let jti = "token-id-12345"; + + c.bench_function("revocation_check_cache_hit", |b| { + b.iter(|| { + let is_revoked = cache.is_revoked(black_box(jti)); + black_box(is_revoked); + }); + }); +} + +/// Benchmark 4: RBAC Permission Check (TARGET: <100ns) +fn bench_rbac_check(c: &mut Criterion) { + let cache = RbacCache::new(); + let user_id = "user123"; + let permission = "trade:write"; + + c.bench_function("rbac_permission_check", |b| { + b.iter(|| { + let has_perm = cache.has_permission(black_box(user_id), black_box(permission)); + black_box(has_perm); + }); + }); +} + +/// Benchmark 5: Rate Limiting Check (TARGET: <50ns) +fn bench_rate_limiting(c: &mut Criterion) { + let limiter = RateLimiter::new(1_000_000); + let user_id = "user123"; + + c.bench_function("rate_limit_check", |b| { + b.iter(|| { + let allowed = limiter.check_limit(black_box(user_id)); + black_box(allowed); + }); + }); +} + +/// Benchmark 6: User Context Creation (TARGET: <50ns) +fn bench_user_context_creation(c: &mut Criterion) { + let claims = JwtClaims { + sub: "user123".to_string(), + jti: "token-id-12345".to_string(), + exp: 1234567890, + nbf: 1234567800, + iat: 1234567800, + iss: "foxhunt-api".to_string(), + aud: "trading-service".to_string(), + roles: vec!["trader".to_string()], + permissions: vec!["trade:read".to_string(), "trade:write".to_string()], + }; + + c.bench_function("user_context_creation", |b| { + b.iter(|| { + let context = ( + black_box(&claims.sub), + black_box(&claims.roles), + black_box(&claims.permissions), + ); + black_box(context); + }); + }); +} + +/// Benchmark 7: Full 8-Layer Pipeline (TARGET: <10μs) +fn bench_full_auth_pipeline(c: &mut Criterion) { + let secret = "test-secret-key-32-bytes-long!"; + let token = create_test_jwt(secret); + let auth_header = format!("Bearer {}", token); + + let decoding_key = DecodingKey::from_secret(secret.as_bytes()); + let mut validation = Validation::new(Algorithm::HS256); + validation.set_issuer(&["foxhunt-api"]); + validation.set_audience(&["trading-service"]); + + let revocation_cache = RevocationCache::new(); + let rbac_cache = RbacCache::new(); + let rate_limiter = RateLimiter::new(1_000_000); + + c.bench_function("8_layer_auth_pipeline", |b| { + b.iter(|| { + // Layer 1: Extract JWT + let token_str = black_box(&auth_header).strip_prefix("Bearer ").unwrap(); + + // Layer 2: Validate JWT signature + let token_data = decode::(token_str, &decoding_key, &validation).unwrap(); + + // Layer 3: Check revocation + let is_revoked = revocation_cache.is_revoked(&token_data.claims.jti); + assert!(!is_revoked); + + // Layer 4: Check RBAC permissions + let has_permission = rbac_cache.has_permission(&token_data.claims.sub, "trade:write"); + assert!(has_permission); + + // Layer 5: Check rate limit + let allowed = rate_limiter.check_limit(&token_data.claims.sub); + assert!(allowed); + + // Layer 6: Create user context (metadata injection) + let _context = ( + &token_data.claims.sub, + &token_data.claims.roles, + &token_data.claims.permissions, + ); + + // Layer 7: Audit logging (simulated - non-blocking) + // (In production, this would be async) + + // Layer 8: Metrics recording + // (In production, this would increment Prometheus counters) + + black_box(()); + }); + }); +} + +/// Benchmark 8: Different JWT Sizes +fn bench_jwt_sizes(c: &mut Criterion) { + let secret = "test-secret-key-32-bytes-long!"; + let decoding_key = DecodingKey::from_secret(secret.as_bytes()); + let mut validation = Validation::new(Algorithm::HS256); + validation.set_issuer(&["foxhunt-api"]); + validation.set_audience(&["trading-service"]); + + let mut group = c.benchmark_group("jwt_validation_by_size"); + + // Small JWT (minimal claims) + let small_jwt = { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + let claims = JwtClaims { + sub: "user123".to_string(), + jti: "token-id".to_string(), + exp: now + 3600, + nbf: now, + iat: now, + iss: "foxhunt-api".to_string(), + aud: "trading-service".to_string(), + roles: vec!["trader".to_string()], + permissions: vec!["trade:read".to_string()], + }; + encode( + &Header::new(Algorithm::HS256), + &claims, + &EncodingKey::from_secret(secret.as_bytes()), + ) + .unwrap() + }; + + // Large JWT (many permissions) + let large_jwt = { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + let claims = JwtClaims { + sub: "user123".to_string(), + jti: "token-id-very-long-identifier-12345".to_string(), + exp: now + 3600, + nbf: now, + iat: now, + iss: "foxhunt-api".to_string(), + aud: "trading-service".to_string(), + roles: vec![ + "trader".to_string(), + "admin".to_string(), + "analyst".to_string(), + ], + permissions: (0..50).map(|i| format!("permission:{}", i)).collect(), + }; + encode( + &Header::new(Algorithm::HS256), + &claims, + &EncodingKey::from_secret(secret.as_bytes()), + ) + .unwrap() + }; + + group.bench_with_input( + BenchmarkId::new("jwt_validation", "small"), + &small_jwt, + |b, jwt| { + b.iter(|| { + let result = decode::(black_box(jwt), &decoding_key, &validation); + black_box(result); + }); + }, + ); + + group.bench_with_input( + BenchmarkId::new("jwt_validation", "large"), + &large_jwt, + |b, jwt| { + b.iter(|| { + let result = decode::(black_box(jwt), &decoding_key, &validation); + black_box(result); + }); + }, + ); + + group.finish(); +} + +criterion_group!( + auth_benches, + bench_jwt_extraction, + bench_jwt_validation, + bench_revocation_check, + bench_rbac_check, + bench_rate_limiting, + bench_user_context_creation, + bench_full_auth_pipeline, + bench_jwt_sizes +); + +criterion_main!(auth_benches); diff --git a/services/api/benches/authz_dashmap_benchmark.rs b/services/api/benches/authz_dashmap_benchmark.rs new file mode 100644 index 000000000..d0648fdb2 --- /dev/null +++ b/services/api/benches/authz_dashmap_benchmark.rs @@ -0,0 +1,341 @@ +//! Authorization Service Performance Benchmark +//! +//! Compares performance between: +//! - RwLock (baseline) +//! - DashMap (optimized) +//! +//! Target: <8ns per RBAC check (12x improvement from ~100ns baseline) + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use dashmap::DashMap; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::time::Instant; +use tokio::sync::RwLock; +use uuid::Uuid; + +/// User permissions structure +#[derive(Clone, Debug)] +struct UserPermissions { + user_id: Uuid, + permissions: HashSet, + loaded_at: Instant, +} + +/// RwLock-based cache (baseline) +struct RwLockAuthzCache { + cache: Arc>>, +} + +impl RwLockAuthzCache { + fn new() -> Self { + Self { + cache: Arc::new(RwLock::new(HashMap::new())), + } + } + + async fn check_permission(&self, user_id: &Uuid, endpoint: &str) -> bool { + let cache = self.cache.read().await; + if let Some(user_perms) = cache.get(user_id) { + user_perms.permissions.contains(endpoint) + } else { + false + } + } + + async fn insert(&self, user_id: Uuid, perms: UserPermissions) { + let mut cache = self.cache.write().await; + cache.insert(user_id, perms); + } +} + +/// DashMap-based cache (optimized) +struct DashMapAuthzCache { + cache: Arc>, +} + +impl DashMapAuthzCache { + fn new() -> Self { + Self { + cache: Arc::new(DashMap::new()), + } + } + + fn check_permission(&self, user_id: &Uuid, endpoint: &str) -> bool { + if let Some(user_perms_ref) = self.cache.get(user_id) { + user_perms_ref.permissions.contains(endpoint) + } else { + false + } + } + + fn insert(&self, user_id: Uuid, perms: UserPermissions) { + self.cache.insert(user_id, perms); + } +} + +/// Benchmark: RwLock cache read (baseline) +fn bench_rwlock_read(c: &mut Criterion) { + let rt = tokio::runtime::Runtime::new().unwrap(); + let cache = RwLockAuthzCache::new(); + + // Prepopulate with 1000 users + rt.block_on(async { + for i in 0..1000 { + let user_id = Uuid::new_v4(); + let perms = UserPermissions { + user_id, + permissions: vec![ + "/api/trade".to_string(), + "/api/portfolio".to_string(), + "/api/risk".to_string(), + ] + .into_iter() + .collect(), + loaded_at: Instant::now(), + }; + cache.insert(user_id, perms).await; + } + }); + + // Get a user ID for benchmarking + let test_user_id = rt.block_on(async { + let cache_guard = cache.cache.read().await; + cache_guard.keys().next().copied().unwrap() + }); + + c.bench_function("rwlock_permission_check", |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + rt.block_on(async { + for _ in 0..iters { + let result = cache + .check_permission(black_box(&test_user_id), black_box("/api/trade")) + .await; + black_box(result); + } + }); + start.elapsed() + }) + }); +} + +/// Benchmark: DashMap cache read (optimized) +fn bench_dashmap_read(c: &mut Criterion) { + let cache = DashMapAuthzCache::new(); + + // Prepopulate with 1000 users + for i in 0..1000 { + let user_id = Uuid::new_v4(); + let perms = UserPermissions { + user_id, + permissions: vec![ + "/api/trade".to_string(), + "/api/portfolio".to_string(), + "/api/risk".to_string(), + ] + .into_iter() + .collect(), + loaded_at: Instant::now(), + }; + cache.insert(user_id, perms); + } + + // Get a user ID for benchmarking + let test_user_id = cache.cache.iter().next().expect("INVARIANT: Iterator should have next element").key().clone(); + + c.bench_function("dashmap_permission_check", |b| { + b.iter(|| { + let result = cache.check_permission(black_box(&test_user_id), black_box("/api/trade")); + black_box(result); + }); + }); +} + +/// Benchmark: Compare different cache sizes +fn bench_cache_sizes(c: &mut Criterion) { + let mut group = c.benchmark_group("authz_cache_sizes"); + + for size in &[100, 1_000, 10_000, 100_000] { + // DashMap benchmark + let dashmap_cache = DashMapAuthzCache::new(); + let mut test_user_ids = Vec::new(); + + for i in 0..*size { + let user_id = Uuid::new_v4(); + test_user_ids.push(user_id); + let perms = UserPermissions { + user_id, + permissions: vec!["/api/trade".to_string(), "/api/portfolio".to_string()] + .into_iter() + .collect(), + loaded_at: Instant::now(), + }; + dashmap_cache.insert(user_id, perms); + } + + let mid_user = test_user_ids[size / 2]; + + group.bench_with_input(BenchmarkId::new("dashmap", size), size, |b, _| { + b.iter(|| { + let result = + dashmap_cache.check_permission(black_box(&mid_user), black_box("/api/trade")); + black_box(result); + }); + }); + } + + group.finish(); +} + +/// Benchmark: Concurrent reads +fn bench_concurrent_reads(c: &mut Criterion) { + let mut group = c.benchmark_group("concurrent_reads"); + + // DashMap - lock-free concurrent reads + let dashmap_cache = Arc::new(DashMapAuthzCache::new()); + let mut test_user_ids = Vec::new(); + + for i in 0..1000 { + let user_id = Uuid::new_v4(); + test_user_ids.push(user_id); + let perms = UserPermissions { + user_id, + permissions: vec!["/api/trade".to_string()].into_iter().collect(), + loaded_at: Instant::now(), + }; + dashmap_cache.insert(user_id, perms); + } + + group.bench_function("dashmap_concurrent_8_threads", |b| { + b.iter(|| { + let rt = tokio::runtime::Runtime::new().unwrap(); + let cache = Arc::clone(&dashmap_cache); + + rt.block_on(async { + let mut handles = Vec::new(); + + for i in 0..8 { + let cache_clone = Arc::clone(&cache); + let user_id = test_user_ids[i * 100]; + + let handle = tokio::spawn(async move { + for _ in 0..100 { + let result = cache_clone.check_permission(&user_id, "/api/trade"); + black_box(result); + } + }); + + handles.push(handle); + } + + for handle in handles { + handle.await.unwrap(); + } + }); + }); + }); + + group.finish(); +} + +/// Benchmark: Hot path - permission check only +fn bench_hot_path(c: &mut Criterion) { + let dashmap_cache = DashMapAuthzCache::new(); + + // Prepopulate with realistic data + let mut test_users = Vec::new(); + for i in 0..100 { + let user_id = Uuid::new_v4(); + test_users.push(user_id); + let perms = UserPermissions { + user_id, + permissions: vec![ + "/api/trade".to_string(), + "/api/portfolio".to_string(), + "/api/risk".to_string(), + "/api/market-data".to_string(), + ] + .into_iter() + .collect(), + loaded_at: Instant::now(), + }; + dashmap_cache.insert(user_id, perms); + } + + let hot_user = test_users[50]; + + c.bench_function("hot_path_permission_check", |b| { + b.iter(|| { + // Simulate typical RBAC check pattern + let has_trade = + dashmap_cache.check_permission(black_box(&hot_user), black_box("/api/trade")); + let has_portfolio = + dashmap_cache.check_permission(black_box(&hot_user), black_box("/api/portfolio")); + black_box((has_trade, has_portfolio)); + }); + }); +} + +/// Benchmark: Cache invalidation +fn bench_cache_invalidation(c: &mut Criterion) { + let mut group = c.benchmark_group("cache_invalidation"); + + // DashMap invalidation + let dashmap_cache = DashMapAuthzCache::new(); + + for i in 0..1000 { + let user_id = Uuid::new_v4(); + let perms = UserPermissions { + user_id, + permissions: vec!["/api/trade".to_string()].into_iter().collect(), + loaded_at: Instant::now(), + }; + dashmap_cache.insert(user_id, perms); + } + + let test_user = dashmap_cache.cache.iter().next().expect("INVARIANT: Iterator should have next element").key().clone(); + + group.bench_function("dashmap_remove", |b| { + b.iter(|| { + dashmap_cache.cache.remove(black_box(&test_user)); + // Re-insert for next iteration + let perms = UserPermissions { + user_id: test_user, + permissions: vec!["/api/trade".to_string()].into_iter().collect(), + loaded_at: Instant::now(), + }; + dashmap_cache.insert(test_user, perms); + }); + }); + + group.bench_function("dashmap_clear_all", |b| { + b.iter(|| { + dashmap_cache.cache.clear(); + // Repopulate for next iteration + for i in 0..100 { + let user_id = Uuid::new_v4(); + let perms = UserPermissions { + user_id, + permissions: vec!["/api/trade".to_string()].into_iter().collect(), + loaded_at: Instant::now(), + }; + dashmap_cache.insert(user_id, perms); + } + }); + }); + + group.finish(); +} + +criterion_group!( + authz_benches, + bench_rwlock_read, + bench_dashmap_read, + bench_cache_sizes, + bench_concurrent_reads, + bench_hot_path, + bench_cache_invalidation +); + +criterion_main!(authz_benches); diff --git a/services/api/benches/cache_performance.rs b/services/api/benches/cache_performance.rs new file mode 100644 index 000000000..8da412900 --- /dev/null +++ b/services/api/benches/cache_performance.rs @@ -0,0 +1,408 @@ +//! Cache Performance Benchmark +//! +//! Measures caching layer performance for: +//! - JWT validation cache (decoded tokens) +//! - RBAC permission cache +//! - Revocation list cache +//! - User session cache +//! +//! Targets: +//! - Cache hit: <100ns +//! - Cache miss: <1μs (with lookup) + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; + +/// Simple LRU cache implementation +struct LruCache { + cache: HashMap, + capacity: usize, + ttl: Duration, +} + +impl LruCache { + fn new(capacity: usize, ttl: Duration) -> Self { + Self { + cache: HashMap::with_capacity(capacity), + capacity, + ttl, + } + } + + fn get(&mut self, key: &K) -> Option { + if let Some((value, timestamp)) = self.cache.get(key) { + // Check TTL + if timestamp.elapsed() < self.ttl { + return Some(value.clone()); + } else { + // Expired + self.cache.remove(key); + } + } + None + } + + fn put(&mut self, key: K, value: V) { + // Evict if at capacity + if self.cache.len() >= self.capacity { + // Simple eviction: remove first entry + if let Some(k) = self.cache.keys().next().cloned() { + self.cache.remove(&k); + } + } + self.cache.insert(key, (value, Instant::now())); + } +} + +/// Thread-safe cache wrapper +struct ThreadSafeCache { + cache: Arc>>, +} + +impl ThreadSafeCache { + fn new(capacity: usize, ttl: Duration) -> Self { + Self { + cache: Arc::new(RwLock::new(LruCache::new(capacity, ttl))), + } + } + + fn get(&self, key: &K) -> Option { + self.cache.write().expect("INVARIANT: RwLock should not be poisoned").get(key) + } + + fn put(&self, key: K, value: V) { + self.cache.write().expect("INVARIANT: RwLock should not be poisoned").put(key, value); + } +} + +/// JWT cache entry +#[derive(Clone, Debug)] +struct JwtCacheEntry { + user_id: String, + roles: Vec, + permissions: Vec, + exp: u64, +} + +/// RBAC cache entry +#[derive(Clone, Debug)] +struct RbacCacheEntry { + permissions: Vec, +} + +/// Benchmark 1: JWT cache hit (TARGET: <100ns) +fn bench_jwt_cache_hit(c: &mut Criterion) { + let mut cache = LruCache::new(10_000, Duration::from_secs(300)); + + // Prepopulate cache + for i in 0..1000 { + let entry = JwtCacheEntry { + user_id: format!("user{}", i), + roles: vec!["trader".to_string()], + permissions: vec!["trade:read".to_string(), "trade:write".to_string()], + exp: 1234567890, + }; + cache.put(format!("jwt_token_{}", i), entry); + } + + c.bench_function("jwt_cache_hit", |b| { + b.iter(|| { + let key = format!("jwt_token_{}", black_box(500)); + let entry = cache.get(&key); + black_box(entry); + }); + }); +} + +/// Benchmark 2: JWT cache miss (with decode simulation) +fn bench_jwt_cache_miss(c: &mut Criterion) { + let mut cache = LruCache::new(10_000, Duration::from_secs(300)); + + c.bench_function("jwt_cache_miss_with_decode", |b| { + b.iter(|| { + let key = format!("jwt_token_{}", black_box(9999)); + let entry = cache.get(&key); + + if entry.is_none() { + // Simulate JWT decode (expensive operation) + let decoded = JwtCacheEntry { + user_id: "user9999".to_string(), + roles: vec!["trader".to_string()], + permissions: vec!["trade:read".to_string()], + exp: 1234567890, + }; + cache.put(key.clone(), decoded.clone()); + black_box(decoded); + } + }); + }); +} + +/// Benchmark 3: RBAC cache hit (TARGET: <100ns) +fn bench_rbac_cache_hit(c: &mut Criterion) { + let mut cache = LruCache::new(10_000, Duration::from_secs(300)); + + // Prepopulate with permissions + for i in 0..1000 { + let entry = RbacCacheEntry { + permissions: vec![ + "trade:read".to_string(), + "trade:write".to_string(), + "portfolio:read".to_string(), + ], + }; + cache.put(format!("user{}", i), entry); + } + + c.bench_function("rbac_cache_hit", |b| { + b.iter(|| { + let key = format!("user{}", black_box(500)); + let entry = cache.get(&key); + black_box(entry); + }); + }); +} + +/// Benchmark 4: Different cache sizes +fn bench_cache_sizes(c: &mut Criterion) { + let mut group = c.benchmark_group("cache_size_impact"); + + for size in &[100, 1_000, 10_000, 100_000] { + let mut cache = LruCache::new(*size, Duration::from_secs(300)); + + // Prepopulate to capacity + for i in 0..*size { + cache.put(format!("key{}", i), format!("value{}", i)); + } + + group.bench_with_input(BenchmarkId::new("cache_lookup", size), size, |b, &n| { + b.iter(|| { + let key = format!("key{}", black_box(n / 2)); + let value = cache.get(&key); + black_box(value); + }); + }); + } + + group.finish(); +} + +/// Benchmark 5: Cache eviction overhead +fn bench_cache_eviction(c: &mut Criterion) { + let mut cache = LruCache::new(100, Duration::from_secs(300)); + + // Fill to capacity + for i in 0..100 { + cache.put(format!("key{}", i), format!("value{}", i)); + } + + c.bench_function("cache_eviction_on_insert", |b| { + let mut counter = 100; + b.iter(|| { + cache.put( + format!("key{}", black_box(counter)), + format!("value{}", counter), + ); + counter += 1; + }); + }); +} + +/// Benchmark 6: TTL expiration check overhead +fn bench_ttl_expiration(c: &mut Criterion) { + let mut cache_short = LruCache::new(1000, Duration::from_millis(1)); + let mut cache_long = LruCache::new(1000, Duration::from_secs(300)); + + // Prepopulate + for i in 0..100 { + cache_short.put(format!("key{}", i), format!("value{}", i)); + cache_long.put(format!("key{}", i), format!("value{}", i)); + } + + let mut group = c.benchmark_group("ttl_expiration"); + + group.bench_function("short_ttl_1ms", |b| { + b.iter(|| { + let key = format!("key{}", black_box(50)); + let value = cache_short.get(&key); + black_box(value); + }); + }); + + group.bench_function("long_ttl_300s", |b| { + b.iter(|| { + let key = format!("key{}", black_box(50)); + let value = cache_long.get(&key); + black_box(value); + }); + }); + + group.finish(); +} + +/// Benchmark 7: Thread-safe cache with RwLock +fn bench_thread_safe_cache(c: &mut Criterion) { + let cache = ThreadSafeCache::new(10_000, Duration::from_secs(300)); + + // Prepopulate + for i in 0..1000 { + cache.put(format!("key{}", i), format!("value{}", i)); + } + + let mut group = c.benchmark_group("thread_safe_cache"); + + group.bench_function("read_lock_cache_hit", |b| { + b.iter(|| { + let key = format!("key{}", black_box(500)); + let value = cache.get(&key); + black_box(value); + }); + }); + + group.bench_function("write_lock_cache_miss", |b| { + let mut counter = 1000; + b.iter(|| { + let key = format!("key{}", black_box(counter)); + let value = cache.get(&key); + if value.is_none() { + cache.put(key.clone(), format!("value{}", counter)); + } + counter += 1; + }); + }); + + group.finish(); +} + +/// Benchmark 8: Hot/cold cache patterns +fn bench_hot_cold_patterns(c: &mut Criterion) { + let mut group = c.benchmark_group("hot_cold_patterns"); + + // Hot cache: Small working set + let mut hot_cache = LruCache::new(10_000, Duration::from_secs(300)); + for i in 0..10 { + hot_cache.put(format!("hot_key{}", i), format!("value{}", i)); + } + + group.bench_function("hot_cache_10_keys", |b| { + b.iter(|| { + let key = format!("hot_key{}", black_box(5)); + let value = hot_cache.get(&key); + black_box(value); + }); + }); + + // Cold cache: Large working set with misses + let mut cold_cache = LruCache::new(100, Duration::from_secs(300)); + for i in 0..100 { + cold_cache.put(format!("cold_key{}", i), format!("value{}", i)); + } + + let mut counter = 0; + group.bench_function("cold_cache_100_keys_rotating", |b| { + b.iter(|| { + counter += 1; + let key = format!("cold_key{}", black_box(counter % 150)); + let value = cold_cache.get(&key); + if value.is_none() { + cold_cache.put(key.clone(), format!("value{}", counter)); + } + black_box(value); + }); + }); + + group.finish(); +} + +/// Benchmark 9: Multi-tier cache (L1 + L2) +fn bench_multi_tier_cache(c: &mut Criterion) { + let l1_cache = Arc::new(RwLock::new(LruCache::::new( + 100, + Duration::from_secs(60), + ))); + let l2_cache = Arc::new(RwLock::new(LruCache::::new( + 10_000, + Duration::from_secs(300), + ))); + + // Prepopulate L2 + for i in 0..1000 { + l2_cache + .write() + .unwrap() + .put(format!("key{}", i), format!("value{}", i)); + } + + // Prepopulate L1 with hot keys + for i in 0..50 { + l1_cache + .write() + .unwrap() + .put(format!("key{}", i), format!("value{}", i)); + } + + c.bench_function("multi_tier_cache_l1_hit", |b| { + b.iter(|| { + let key = format!("key{}", black_box(25)); + + // Check L1 + let value = l1_cache.write().expect("INVARIANT: RwLock should not be poisoned").get(&key); + if value.is_none() { + // Check L2 + if let Some(v) = l2_cache.write().expect("INVARIANT: RwLock should not be poisoned").get(&key) { + l1_cache.write().expect("INVARIANT: RwLock should not be poisoned").put(key, v.clone()); + black_box(v); + } + } else { + black_box(value); + } + }); + }); +} + +/// Benchmark 10: Revocation list lookup +fn bench_revocation_list(c: &mut Criterion) { + // Simulate revocation list as HashSet + let mut revoked_tokens = std::collections::HashSet::new(); + for i in 0..10000 { + revoked_tokens.insert(format!("revoked_token_{}", i)); + } + + let mut group = c.benchmark_group("revocation_list"); + + group.bench_function("revoked_token_hit", |b| { + b.iter(|| { + let jti = format!("revoked_token_{}", black_box(5000)); + let is_revoked = revoked_tokens.contains(&jti); + black_box(is_revoked); + }); + }); + + group.bench_function("revoked_token_miss", |b| { + b.iter(|| { + let jti = format!("valid_token_{}", black_box(5000)); + let is_revoked = revoked_tokens.contains(&jti); + black_box(is_revoked); + }); + }); + + group.finish(); +} + +criterion_group!( + cache_benches, + bench_jwt_cache_hit, + bench_jwt_cache_miss, + bench_rbac_cache_hit, + bench_cache_sizes, + bench_cache_eviction, + bench_ttl_expiration, + bench_thread_safe_cache, + bench_hot_cold_patterns, + bench_multi_tier_cache, + bench_revocation_list +); + +criterion_main!(cache_benches); diff --git a/services/api/benches/dashmap_rate_limiter_bench.rs b/services/api/benches/dashmap_rate_limiter_bench.rs new file mode 100644 index 000000000..cfe246027 --- /dev/null +++ b/services/api/benches/dashmap_rate_limiter_bench.rs @@ -0,0 +1,334 @@ +//! DashMap vs RwLock Performance Comparison for Rate Limiter +//! +//! Benchmarks: +//! - Sequential reads (cache hit simulation) +//! - Concurrent reads from multiple threads +//! - Mixed read/write workload +//! - Contention scenarios +//! +//! Target: <8ns per operation with DashMap (6x improvement over RwLock) + +use dashmap::DashMap; +use std::collections::HashMap; +use std::hint::black_box; +use std::sync::Arc; +use std::time::Instant; +use tokio::sync::RwLock; + +#[derive(Clone)] +struct CacheEntry { + tokens: f64, + last_access: Instant, +} + +/// Benchmark sequential reads with RwLock +async fn bench_rwlock_sequential(iterations: usize) -> u128 { + let cache: Arc>> = Arc::new(RwLock::new(HashMap::new())); + + // Pre-populate cache + { + let mut map = cache.write().await; + for i in 0..1000 { + map.insert( + format!("key_{}", i), + CacheEntry { + tokens: 100.0, + last_access: Instant::now(), + }, + ); + } + } + + let start = Instant::now(); + for i in 0..iterations { + let key = format!("key_{}", i % 1000); + let map = cache.read().await; + black_box(map.get(&key)); + } + let elapsed = start.elapsed(); + + elapsed.as_nanos() / iterations as u128 +} + +/// Benchmark sequential reads with DashMap +async fn bench_dashmap_sequential(iterations: usize) -> u128 { + let cache: Arc> = Arc::new(DashMap::new()); + + // Pre-populate cache + for i in 0..1000 { + cache.insert( + format!("key_{}", i), + CacheEntry { + tokens: 100.0, + last_access: Instant::now(), + }, + ); + } + + let start = Instant::now(); + for i in 0..iterations { + let key = format!("key_{}", i % 1000); + black_box(cache.get(&key)); + } + let elapsed = start.elapsed(); + + elapsed.as_nanos() / iterations as u128 +} + +/// Benchmark concurrent reads with RwLock +async fn bench_rwlock_concurrent(iterations: usize, num_threads: usize) -> u128 { + let cache: Arc>> = Arc::new(RwLock::new(HashMap::new())); + + // Pre-populate cache + { + let mut map = cache.write().await; + for i in 0..1000 { + map.insert( + format!("key_{}", i), + CacheEntry { + tokens: 100.0, + last_access: Instant::now(), + }, + ); + } + } + + let start = Instant::now(); + let mut handles = vec![]; + + for thread_id in 0..num_threads { + let cache_clone = Arc::clone(&cache); + let handle = tokio::spawn(async move { + for i in 0..(iterations / num_threads) { + let key = format!("key_{}", (thread_id * 1000 + i) % 1000); + let map = cache_clone.read().await; + black_box(map.get(&key)); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.await.unwrap(); + } + + let elapsed = start.elapsed(); + elapsed.as_nanos() / iterations as u128 +} + +/// Benchmark concurrent reads with DashMap +async fn bench_dashmap_concurrent(iterations: usize, num_threads: usize) -> u128 { + let cache: Arc> = Arc::new(DashMap::new()); + + // Pre-populate cache + for i in 0..1000 { + cache.insert( + format!("key_{}", i), + CacheEntry { + tokens: 100.0, + last_access: Instant::now(), + }, + ); + } + + let start = Instant::now(); + let mut handles = vec![]; + + for thread_id in 0..num_threads { + let cache_clone = Arc::clone(&cache); + let handle = tokio::spawn(async move { + for i in 0..(iterations / num_threads) { + let key = format!("key_{}", (thread_id * 1000 + i) % 1000); + black_box(cache_clone.get(&key)); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.await.unwrap(); + } + + let elapsed = start.elapsed(); + elapsed.as_nanos() / iterations as u128 +} + +/// Benchmark mixed read/write with RwLock +async fn bench_rwlock_mixed(iterations: usize, write_ratio: f64) -> u128 { + let cache: Arc>> = Arc::new(RwLock::new(HashMap::new())); + + // Pre-populate cache + { + let mut map = cache.write().await; + for i in 0..1000 { + map.insert( + format!("key_{}", i), + CacheEntry { + tokens: 100.0, + last_access: Instant::now(), + }, + ); + } + } + + let start = Instant::now(); + for i in 0..iterations { + let key = format!("key_{}", i % 1000); + + // Determine if this is a read or write + if (i as f64 / iterations as f64) < write_ratio { + let mut map = cache.write().await; + map.insert( + key, + CacheEntry { + tokens: 99.0, + last_access: Instant::now(), + }, + ); + } else { + let map = cache.read().await; + black_box(map.get(&key)); + } + } + let elapsed = start.elapsed(); + + elapsed.as_nanos() / iterations as u128 +} + +/// Benchmark mixed read/write with DashMap +async fn bench_dashmap_mixed(iterations: usize, write_ratio: f64) -> u128 { + let cache: Arc> = Arc::new(DashMap::new()); + + // Pre-populate cache + for i in 0..1000 { + cache.insert( + format!("key_{}", i), + CacheEntry { + tokens: 100.0, + last_access: Instant::now(), + }, + ); + } + + let start = Instant::now(); + for i in 0..iterations { + let key = format!("key_{}", i % 1000); + + // Determine if this is a read or write + if (i as f64 / iterations as f64) < write_ratio { + cache.insert( + key, + CacheEntry { + tokens: 99.0, + last_access: Instant::now(), + }, + ); + } else { + black_box(cache.get(&key)); + } + } + let elapsed = start.elapsed(); + + elapsed.as_nanos() / iterations as u128 +} + +#[tokio::main] +async fn main() { + println!("DashMap vs RwLock Performance Comparison"); + println!("==========================================\n"); + + let iterations = 100_000; + + // Benchmark 1: Sequential reads + println!("Benchmark 1: Sequential Reads ({} iterations)", iterations); + let rwlock_seq = bench_rwlock_sequential(iterations).await; + let dashmap_seq = bench_dashmap_sequential(iterations).await; + let improvement_seq = rwlock_seq as f64 / dashmap_seq as f64; + + println!(" RwLock: {} ns/op", rwlock_seq); + println!(" DashMap: {} ns/op", dashmap_seq); + println!(" Speedup: {:.2}x", improvement_seq); + println!(" Target: <8ns ✓\n"); + + // Benchmark 2: Concurrent reads (4 threads) + println!( + "Benchmark 2: Concurrent Reads (4 threads, {} total ops)", + iterations + ); + let rwlock_conc = bench_rwlock_concurrent(iterations, 4).await; + let dashmap_conc = bench_dashmap_concurrent(iterations, 4).await; + let improvement_conc = rwlock_conc as f64 / dashmap_conc as f64; + + println!(" RwLock: {} ns/op", rwlock_conc); + println!(" DashMap: {} ns/op", dashmap_conc); + println!(" Speedup: {:.2}x", improvement_conc); + println!(" Target: <8ns ✓\n"); + + // Benchmark 3: Concurrent reads (8 threads) + println!( + "Benchmark 3: High Contention (8 threads, {} total ops)", + iterations + ); + let rwlock_high = bench_rwlock_concurrent(iterations, 8).await; + let dashmap_high = bench_dashmap_concurrent(iterations, 8).await; + let improvement_high = rwlock_high as f64 / dashmap_high as f64; + + println!(" RwLock: {} ns/op", rwlock_high); + println!(" DashMap: {} ns/op", dashmap_high); + println!(" Speedup: {:.2}x", improvement_high); + println!(" Target: <8ns ✓\n"); + + // Benchmark 4: Mixed read/write (10% writes) + println!( + "Benchmark 4: Mixed Workload - 10% writes ({} ops)", + iterations + ); + let rwlock_mixed = bench_rwlock_mixed(iterations, 0.10).await; + let dashmap_mixed = bench_dashmap_mixed(iterations, 0.10).await; + let improvement_mixed = rwlock_mixed as f64 / dashmap_mixed as f64; + + println!(" RwLock: {} ns/op", rwlock_mixed); + println!(" DashMap: {} ns/op", dashmap_mixed); + println!(" Speedup: {:.2}x", improvement_mixed); + println!(" Target: <8ns ✓\n"); + + // Benchmark 5: Mixed read/write (1% writes - typical rate limiter) + println!( + "Benchmark 5: Rate Limiter Workload - 1% writes ({} ops)", + iterations + ); + let rwlock_rl = bench_rwlock_mixed(iterations, 0.01).await; + let dashmap_rl = bench_dashmap_mixed(iterations, 0.01).await; + let improvement_rl = rwlock_rl as f64 / dashmap_rl as f64; + + println!(" RwLock: {} ns/op", rwlock_rl); + println!(" DashMap: {} ns/op", dashmap_rl); + println!(" Speedup: {:.2}x", improvement_rl); + println!(" Target: <8ns ✓\n"); + + // Summary + println!("=========================================="); + println!("Performance Summary:"); + println!( + " Sequential: {:.2}x improvement ({} ns → {} ns)", + improvement_seq, rwlock_seq, dashmap_seq + ); + println!( + " Concurrent (4T): {:.2}x improvement ({} ns → {} ns)", + improvement_conc, rwlock_conc, dashmap_conc + ); + println!( + " Concurrent (8T): {:.2}x improvement ({} ns → {} ns)", + improvement_high, rwlock_high, dashmap_high + ); + println!( + " Mixed (10% W): {:.2}x improvement ({} ns → {} ns)", + improvement_mixed, rwlock_mixed, dashmap_mixed + ); + println!( + " Rate Limiter: {:.2}x improvement ({} ns → {} ns)", + improvement_rl, rwlock_rl, dashmap_rl + ); + println!("\n✓ All benchmarks completed successfully"); + println!("✓ Target <8ns achieved: {}", dashmap_seq < 8); +} diff --git a/services/api/benches/proxy_latency.rs b/services/api/benches/proxy_latency.rs new file mode 100644 index 000000000..f8cdd110c --- /dev/null +++ b/services/api/benches/proxy_latency.rs @@ -0,0 +1,413 @@ +//! API Gateway gRPC Proxy Latency Benchmark +//! +//! Measures REAL proxy overhead from API Gateway → Backend Services +//! TARGET: <1ms latency (Wave 132 baseline: 21-488μs warm) +//! +//! Tests: +//! 1. Cold start latency (first request) +//! 2. Warm cache latency (P50, P95, P99) +//! 3. Direct service call vs proxied call overhead +//! 4. Connection pool impact +//! 5. JWT metadata forwarding overhead + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use tokio::runtime::Runtime; +use tonic::{metadata::MetadataValue, Request}; +use uuid::Uuid; + +// Import proto definitions from API Gateway's embedded protos +// We use the TLI client interface (foxhunt.tli.trading) which is what external clients use +use api::foxhunt::tli::{ + trading_service_client::TradingServiceClient, OrderSide, OrderType, SubmitOrderRequest, +}; + +/// Test JWT configuration (matches api/tests/common/mod.rs) +fn generate_test_jwt() -> String { + use jsonwebtoken::{encode, EncodingKey, Header}; + + #[derive(serde::Serialize)] + struct TestClaims { + jti: String, + sub: String, + iat: u64, + exp: u64, + nbf: Option, + iss: String, + aud: String, + roles: Vec, + permissions: Vec, + token_type: String, + session_id: Option, + } + + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + + let claims = TestClaims { + jti: Uuid::new_v4().to_string(), + sub: "bench-user".to_string(), + iat: now, + exp: now + 3600, + nbf: Some(now), + iss: "foxhunt-api-gateway".to_string(), + aud: "foxhunt-services".to_string(), + roles: vec!["trader".to_string()], + permissions: vec!["trading.submit_order".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let secret = + "test-secret-must-be-at-least-64-characters-long-for-security-validation-ok-1234567890"; + + encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(secret.as_bytes()), + ) + .unwrap() +} + +/// Create request with JWT metadata +fn create_authenticated_request() -> (Request, String) { + let jwt_token = generate_test_jwt(); + + let order = SubmitOrderRequest { + symbol: "BTC/USD".to_string(), + side: OrderSide::Buy as i32, + order_type: OrderType::Limit as i32, + quantity: 1.0, + price: Some(50000.0), + stop_price: None, + time_in_force: "GTC".to_string(), + client_order_id: Uuid::new_v4().to_string(), + }; + + let mut request = Request::new(order); + let metadata = request.metadata_mut(); + + // Add authorization header (matches API Gateway format) + let auth_value = MetadataValue::try_from(format!("Bearer {}", jwt_token)).unwrap(); + metadata.insert("authorization", auth_value); + + (request, jwt_token) +} + +/// Benchmark 1: Cold start latency (first proxy call) +fn bench_proxy_cold_start(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + + c.bench_function("proxy_cold_start", |b| { + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + + for _ in 0..iters { + // Create fresh client each iteration to measure cold start + let start = Instant::now(); + + rt.block_on(async { + // Connect through API Gateway (proxy) + let mut client = + match TradingServiceClient::connect("http://localhost:50051").await { + Ok(c) => c, + Err(e) => { + eprintln!("⚠️ Failed to connect to API Gateway: {}", e); + return; + }, + }; + + let (request, _) = create_authenticated_request(); + + // Make proxied call + match client.submit_order(black_box(request)).await { + Ok(_) => {}, + Err(e) => { + // Expected to fail (no real order), but we measure connection overhead + let _ = black_box(e); + }, + } + }); + + total += start.elapsed(); + } + + total + }); + }); +} + +/// Benchmark 2: Warm cache latency (reused connection) +fn bench_proxy_warm_cache(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + + c.bench_function("proxy_warm_cache", |b| { + // Setup: Create persistent client + let mut client = rt.block_on(async { + match TradingServiceClient::connect("http://localhost:50051").await { + Ok(c) => c, + Err(e) => { + panic!("❌ Failed to connect to API Gateway: {}", e); + }, + } + }); + + // Warmup: Make 100 requests to JIT compile and warm caches + rt.block_on(async { + for _ in 0..100 { + let (request, _) = create_authenticated_request(); + let _ = client.submit_order(request).await; + } + }); + + b.iter(|| { + rt.block_on(async { + let (request, _) = create_authenticated_request(); + + let _ = black_box(client.submit_order(black_box(request)).await); + }); + }); + }); +} + +/// Benchmark 3: Direct service call (baseline - no proxy) +fn bench_direct_service_call(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + + c.bench_function("direct_service_call_baseline", |b| { + // Connect directly to Trading Service (bypass API Gateway) + let mut client = rt.block_on(async { + match TradingServiceClient::connect("http://localhost:50052").await { + Ok(c) => c, + Err(e) => { + panic!("❌ Failed to connect to Trading Service: {}", e); + }, + } + }); + + // Warmup + rt.block_on(async { + for _ in 0..100 { + let (request, _) = create_authenticated_request(); + let _ = client.submit_order(request).await; + } + }); + + b.iter(|| { + rt.block_on(async { + let (request, _) = create_authenticated_request(); + + let _ = black_box(client.submit_order(black_box(request)).await); + }); + }); + }); +} + +/// Benchmark 4: Proxy overhead calculation (proxied - direct) +fn bench_proxy_overhead_only(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + + let mut group = c.benchmark_group("proxy_overhead"); + + // Setup both clients + let (mut proxy_client, mut direct_client) = rt.block_on(async { + let proxy = TradingServiceClient::connect("http://localhost:50051") + .await + .expect("API Gateway not running"); + let direct = TradingServiceClient::connect("http://localhost:50052") + .await + .expect("Trading Service not running"); + (proxy, direct) + }); + + // Warmup both + rt.block_on(async { + for _ in 0..100 { + let (r1, _) = create_authenticated_request(); + let (r2, _) = create_authenticated_request(); + let _ = proxy_client.submit_order(r1).await; + let _ = direct_client.submit_order(r2).await; + } + }); + + group.bench_function("through_proxy", |b| { + b.iter(|| { + rt.block_on(async { + let (request, _) = create_authenticated_request(); + let _ = black_box(proxy_client.submit_order(black_box(request)).await); + }); + }); + }); + + group.bench_function("direct_call", |b| { + b.iter(|| { + rt.block_on(async { + let (request, _) = create_authenticated_request(); + let _ = black_box(direct_client.submit_order(black_box(request)).await); + }); + }); + }); + + group.finish(); +} + +/// Benchmark 5: JWT metadata forwarding overhead +fn bench_jwt_metadata_overhead(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + + let mut group = c.benchmark_group("jwt_metadata_forwarding"); + + let mut client = rt.block_on(async { + TradingServiceClient::connect("http://localhost:50051") + .await + .expect("API Gateway not running") + }); + + // Warmup + rt.block_on(async { + for _ in 0..100 { + let (request, _) = create_authenticated_request(); + let _ = client.submit_order(request).await; + } + }); + + // Measure with different JWT sizes + for jwt_size in &["small", "medium", "large"] { + group.bench_with_input( + BenchmarkId::from_parameter(jwt_size), + jwt_size, + |b, _size| { + b.iter(|| { + rt.block_on(async { + let (request, _) = create_authenticated_request(); + let _ = black_box(client.submit_order(black_box(request)).await); + }); + }); + }, + ); + } + + group.finish(); +} + +/// Benchmark 6: Connection pool impact +fn bench_connection_pool_impact(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + + let mut group = c.benchmark_group("connection_pool"); + + for pool_size in &[1, 10, 100] { + group.bench_with_input( + BenchmarkId::new("concurrent_requests", pool_size), + pool_size, + |b, &size| { + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + + for _ in 0..iters { + let start = Instant::now(); + + rt.block_on(async { + let mut handles = vec![]; + + for _ in 0..size { + let handle = tokio::spawn(async move { + let mut client = + TradingServiceClient::connect("http://localhost:50051") + .await + .unwrap(); + + let (request, _) = create_authenticated_request(); + let _ = client.submit_order(request).await; + }); + + handles.push(handle); + } + + for handle in handles { + let _ = handle.await; + } + }); + + total += start.elapsed(); + } + + total + }); + }, + ); + } + + group.finish(); +} + +/// Benchmark 7: Percentile analysis (P50, P95, P99) +fn bench_latency_percentiles(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + + c.bench_function("latency_percentiles", |b| { + let mut client = rt.block_on(async { + TradingServiceClient::connect("http://localhost:50051") + .await + .expect("API Gateway not running") + }); + + // Warmup + rt.block_on(async { + for _ in 0..1000 { + let (request, _) = create_authenticated_request(); + let _ = client.submit_order(request).await; + } + }); + + b.iter_custom(|iters| { + let mut latencies = Vec::with_capacity(iters as usize); + + for _ in 0..iters { + let start = Instant::now(); + + rt.block_on(async { + let (request, _) = create_authenticated_request(); + let _ = black_box(client.submit_order(black_box(request)).await); + }); + + latencies.push(start.elapsed()); + } + + // Calculate percentiles + latencies.sort(); + let p50_idx = (iters as f64 * 0.50) as usize; + let p95_idx = (iters as f64 * 0.95) as usize; + let p99_idx = (iters as f64 * 0.99) as usize; + + println!("\n📊 Latency Percentiles:"); + println!(" P50: {:?}", latencies[p50_idx]); + println!(" P95: {:?}", latencies[p95_idx]); + println!(" P99: {:?}", latencies[p99_idx]); + println!(" Target: <1ms (1,000,000ns)"); + + if latencies[p99_idx] < Duration::from_millis(1) { + println!(" ✅ PASS: P99 < 1ms"); + } else { + println!(" ❌ FAIL: P99 >= 1ms"); + } + + latencies.iter().sum() + }); + }); +} + +criterion_group!( + proxy_benches, + bench_proxy_cold_start, + bench_proxy_warm_cache, + bench_direct_service_call, + bench_proxy_overhead_only, + bench_jwt_metadata_overhead, + bench_connection_pool_impact, + bench_latency_percentiles +); + +criterion_main!(proxy_benches); diff --git a/services/api/benches/rate_limiter_bench.rs b/services/api/benches/rate_limiter_bench.rs new file mode 100644 index 000000000..7783a31de --- /dev/null +++ b/services/api/benches/rate_limiter_bench.rs @@ -0,0 +1,143 @@ +//! Performance benchmarks for Rate Limiter +//! +//! Demonstrates: +//! - Cache hit performance (<50ns target) +//! - Token bucket algorithm overhead +//! - Concurrent access patterns + +use std::hint::black_box; +use std::time::{Duration, Instant}; + +/// Token bucket for rate limiting (simplified for benchmark) +struct TokenBucket { + tokens: f64, + last_refill: Instant, + capacity: f64, + refill_rate: f64, +} + +impl TokenBucket { + fn new(capacity: f64, refill_rate: f64) -> Self { + Self { + tokens: capacity, + last_refill: Instant::now(), + capacity, + refill_rate, + } + } + + fn consume(&mut self) -> bool { + let now = Instant::now(); + let elapsed = now.duration_since(self.last_refill).as_secs_f64(); + self.tokens = (self.tokens + (elapsed * self.refill_rate)).min(self.capacity); + self.last_refill = now; + + if self.tokens >= 1.0 { + self.tokens -= 1.0; + true + } else { + false + } + } +} + +fn main() { + println!("Rate Limiter Performance Benchmarks\n"); + println!("========================================\n"); + + // Benchmark 1: Cache hit simulation (in-memory check) + println!("Benchmark 1: Cache Hit Performance (in-memory)"); + let mut bucket = TokenBucket::new(10000.0, 10000.0); // High capacity to avoid refills + let iterations = 1_000_000; + let start = Instant::now(); + + for _ in 0..iterations { + black_box(bucket.consume()); + } + + let elapsed = start.elapsed(); + let ns_per_op = elapsed.as_nanos() / iterations as u128; + println!("Total time: {:?}", elapsed); + println!("Operations: {}", iterations); + println!("Time per operation: {} ns", ns_per_op); + println!("Target: <50ns ✓\n"); + + // Benchmark 2: Token bucket refill overhead + println!("Benchmark 2: Token Bucket Refill Overhead"); + let mut bucket2 = TokenBucket::new(100.0, 100.0); + let iterations2 = 100_000; + let start2 = Instant::now(); + + for i in 0..iterations2 { + // Consume token + bucket2.consume(); + // Simulate small delay between requests + if i % 100 == 0 { + std::thread::sleep(Duration::from_micros(10)); + } + } + + let elapsed2 = start2.elapsed(); + let ns_per_op2 = elapsed2.as_nanos() / iterations2 as u128; + println!("Total time: {:?}", elapsed2); + println!("Operations: {}", iterations2); + println!("Time per operation: {} ns", ns_per_op2); + println!("(includes 10μs sleeps every 100 operations)\n"); + + // Benchmark 3: Burst handling + println!("Benchmark 3: Burst Handling (100 requests at once)"); + let mut bucket3 = TokenBucket::new(100.0, 100.0); + let burst_size = 100; + let start3 = Instant::now(); + let mut allowed = 0; + + for _ in 0..burst_size { + if bucket3.consume() { + allowed += 1; + } + } + + let elapsed3 = start3.elapsed(); + println!("Total time: {:?}", elapsed3); + println!("Allowed requests: {}/{}", allowed, burst_size); + println!( + "Average per request: {} ns\n", + elapsed3.as_nanos() / burst_size + ); + + // Benchmark 4: High-frequency trading scenario + println!("Benchmark 4: HFT Scenario (10,000 requests, 100 req/sec limit)"); + let mut bucket4 = TokenBucket::new(100.0, 100.0); + let hft_requests = 10_000; + let start4 = Instant::now(); + let mut hft_allowed = 0; + + for _ in 0..hft_requests { + if bucket4.consume() { + hft_allowed += 1; + } + } + + let elapsed4 = start4.elapsed(); + println!("Total time: {:?}", elapsed4); + println!("Allowed: {}/{} requests", hft_allowed, hft_requests); + println!("Denied: {} requests", hft_requests - hft_allowed); + println!( + "Average per check: {} ns\n", + elapsed4.as_nanos() / hft_requests + ); + + println!("========================================"); + println!("Performance Summary:"); + println!(" - Cache hit: {} ns (target <50ns)", ns_per_op); + println!(" - Token bucket: {} ns", ns_per_op2); + println!( + " - Burst handling: {} ns", + elapsed3.as_nanos() / burst_size + ); + println!( + " - HFT scenario: {} ns", + elapsed4.as_nanos() / hft_requests + ); + println!("\n✓ All benchmarks completed successfully"); +} diff --git a/services/api/benches/rate_limiting_perf.rs b/services/api/benches/rate_limiting_perf.rs new file mode 100644 index 000000000..2021bd08d --- /dev/null +++ b/services/api/benches/rate_limiting_perf.rs @@ -0,0 +1,320 @@ +//! Rate Limiting Performance Benchmark +//! +//! Measures rate limiter performance: +//! - TARGET: <50ns per check +//! - Atomic counter operations +//! - Token bucket algorithm +//! - Sliding window counters +//! - Concurrent access patterns + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +/// Atomic counter-based rate limiter (fastest) +struct AtomicRateLimiter { + counters: HashMap>, + limit: u64, +} + +impl AtomicRateLimiter { + fn new(limit: u64) -> Self { + Self { + counters: HashMap::new(), + limit, + } + } + + fn check(&self, user_id: &str) -> bool { + if let Some(counter) = self.counters.get(user_id) { + let count = counter.fetch_add(1, Ordering::Relaxed); + count < self.limit + } else { + true // First request always allowed + } + } + + fn with_user(mut self, user_id: &str) -> Self { + self.counters + .insert(user_id.to_string(), Arc::new(AtomicU64::new(0))); + self + } +} + +/// Token bucket rate limiter +struct TokenBucket { + tokens: Mutex, + last_refill: Mutex, + capacity: f64, + refill_rate: f64, // tokens per second +} + +impl TokenBucket { + fn new(capacity: f64, refill_rate: f64) -> Self { + Self { + tokens: Mutex::new(capacity), + last_refill: Mutex::new(Instant::now()), + capacity, + refill_rate, + } + } + + fn check(&self) -> bool { + let mut tokens = self.tokens.lock().expect("INVARIANT: Lock should not be poisoned"); + let mut last_refill = self.last_refill.lock().expect("INVARIANT: Lock should not be poisoned"); + + let now = Instant::now(); + let elapsed = now.duration_since(*last_refill).as_secs_f64(); + + // Refill tokens + *tokens = (*tokens + (elapsed * self.refill_rate)).min(self.capacity); + *last_refill = now; + + if *tokens >= 1.0 { + *tokens -= 1.0; + true + } else { + false + } + } +} + +/// Sliding window counter +struct SlidingWindow { + window: Mutex>, + window_size: Duration, + limit: usize, +} + +impl SlidingWindow { + fn new(window_size: Duration, limit: usize) -> Self { + Self { + window: Mutex::new(Vec::new()), + window_size, + limit, + } + } + + fn check(&self) -> bool { + let mut window = self.window.lock().expect("INVARIANT: Lock should not be poisoned"); + let now = Instant::now(); + + // Remove expired entries + window.retain(|&time| now.duration_since(time) < self.window_size); + + if window.len() < self.limit { + window.push(now); + true + } else { + false + } + } +} + +/// Benchmark 1: Atomic counter (TARGET: <50ns) +fn bench_atomic_rate_limiter(c: &mut Criterion) { + let limiter = AtomicRateLimiter::new(1_000_000).with_user("user123"); + + c.bench_function("atomic_rate_limiter", |b| { + b.iter(|| { + let allowed = limiter.check(black_box("user123")); + black_box(allowed); + }); + }); +} + +/// Benchmark 2: Token bucket algorithm +fn bench_token_bucket(c: &mut Criterion) { + let bucket = TokenBucket::new(100.0, 100.0); + + c.bench_function("token_bucket_rate_limiter", |b| { + b.iter(|| { + let allowed = bucket.check(); + black_box(allowed); + }); + }); +} + +/// Benchmark 3: Sliding window counter +fn bench_sliding_window(c: &mut Criterion) { + let window = SlidingWindow::new(Duration::from_secs(1), 100); + + c.bench_function("sliding_window_rate_limiter", |b| { + b.iter(|| { + let allowed = window.check(); + black_box(allowed); + }); + }); +} + +/// Benchmark 4: Different user counts +fn bench_user_scaling(c: &mut Criterion) { + let mut group = c.benchmark_group("rate_limiter_user_scaling"); + + for num_users in &[10, 100, 1_000, 10_000] { + let mut limiter = AtomicRateLimiter::new(1_000_000); + for i in 0..*num_users { + limiter = limiter.with_user(&format!("user{}", i)); + } + + group.bench_with_input( + BenchmarkId::new("atomic_limiter", num_users), + num_users, + |b, &n| { + b.iter(|| { + let user_id = format!("user{}", black_box(n / 2)); + let allowed = limiter.check(&user_id); + black_box(allowed); + }); + }, + ); + } + + group.finish(); +} + +/// Benchmark 5: Burst handling +fn bench_burst_handling(c: &mut Criterion) { + let bucket = TokenBucket::new(100.0, 100.0); + + c.bench_function("burst_100_requests", |b| { + b.iter(|| { + let mut allowed_count = 0; + for _ in 0..100 { + if bucket.check() { + allowed_count += 1; + } + } + black_box(allowed_count); + }); + }); +} + +/// Benchmark 6: Rate limiter with refill overhead +fn bench_refill_overhead(c: &mut Criterion) { + let bucket = TokenBucket::new(1000.0, 1000.0); + + let mut group = c.benchmark_group("refill_overhead"); + + // Simulate different request patterns + group.bench_function("high_frequency_no_wait", |b| { + b.iter(|| { + let allowed = bucket.check(); + black_box(allowed); + }); + }); + + group.bench_function("with_small_delay", |b| { + b.iter(|| { + std::thread::sleep(Duration::from_micros(1)); + let allowed = bucket.check(); + black_box(allowed); + }); + }); + + group.finish(); +} + +/// Benchmark 7: Concurrent access to rate limiter +fn bench_concurrent_access(c: &mut Criterion) { + use std::thread; + + let limiter = Arc::new(AtomicRateLimiter::new(1_000_000).with_user("user123")); + + c.bench_function("concurrent_rate_limiter_4_threads", |b| { + b.iter(|| { + let mut handles = vec![]; + + for _ in 0..4 { + let limiter_clone = limiter.clone(); + let handle = thread::spawn(move || { + for _ in 0..1000 { + black_box(limiter_clone.check("user123")); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.join().expect("INVARIANT: Thread should complete successfully"); + } + }); + }); +} + +/// Benchmark 8: Deny rate (when limit exceeded) +fn bench_deny_rate(c: &mut Criterion) { + let limiter = AtomicRateLimiter::new(0).with_user("user123"); // Zero limit + + c.bench_function("rate_limiter_deny_path", |b| { + b.iter(|| { + let allowed = limiter.check(black_box("user123")); + assert!(!allowed); // Should always deny + black_box(allowed); + }); + }); +} + +/// Benchmark 9: Cache hit patterns +fn bench_cache_patterns(c: &mut Criterion) { + let mut group = c.benchmark_group("cache_hit_patterns"); + + // Hot path: Same user repeatedly (cache hit) + let limiter_hot = AtomicRateLimiter::new(1_000_000).with_user("user123"); + group.bench_function("hot_path_same_user", |b| { + b.iter(|| { + let allowed = limiter_hot.check(black_box("user123")); + black_box(allowed); + }); + }); + + // Cold path: Different users (cache miss simulation) + let mut limiter_cold = AtomicRateLimiter::new(1_000_000); + for i in 0..1000 { + limiter_cold = limiter_cold.with_user(&format!("user{}", i)); + } + let mut counter = 0; + group.bench_function("cold_path_different_users", |b| { + b.iter(|| { + counter += 1; + let user_id = format!("user{}", counter % 1000); + let allowed = limiter_cold.check(&user_id); + black_box(allowed); + }); + }); + + group.finish(); +} + +/// Benchmark 10: HFT scenario (100K requests/sec target) +fn bench_hft_scenario(c: &mut Criterion) { + let limiter = AtomicRateLimiter::new(100_000).with_user("hft_trader"); + + c.bench_function("hft_100k_rps_scenario", |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + for _ in 0..iters { + black_box(limiter.check("hft_trader")); + } + start.elapsed() + }); + }); +} + +criterion_group!( + rate_limiting_benches, + bench_atomic_rate_limiter, + bench_token_bucket, + bench_sliding_window, + bench_user_scaling, + bench_burst_handling, + bench_refill_overhead, + bench_concurrent_access, + bench_deny_rate, + bench_cache_patterns, + bench_hft_scenario +); + +criterion_main!(rate_limiting_benches); diff --git a/services/api/benches/revocation_cache_perf.rs b/services/api/benches/revocation_cache_perf.rs new file mode 100644 index 000000000..658f1c030 --- /dev/null +++ b/services/api/benches/revocation_cache_perf.rs @@ -0,0 +1,382 @@ +//! Revocation Cache Performance Benchmark +//! +//! Measures the performance impact of local revocation cache: +//! - Cache hit latency: TARGET <10ns +//! - Cache miss latency: ~500μs (Redis network) +//! - Cache hit rate: TARGET >95% +//! - Memory overhead: Minimal with TTL expiration +//! +//! This benchmark compares: +//! 1. Direct Redis calls (no cache) +//! 2. Local DashMap cache with 60s TTL +//! 3. Cache behavior under different access patterns + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use dashmap::DashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +/// Simulated revocation result +#[derive(Debug, Clone)] +struct CachedRevocationResult { + is_revoked: bool, + cached_at: Instant, +} + +/// Local revocation cache implementation +struct LocalRevocationCache { + cache: Arc>, + ttl: Duration, + hits: Arc, + misses: Arc, +} + +impl LocalRevocationCache { + fn new(ttl: Duration) -> Self { + Self { + cache: Arc::new(DashMap::new()), + ttl, + hits: Arc::new(std::sync::atomic::AtomicU64::new(0)), + misses: Arc::new(std::sync::atomic::AtomicU64::new(0)), + } + } + + fn check_revoked(&self, token_id: &str, simulate_redis_latency: bool) -> bool { + // Check cache first + if let Some(entry) = self.cache.get(token_id) { + if entry.cached_at.elapsed() < self.ttl { + // Cache hit + self.hits.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + return entry.is_revoked; + } else { + // Expired - remove it + drop(entry); + self.cache.remove(token_id); + } + } + + // Cache miss - simulate Redis lookup + self.misses + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + + if simulate_redis_latency { + // Simulate 500μs Redis network latency + std::thread::sleep(Duration::from_micros(500)); + } + + // Simulate Redis result (99% not revoked in production) + let is_revoked = token_id.contains("revoked"); + + // Update cache + self.cache.insert( + token_id.to_string(), + CachedRevocationResult { + is_revoked, + cached_at: Instant::now(), + }, + ); + + is_revoked + } + + fn stats(&self) -> (u64, u64, f64) { + let hits = self.hits.load(std::sync::atomic::Ordering::Relaxed); + let misses = self.misses.load(std::sync::atomic::Ordering::Relaxed); + let total = hits + misses; + let hit_rate = if total > 0 { + (hits as f64 / total as f64) * 100.0 + } else { + 0.0 + }; + (hits, misses, hit_rate) + } +} + +/// Benchmark 1: Cache hit latency (TARGET: <10ns) +fn bench_cache_hit_latency(c: &mut Criterion) { + let cache = LocalRevocationCache::new(Duration::from_secs(60)); + + // Prepopulate cache with 1000 tokens + for i in 0..1000 { + let token_id = format!("token_{}", i); + cache.check_revoked(&token_id, false); // Prime cache without latency + } + + c.bench_function("revocation_cache_hit", |b| { + b.iter(|| { + let token_id = format!("token_{}", black_box(500)); + let is_revoked = cache.check_revoked(&token_id, false); + black_box(is_revoked); + }); + }); +} + +/// Benchmark 2: Cache miss latency (with simulated Redis) +fn bench_cache_miss_latency(c: &mut Criterion) { + let cache = LocalRevocationCache::new(Duration::from_secs(60)); + + c.bench_function("revocation_cache_miss_with_redis", |b| { + let mut counter = 0; + b.iter(|| { + let token_id = format!("new_token_{}", black_box(counter)); + let is_revoked = cache.check_revoked(&token_id, true); // Simulate Redis latency + black_box(is_revoked); + counter += 1; + }); + }); +} + +/// Benchmark 3: Hot token pattern (95% cache hit rate) +fn bench_hot_token_pattern(c: &mut Criterion) { + let cache = LocalRevocationCache::new(Duration::from_secs(60)); + + // Prepopulate with hot tokens + for i in 0..10 { + let token_id = format!("hot_token_{}", i); + cache.check_revoked(&token_id, false); + } + + c.bench_function("hot_token_pattern_95pct_hits", |b| { + let mut counter = 0; + b.iter(|| { + // 95% hits (tokens 0-9), 5% misses (tokens 10-19) + let token_num = if counter % 20 < 19 { + counter % 10 + } else { + 10 + (counter % 10) + }; + let token_id = format!("hot_token_{}", black_box(token_num)); + let is_revoked = cache.check_revoked(&token_id, false); + black_box(is_revoked); + counter += 1; + }); + }); + + // Report statistics + let (hits, misses, hit_rate) = cache.stats(); + println!( + "\nHot token pattern stats: {} hits, {} misses, {:.2}% hit rate", + hits, misses, hit_rate + ); +} + +/// Benchmark 4: TTL expiration behavior +fn bench_ttl_expiration(c: &mut Criterion) { + let mut group = c.benchmark_group("ttl_expiration"); + + // Short TTL (1ms) - high expiration rate + let cache_short = LocalRevocationCache::new(Duration::from_millis(1)); + for i in 0..100 { + cache_short.check_revoked(&format!("token_{}", i), false); + } + + group.bench_function("1ms_ttl", |b| { + b.iter(|| { + let token_id = format!("token_{}", black_box(50)); + let is_revoked = cache_short.check_revoked(&token_id, false); + black_box(is_revoked); + }); + }); + + // Long TTL (60s) - low expiration rate + let cache_long = LocalRevocationCache::new(Duration::from_secs(60)); + for i in 0..100 { + cache_long.check_revoked(&format!("token_{}", i), false); + } + + group.bench_function("60s_ttl", |b| { + b.iter(|| { + let token_id = format!("token_{}", black_box(50)); + let is_revoked = cache_long.check_revoked(&token_id, false); + black_box(is_revoked); + }); + }); + + group.finish(); +} + +/// Benchmark 5: Cache size impact +fn bench_cache_size_impact(c: &mut Criterion) { + let mut group = c.benchmark_group("cache_size_impact"); + + for size in &[100, 1_000, 10_000, 100_000] { + let cache = LocalRevocationCache::new(Duration::from_secs(60)); + + // Prepopulate to target size + for i in 0..*size { + cache.check_revoked(&format!("token_{}", i), false); + } + + group.bench_with_input(BenchmarkId::new("lookup", size), size, |b, &n| { + b.iter(|| { + let token_id = format!("token_{}", black_box(n / 2)); + let is_revoked = cache.check_revoked(&token_id, false); + black_box(is_revoked); + }); + }); + } + + group.finish(); +} + +/// Benchmark 6: Concurrent access pattern +fn bench_concurrent_access(c: &mut Criterion) { + let cache = Arc::new(LocalRevocationCache::new(Duration::from_secs(60))); + + // Prepopulate + for i in 0..1000 { + cache.check_revoked(&format!("token_{}", i), false); + } + + c.bench_function("concurrent_cache_access", |b| { + b.iter(|| { + let cache_clone = cache.clone(); + let token_id = format!("token_{}", black_box(500)); + let is_revoked = cache_clone.check_revoked(&token_id, false); + black_box(is_revoked); + }); + }); +} + +/// Benchmark 7: Mixed revoked/valid token pattern +fn bench_mixed_revocation(c: &mut Criterion) { + let cache = LocalRevocationCache::new(Duration::from_secs(60)); + + // Prepopulate with mix of revoked and valid tokens + for i in 0..1000 { + let token_id = if i % 10 == 0 { + format!("revoked_token_{}", i) // 10% revoked + } else { + format!("valid_token_{}", i) // 90% valid + }; + cache.check_revoked(&token_id, false); + } + + c.bench_function("mixed_revocation_pattern", |b| { + let mut counter = 0; + b.iter(|| { + let token_id = if counter % 10 == 0 { + format!("revoked_token_{}", black_box(counter)) + } else { + format!("valid_token_{}", black_box(counter)) + }; + let is_revoked = cache.check_revoked(&token_id, false); + black_box(is_revoked); + counter = (counter + 1) % 1000; + }); + }); +} + +/// Benchmark 8: Cache vs no-cache comparison +fn bench_cache_vs_no_cache(c: &mut Criterion) { + let mut group = c.benchmark_group("cache_vs_no_cache"); + + // No cache - direct Redis simulation + group.bench_function("no_cache_direct_redis", |b| { + b.iter(|| { + // Simulate 500μs Redis latency every time + std::thread::sleep(Duration::from_micros(500)); + let is_revoked = false; // Simulate result + black_box(is_revoked); + }); + }); + + // With cache - 95% hit rate + let cache = LocalRevocationCache::new(Duration::from_secs(60)); + for i in 0..10 { + cache.check_revoked(&format!("token_{}", i), false); + } + + group.bench_function("with_cache_95pct_hits", |b| { + let mut counter = 0; + b.iter(|| { + let token_num = if counter % 20 < 19 { + counter % 10 // Hit + } else { + 10 + (counter % 10) // Miss + }; + let token_id = format!("token_{}", black_box(token_num)); + let is_revoked = cache.check_revoked(&token_id, counter % 20 >= 19); // Only simulate latency on misses + black_box(is_revoked); + counter += 1; + }); + }); + + group.finish(); +} + +/// Benchmark 9: Memory overhead measurement +fn bench_memory_overhead(c: &mut Criterion) { + let cache = LocalRevocationCache::new(Duration::from_secs(60)); + + c.bench_function("cache_entry_insertion", |b| { + let mut counter = 0; + b.iter(|| { + let token_id = format!("token_{}", black_box(counter)); + cache.check_revoked(&token_id, false); + counter += 1; + }); + }); +} + +/// Benchmark 10: Realistic production workload +fn bench_production_workload(c: &mut Criterion) { + let cache = LocalRevocationCache::new(Duration::from_secs(60)); + + // Simulate production: 1000 active users, 95% cache hit rate, 1% revoked + for i in 0..1000 { + let token_id = if i % 100 == 0 { + format!("revoked_token_{}", i) + } else { + format!("valid_token_{}", i) + }; + cache.check_revoked(&token_id, false); + } + + c.bench_function("production_workload_simulation", |b| { + let mut counter = 0; + b.iter(|| { + // 95% hits to existing tokens, 5% new tokens + let token_id = if counter % 20 < 19 { + // Cache hit - existing token + let idx = counter % 1000; + if idx % 100 == 0 { + format!("revoked_token_{}", idx) + } else { + format!("valid_token_{}", idx) + } + } else { + // Cache miss - new token + format!("new_token_{}", counter) + }; + + let simulate_latency = counter % 20 >= 19; + let is_revoked = cache.check_revoked(&token_id, simulate_latency); + black_box(is_revoked); + counter += 1; + }); + }); + + // Report final statistics + let (hits, misses, hit_rate) = cache.stats(); + println!( + "\nProduction workload stats: {} hits, {} misses, {:.2}% hit rate", + hits, misses, hit_rate + ); +} + +criterion_group!( + revocation_cache_benches, + bench_cache_hit_latency, + bench_cache_miss_latency, + bench_hot_token_pattern, + bench_ttl_expiration, + bench_cache_size_impact, + bench_concurrent_access, + bench_mixed_revocation, + bench_cache_vs_no_cache, + bench_memory_overhead, + bench_production_workload +); + +criterion_main!(revocation_cache_benches); diff --git a/services/api/benches/routing_latency.rs b/services/api/benches/routing_latency.rs new file mode 100644 index 000000000..0b7b635d1 --- /dev/null +++ b/services/api/benches/routing_latency.rs @@ -0,0 +1,294 @@ +//! End-to-End Routing Latency Benchmark +//! +//! Measures complete request flow: +//! - Client request → Auth pipeline → Backend proxy → Response +//! - TARGET: <10μs total overhead (auth + proxying) +//! +//! Tests: +//! 1. Auth overhead only (mock backend) +//! 2. Proxy overhead only (no auth) +//! 3. Full end-to-end (auth + proxy) +//! 4. Different request sizes +//! 5. Concurrent requests + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::runtime::Runtime; + +/// Mock backend service that responds immediately +struct MockBackend { + response_time_ns: u64, +} + +impl MockBackend { + fn new(response_time_ns: u64) -> Self { + Self { response_time_ns } + } + + async fn handle_request(&self, _request: &[u8]) -> Vec { + // Simulate backend processing time + if self.response_time_ns > 0 { + tokio::time::sleep(Duration::from_nanos(self.response_time_ns)).await; + } + b"response_data".to_vec() + } +} + +/// Mock auth layer +struct MockAuthLayer { + overhead_ns: u64, +} + +impl MockAuthLayer { + fn new(overhead_ns: u64) -> Self { + Self { overhead_ns } + } + + async fn authenticate(&self, _token: &str) -> bool { + // Simulate auth overhead + if self.overhead_ns > 0 { + tokio::time::sleep(Duration::from_nanos(self.overhead_ns)).await; + } + true + } +} + +/// Request router +struct Router { + auth: Arc, + backend: Arc, +} + +impl Router { + fn new(auth_overhead_ns: u64, backend_response_ns: u64) -> Self { + Self { + auth: Arc::new(MockAuthLayer::new(auth_overhead_ns)), + backend: Arc::new(MockBackend::new(backend_response_ns)), + } + } + + async fn route_request(&self, token: &str, request: &[u8]) -> Option> { + // Authenticate + if !self.auth.authenticate(token).await { + return None; + } + + // Forward to backend + Some(self.backend.handle_request(request).await) + } +} + +/// Benchmark 1: Auth overhead only (backend responds instantly) +fn bench_auth_only_overhead(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let router = Router::new(5_000, 0); // 5μs auth, instant backend + + c.bench_function("auth_overhead_only", |b| { + b.iter(|| { + rt.block_on(async { + let result = router + .route_request(black_box("valid-jwt-token"), black_box(b"request")) + .await; + black_box(result); + }); + }); + }); +} + +/// Benchmark 2: Proxy overhead only (no auth) +fn bench_proxy_only_overhead(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let router = Router::new(0, 0); // No auth, instant backend + + c.bench_function("proxy_overhead_only", |b| { + b.iter(|| { + rt.block_on(async { + let result = router + .route_request(black_box("valid-jwt-token"), black_box(b"request")) + .await; + black_box(result); + }); + }); + }); +} + +/// Benchmark 3: Full end-to-end with realistic backend +fn bench_end_to_end_realistic(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + // 8μs auth + 100μs backend (simulates actual service) + let router = Router::new(8_000, 100_000); + + c.bench_function("end_to_end_realistic_backend", |b| { + b.iter(|| { + rt.block_on(async { + let result = router + .route_request(black_box("valid-jwt-token"), black_box(b"request")) + .await; + black_box(result); + }); + }); + }); +} + +/// Benchmark 4: Target performance (10μs total overhead) +fn bench_target_performance(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + // 8μs auth + instant backend = <10μs total + let router = Router::new(8_000, 0); + + c.bench_function("target_10us_overhead", |b| { + b.iter(|| { + rt.block_on(async { + let result = router + .route_request(black_box("valid-jwt-token"), black_box(b"request")) + .await; + black_box(result); + }); + }); + }); +} + +/// Benchmark 5: Different request sizes +fn bench_request_sizes(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let router = Router::new(8_000, 0); // 8μs auth overhead + + let mut group = c.benchmark_group("request_size_impact"); + + for size in &[100, 1_000, 10_000, 100_000] { + let request = vec![0u8; *size]; + + group.bench_with_input( + BenchmarkId::new("request_routing", format!("{}B", size)), + &request, + |b, req| { + b.iter(|| { + rt.block_on(async { + let result = router + .route_request("valid-jwt-token", black_box(req)) + .await; + black_box(result); + }); + }); + }, + ); + } + + group.finish(); +} + +/// Benchmark 6: Concurrent request handling +fn bench_concurrent_requests(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let router = Arc::new(Router::new(8_000, 0)); + + let mut group = c.benchmark_group("concurrent_requests"); + + for concurrency in &[1, 10, 100] { + group.bench_with_input( + BenchmarkId::new("parallel_routing", concurrency), + concurrency, + |b, &n| { + b.iter(|| { + rt.block_on(async { + let mut handles = vec![]; + let router_clone = router.clone(); + + for _ in 0..n { + let router_ref = router_clone.clone(); + let handle = tokio::spawn(async move { + router_ref + .route_request("valid-jwt-token", b"request") + .await + }); + handles.push(handle); + } + + for handle in handles { + black_box(handle.await.unwrap()); + } + }); + }); + }, + ); + } + + group.finish(); +} + +/// Benchmark 7: Failed auth fast-path +fn bench_auth_failure_fast_path(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + + // Custom router that fails auth immediately + struct FastFailRouter { + auth_overhead_ns: u64, + } + + impl FastFailRouter { + async fn route_request(&self, token: &str, _request: &[u8]) -> Option> { + if self.auth_overhead_ns > 0 { + tokio::time::sleep(Duration::from_nanos(self.auth_overhead_ns)).await; + } + + // Simulate quick rejection (invalid JWT signature) + if token.len() < 10 { + return None; + } + None + } + } + + let router = FastFailRouter { + auth_overhead_ns: 100, // 100ns for quick rejection + }; + + c.bench_function("auth_failure_fast_path", |b| { + b.iter(|| { + rt.block_on(async { + let result = router + .route_request(black_box("invalid"), black_box(b"request")) + .await; + black_box(result); + }); + }); + }); +} + +/// Benchmark 8: Latency percentiles measurement +fn bench_latency_percentiles(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let router = Router::new(8_000, 0); + + c.bench_function("latency_distribution", |b| { + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + + for _ in 0..iters { + let start = Instant::now(); + rt.block_on(async { + let result = router.route_request("valid-jwt-token", b"request").await; + black_box(result); + }); + total += start.elapsed(); + } + + total + }); + }); +} + +criterion_group!( + routing_benches, + bench_auth_only_overhead, + bench_proxy_only_overhead, + bench_end_to_end_realistic, + bench_target_performance, + bench_request_sizes, + bench_concurrent_requests, + bench_auth_failure_fast_path, + bench_latency_percentiles +); + +criterion_main!(routing_benches); diff --git a/services/api/benches/throughput.rs b/services/api/benches/throughput.rs new file mode 100644 index 000000000..5e531e872 --- /dev/null +++ b/services/api/benches/throughput.rs @@ -0,0 +1,448 @@ +//! Throughput Benchmark - Concurrent Authenticated Requests +//! +//! Measures maximum requests per second: +//! - TARGET: >100,000 req/s single-threaded +//! - Multi-threaded scaling +//! - Different authentication workloads +//! - Realistic traffic patterns + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::runtime::Runtime; + +/// Lightweight auth simulator +struct AuthSimulator { + success_rate: f64, + counter: Arc, +} + +impl AuthSimulator { + fn new(success_rate: f64) -> Self { + Self { + success_rate, + counter: Arc::new(AtomicU64::new(0)), + } + } + + async fn authenticate(&self, _request_id: u64) -> bool { + let count = self.counter.fetch_add(1, Ordering::Relaxed); + // Simulate success rate + (count as f64 / 100.0) % 1.0 < self.success_rate + } + + fn requests_processed(&self) -> u64 { + self.counter.load(Ordering::Relaxed) + } +} + +/// Request handler +struct RequestHandler { + auth: Arc, +} + +impl RequestHandler { + fn new(auth: Arc) -> Self { + Self { auth } + } + + async fn handle_request(&self, request_id: u64) -> bool { + self.auth.authenticate(request_id).await + } +} + +/// Benchmark 1: Single-threaded throughput (TARGET: >100K req/s) +fn bench_single_threaded_throughput(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let auth = Arc::new(AuthSimulator::new(0.95)); // 95% success rate + let handler = RequestHandler::new(auth.clone()); + + let mut group = c.benchmark_group("single_threaded_throughput"); + group.throughput(Throughput::Elements(1)); + + group.bench_function("100k_req_target", |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + rt.block_on(async { + for i in 0..iters { + black_box(handler.handle_request(i).await); + } + }); + start.elapsed() + }); + }); + + group.finish(); +} + +/// Benchmark 2: Multi-threaded throughput +fn bench_multi_threaded_throughput(c: &mut Criterion) { + let mut group = c.benchmark_group("multi_threaded_throughput"); + + for num_threads in &[1, 2, 4, 8, 16] { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(*num_threads) + .build() + .unwrap(); + + let auth = Arc::new(AuthSimulator::new(0.95)); + let handler = Arc::new(RequestHandler::new(auth.clone())); + + group.throughput(Throughput::Elements(1000)); + + group.bench_with_input( + BenchmarkId::new("concurrent_requests", num_threads), + num_threads, + |b, &threads| { + b.iter_custom(|iters| { + let start = Instant::now(); + rt.block_on(async { + let mut handles = vec![]; + let requests_per_thread = iters / threads as u64; + + for _ in 0..threads { + let handler_clone = handler.clone(); + let handle = tokio::spawn(async move { + for i in 0..requests_per_thread { + black_box(handler_clone.handle_request(i).await); + } + }); + handles.push(handle); + } + + for handle in handles { + handle.await.unwrap(); + } + }); + start.elapsed() + }); + }, + ); + } + + group.finish(); +} + +/// Benchmark 3: Different success rates +fn bench_success_rate_impact(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let mut group = c.benchmark_group("success_rate_impact"); + + for success_rate in &[0.5, 0.8, 0.95, 0.99, 1.0] { + let auth = Arc::new(AuthSimulator::new(*success_rate)); + let handler = RequestHandler::new(auth.clone()); + + group.throughput(Throughput::Elements(1000)); + + group.bench_with_input( + BenchmarkId::new("throughput", format!("{}%", (success_rate * 100.0) as u32)), + success_rate, + |b, _rate| { + b.iter_custom(|iters| { + let start = Instant::now(); + rt.block_on(async { + for i in 0..iters { + black_box(handler.handle_request(i).await); + } + }); + start.elapsed() + }); + }, + ); + } + + group.finish(); +} + +/// Benchmark 4: Burst traffic patterns +fn bench_burst_patterns(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let auth = Arc::new(AuthSimulator::new(0.95)); + let handler = Arc::new(RequestHandler::new(auth.clone())); + + let mut group = c.benchmark_group("burst_patterns"); + + // Constant load + group.bench_function("constant_load_1000_req", |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + rt.block_on(async { + for i in 0..iters.min(1000) { + black_box(handler.handle_request(i).await); + } + }); + start.elapsed() + }); + }); + + // Burst pattern: All at once + group.bench_function("burst_1000_concurrent", |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + rt.block_on(async { + let mut handles = vec![]; + for i in 0..iters.min(1000) { + let handler_clone = handler.clone(); + let handle = tokio::spawn(async move { handler_clone.handle_request(i).await }); + handles.push(handle); + } + for handle in handles { + black_box(handle.await.unwrap()); + } + }); + start.elapsed() + }); + }); + + group.finish(); +} + +/// Benchmark 5: Request size impact on throughput +fn bench_request_size_throughput(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + + struct RequestProcessor { + auth: Arc, + } + + impl RequestProcessor { + async fn process(&self, _data: &[u8]) -> bool { + self.auth.authenticate(0).await + } + } + + let auth = Arc::new(AuthSimulator::new(0.95)); + let processor = RequestProcessor { auth: auth.clone() }; + + let mut group = c.benchmark_group("request_size_throughput"); + + for size in &[100, 1_000, 10_000, 100_000] { + let data = vec![0u8; *size]; + + group.throughput(Throughput::Bytes(*size as u64)); + + group.bench_with_input( + BenchmarkId::new("process_request", format!("{}B", size)), + &data, + |b, request_data| { + b.iter_custom(|iters| { + let start = Instant::now(); + rt.block_on(async { + for _ in 0..iters { + black_box(processor.process(request_data).await); + } + }); + start.elapsed() + }); + }, + ); + } + + group.finish(); +} + +/// Benchmark 6: Sustained throughput over time +fn bench_sustained_throughput(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let auth = Arc::new(AuthSimulator::new(0.95)); + let handler = RequestHandler::new(auth.clone()); + + c.bench_function("sustained_1_second", |b| { + b.iter_custom(|_iters| { + let start = Instant::now(); + let mut count = 0u64; + + rt.block_on(async { + let end_time = Instant::now() + Duration::from_secs(1); + while Instant::now() < end_time { + black_box(handler.handle_request(count).await); + count += 1; + } + }); + + let elapsed = start.elapsed(); + let rps = count as f64 / elapsed.as_secs_f64(); + println!("Sustained throughput: {:.0} req/s", rps); + elapsed + }); + }); +} + +/// Benchmark 7: Request rate limits +fn bench_rate_limited_throughput(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + + struct RateLimitedHandler { + auth: Arc, + limit: AtomicU64, + max_rps: u64, + } + + impl RateLimitedHandler { + fn new(auth: Arc, max_rps: u64) -> Self { + Self { + auth, + limit: AtomicU64::new(0), + max_rps, + } + } + + async fn handle(&self, request_id: u64) -> bool { + let count = self.limit.fetch_add(1, Ordering::Relaxed); + if count >= self.max_rps { + return false; // Rate limited + } + self.auth.authenticate(request_id).await + } + } + + let mut group = c.benchmark_group("rate_limited_throughput"); + + for limit in &[1_000, 10_000, 100_000] { + let auth = Arc::new(AuthSimulator::new(0.95)); + let handler = RateLimitedHandler::new(auth.clone(), *limit); + + group.bench_with_input(BenchmarkId::new("max_rps", limit), limit, |b, _| { + b.iter_custom(|iters| { + let start = Instant::now(); + rt.block_on(async { + for i in 0..iters { + black_box(handler.handle(i).await); + } + }); + start.elapsed() + }); + }); + } + + group.finish(); +} + +/// Benchmark 8: HFT scenario (TARGET: 100K req/s minimum) +fn bench_hft_scenario(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let auth = Arc::new(AuthSimulator::new(0.99)); // 99% success (HFT quality) + let handler = RequestHandler::new(auth.clone()); + + let mut group = c.benchmark_group("hft_scenario"); + group.throughput(Throughput::Elements(100_000)); + + group.bench_function("hft_100k_target", |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + rt.block_on(async { + for i in 0..iters { + black_box(handler.handle_request(i).await); + } + }); + let elapsed = start.elapsed(); + + // Calculate actual throughput + let rps = iters as f64 / elapsed.as_secs_f64(); + if rps < 100_000.0 { + println!("⚠️ Below target: {:.0} req/s (target: 100K)", rps); + } else { + println!("✓ Target met: {:.0} req/s", rps); + } + + elapsed + }); + }); + + group.finish(); +} + +/// Benchmark 9: Latency under load +fn bench_latency_under_load(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let auth = Arc::new(AuthSimulator::new(0.95)); + let handler = Arc::new(RequestHandler::new(auth.clone())); + + let mut group = c.benchmark_group("latency_under_load"); + + for load in &[100, 1_000, 10_000, 100_000] { + group.bench_with_input( + BenchmarkId::new("requests_in_flight", load), + load, + |b, &n| { + b.iter_custom(|_iters| { + let start = Instant::now(); + rt.block_on(async { + let mut handles = vec![]; + for i in 0..n { + let handler_clone = handler.clone(); + let handle = + tokio::spawn(async move { handler_clone.handle_request(i).await }); + handles.push(handle); + } + for handle in handles { + black_box(handle.await.unwrap()); + } + }); + start.elapsed() + }); + }, + ); + } + + group.finish(); +} + +/// Benchmark 10: Request batching efficiency +fn bench_batching_efficiency(c: &mut Criterion) { + let rt = Runtime::new().unwrap(); + let auth = Arc::new(AuthSimulator::new(0.95)); + let handler = Arc::new(RequestHandler::new(auth.clone())); + + let mut group = c.benchmark_group("batching_efficiency"); + + for batch_size in &[1, 10, 100, 1000] { + group.throughput(Throughput::Elements(*batch_size as u64)); + + group.bench_with_input( + BenchmarkId::new("batch_processing", batch_size), + batch_size, + |b, &n| { + b.iter_custom(|iters| { + let start = Instant::now(); + rt.block_on(async { + for batch in 0..(iters / n as u64) { + let mut handles = vec![]; + for i in 0..n { + let handler_clone = handler.clone(); + let request_id = batch * n as u64 + i as u64; + let handle = tokio::spawn(async move { + handler_clone.handle_request(request_id).await + }); + handles.push(handle); + } + for handle in handles { + black_box(handle.await.unwrap()); + } + } + }); + start.elapsed() + }); + }, + ); + } + + group.finish(); +} + +criterion_group!( + throughput_benches, + bench_single_threaded_throughput, + bench_multi_threaded_throughput, + bench_success_rate_impact, + bench_burst_patterns, + bench_request_size_throughput, + bench_sustained_throughput, + bench_rate_limited_throughput, + bench_hft_scenario, + bench_latency_under_load, + bench_batching_efficiency +); + +criterion_main!(throughput_benches); diff --git a/services/api/build.rs b/services/api/build.rs new file mode 100644 index 000000000..b82355483 --- /dev/null +++ b/services/api/build.rs @@ -0,0 +1,194 @@ +//! Build script for API Gateway service +//! +//! Compiles protobuf definitions from the canonical proto/ directory at workspace root. +//! Proto packages: +//! - `foxhunt.tli` (fxt_trading.proto) - Fat-client unified interface (server+client) +//! - `trading` (trading.proto) - Backend trading service (client-only) +//! - `risk` (risk.proto) - Risk service (client-only) +//! - `monitoring` (monitoring.proto) - Monitoring service (server+client) +//! - `config` (config.proto) - Config service (server+client) +//! - `ml_training` (ml_training.proto) - ML Training service (server+client) +//! - `trading_agent` (trading_agent.proto) - Trading Agent service (server+client) + +fn main() -> Result<(), Box> { + // NOTE: Tonic 0.14+ uses tonic_prost_build instead of tonic_build + let config = tonic_prost_build::configure(); + + let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR")?); + + // Compile fat-client TLI proto (package: foxhunt.tli) — server+client + // Contains TradingService, BacktestingService, and MLService + // API Gateway acts as server for incoming client requests + config + .clone() + .build_server(true) + .build_client(true) + .file_descriptor_set_path(out_dir.join("tli_descriptor.bin")) + .compile_well_known_types(true) + .extern_path(".google.protobuf", "::prost_types") + .type_attribute(".", "#[derive(serde::Serialize, serde::Deserialize)]") + .server_mod_attribute(".", "#[allow(unused_qualifications)]") + .client_mod_attribute(".", "#[allow(unused_qualifications)]") + .compile_protos( + &["../../proto/fxt_trading.proto"], + &["../../proto"], + )?; + + // Compile Trading Service backend proto (package: trading) — server+client + // API Gateway serves trading.TradingService and forwards to Trading Service backend + config + .clone() + .build_server(true) + .build_client(true) + .file_descriptor_set_path(out_dir.join("trading_descriptor.bin")) + .compile_well_known_types(true) + .extern_path(".google.protobuf", "::prost_types") + .type_attribute(".", "#[derive(serde::Serialize, serde::Deserialize)]") + .server_mod_attribute(".", "#[allow(unused_qualifications)]") + .client_mod_attribute(".", "#[allow(unused_qualifications)]") + .compile_protos( + &["../../proto/trading.proto"], + &["../../proto"], + )?; + + // Compile Risk Service proto (package: risk) — server+client + config + .clone() + .build_server(true) + .build_client(true) + .file_descriptor_set_path(out_dir.join("risk_descriptor.bin")) + .compile_well_known_types(true) + .extern_path(".google.protobuf", "::prost_types") + .type_attribute(".", "#[derive(serde::Serialize, serde::Deserialize)]") + .server_mod_attribute(".", "#[allow(unused_qualifications)]") + .client_mod_attribute(".", "#[allow(unused_qualifications)]") + .compile_protos( + &["../../proto/risk.proto"], + &["../../proto"], + )?; + + // Compile Monitoring Service proto (package: monitoring) — server+client + config + .clone() + .build_server(true) + .build_client(true) + .file_descriptor_set_path(out_dir.join("monitoring_descriptor.bin")) + .compile_well_known_types(true) + .extern_path(".google.protobuf", "::prost_types") + .type_attribute(".", "#[derive(serde::Serialize, serde::Deserialize)]") + .server_mod_attribute(".", "#[allow(unused_qualifications)]") + .client_mod_attribute(".", "#[allow(unused_qualifications)]") + .compile_protos( + &["../../proto/monitoring.proto"], + &["../../proto"], + )?; + + // Compile Config backend proto (package: config) — server+client + config + .clone() + .build_server(true) + .build_client(true) + .file_descriptor_set_path(out_dir.join("config_descriptor.bin")) + .compile_well_known_types(true) + .extern_path(".google.protobuf", "::prost_types") + .type_attribute(".", "#[derive(serde::Serialize, serde::Deserialize)]") + .server_mod_attribute(".", "#[allow(unused_qualifications)]") + .client_mod_attribute(".", "#[allow(unused_qualifications)]") + .compile_protos( + &["../../proto/config.proto"], + &["../../proto"], + )?; + + // Compile ML Training Service proto (package: ml_training) — server+client + config + .clone() + .build_server(true) + .build_client(true) + .file_descriptor_set_path(out_dir.join("ml_training_descriptor.bin")) + .compile_well_known_types(true) + .extern_path(".google.protobuf", "::prost_types") + .type_attribute(".", "#[derive(serde::Serialize, serde::Deserialize)]") + .server_mod_attribute(".", "#[allow(unused_qualifications)]") + .client_mod_attribute(".", "#[allow(unused_qualifications)]") + .compile_protos( + &["../../proto/ml_training.proto"], + &["../../proto"], + )?; + + // Compile Trading Agent Service proto (package: trading_agent) — server+client + config + .clone() + .build_server(true) + .build_client(true) + .file_descriptor_set_path(out_dir.join("trading_agent_descriptor.bin")) + .compile_well_known_types(true) + .extern_path(".google.protobuf", "::prost_types") + .type_attribute(".", "#[derive(serde::Serialize, serde::Deserialize)]") + .server_mod_attribute(".", "#[allow(unused_qualifications)]") + .client_mod_attribute(".", "#[allow(unused_qualifications)]") + .compile_protos( + &["../../proto/trading_agent.proto"], + &["../../proto"], + )?; + + // Compile Broker Gateway Service proto (package: broker_gateway) — server+client + config + .clone() + .build_server(true) + .build_client(true) + .file_descriptor_set_path(out_dir.join("broker_gateway_descriptor.bin")) + .compile_well_known_types(true) + .extern_path(".google.protobuf", "::prost_types") + .type_attribute(".", "#[derive(serde::Serialize, serde::Deserialize)]") + .server_mod_attribute(".", "#[allow(unused_qualifications)]") + .client_mod_attribute(".", "#[allow(unused_qualifications)]") + .compile_protos( + &["../../proto/broker_gateway.proto"], + &["../../proto"], + )?; + + // Compile Data Acquisition Service proto (package: data_acquisition) — server+client + config + .clone() + .build_server(true) + .build_client(true) + .file_descriptor_set_path(out_dir.join("data_acquisition_descriptor.bin")) + .compile_well_known_types(true) + .extern_path(".google.protobuf", "::prost_types") + .type_attribute(".", "#[derive(serde::Serialize, serde::Deserialize)]") + .server_mod_attribute(".", "#[allow(unused_qualifications)]") + .client_mod_attribute(".", "#[allow(unused_qualifications)]") + .compile_protos( + &["../../proto/data_acquisition.proto"], + &["../../proto"], + )?; + + // Compile ML Inference Service proto (package: ml) — server+client + config + .clone() + .build_server(true) + .build_client(true) + .file_descriptor_set_path(out_dir.join("ml_descriptor.bin")) + .compile_well_known_types(true) + .extern_path(".google.protobuf", "::prost_types") + .type_attribute(".", "#[derive(serde::Serialize, serde::Deserialize)]") + .server_mod_attribute(".", "#[allow(unused_qualifications)]") + .client_mod_attribute(".", "#[allow(unused_qualifications)]") + .compile_protos( + &["../../proto/ml.proto"], + &["../../proto"], + )?; + + println!("cargo:rerun-if-changed=../../proto/fxt_trading.proto"); + println!("cargo:rerun-if-changed=../../proto/trading.proto"); + println!("cargo:rerun-if-changed=../../proto/risk.proto"); + println!("cargo:rerun-if-changed=../../proto/monitoring.proto"); + println!("cargo:rerun-if-changed=../../proto/config.proto"); + println!("cargo:rerun-if-changed=../../proto/ml_training.proto"); + println!("cargo:rerun-if-changed=../../proto/trading_agent.proto"); + println!("cargo:rerun-if-changed=../../proto/broker_gateway.proto"); + println!("cargo:rerun-if-changed=../../proto/data_acquisition.proto"); + println!("cargo:rerun-if-changed=../../proto/ml.proto"); + + Ok(()) +} diff --git a/services/api/src/auth/interceptor.rs b/services/api/src/auth/interceptor.rs new file mode 100644 index 000000000..8659de242 --- /dev/null +++ b/services/api/src/auth/interceptor.rs @@ -0,0 +1,1071 @@ +//! High-Performance gRPC Authentication Interceptor with 6-Layer Security +//! +//! This module implements a comprehensive authentication system optimized for HFT requirements: +//! - Total overhead: <10μs per request +//! - JWT validation: <1μs (cached keys) +//! - Revocation check: <500ns (Redis in-memory) +//! - Authorization: <100ns (cached permissions) +//! - Rate limiting: <50ns (in-memory counters) +//! +//! ## 6-Layer Authentication Architecture +//! +//! ```text +//! Layer 1: mTLS Client Certificate (handled by tonic-tls) +//! Layer 2: JWT Extraction from Authorization header +//! Layer 3: JWT Revocation Check (Redis - <500ns) +//! Layer 4: JWT Signature & Expiration Validation (<1μs) +//! Layer 5: RBAC Permission Check (<100ns) +//! Layer 6: Rate Limiting (<50ns) +//! Layer 7: User Context Injection (metadata enrichment) +//! Layer 8: Async Audit Logging (non-blocking) +//! ``` + +use anyhow::{Context, Result}; +use dashmap::DashMap; +use governor::{state::keyed::DefaultKeyedStateStore, Quota, RateLimiter as GovernorRateLimiter}; +use jsonwebtoken::{decode, Algorithm, DecodingKey, Validation}; +use redis::{aio::ConnectionManager, AsyncCommands}; +use serde::{Deserialize, Serialize}; +use std::num::NonZeroU32; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use tonic::{Request, Status}; +use tracing::{debug, error, info, warn}; +use uuid::Uuid; + +/// JWT Token ID (`JTI`) for unique token identification and revocation +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Jti(pub String); + +impl Jti { + /// Create new random JTI + pub fn new() -> Self { + Self(Uuid::new_v4().to_string()) + } + + /// Parse `JTI` from string + pub fn from_string(s: String) -> Self { + Self(s) + } + + /// Get Redis key for this JTI + fn redis_key(&self) -> String { + format!("jwt:blacklist:{}", self.0) + } + + /// Get `JTI` as string reference + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Default for Jti { + fn default() -> Self { + Self::new() + } +} + +/// Enhanced JWT claims with revocation support +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct JwtClaims { + /// JWT ID for revocation tracking (MANDATORY) + pub jti: String, + /// Subject (user ID) + pub sub: String, + /// Issued at timestamp + pub iat: u64, + /// Expiration timestamp + pub exp: u64, + /// Not before timestamp (optional) + pub nbf: Option, + /// Issuer + pub iss: String, + /// Audience + pub aud: String, + /// User roles (for RBAC) + pub roles: Vec, + /// Granular permissions + pub permissions: Vec, + /// Token type: "access" or "refresh" + #[serde(default = "default_token_type")] + pub token_type: String, + /// Session ID for tracking related tokens + #[serde(default)] + pub session_id: Option, +} + +fn default_token_type() -> String { + "access".to_string() +} + +/// User context injected into gRPC metadata after successful authentication +#[derive(Debug, Clone)] +pub struct UserContext { + pub user_id: String, + pub roles: Vec, + pub permissions: Vec, + pub session_id: String, + pub authenticated_at: Instant, +} + +/// Cached revocation result with timestamp +#[derive(Debug, Clone)] +struct CachedRevocationResult { + is_revoked: bool, + cached_at: Instant, +} + +/// Local in-memory cache for revocation checks +/// +/// PERFORMANCE: Reduces Redis network latency (500μs → <10ns for cache hits) +pub struct LocalRevocationCache { + cache: Arc>, + ttl: Duration, + hits: Arc, + misses: Arc, +} + +impl LocalRevocationCache { + /// Create new local revocation cache + /// + /// # Arguments + /// * `ttl` - Time-to-live for cached entries (recommended: 60s) + pub fn new(ttl: Duration) -> Self { + Self { + cache: Arc::new(DashMap::new()), + ttl, + hits: Arc::new(std::sync::atomic::AtomicU64::new(0)), + misses: Arc::new(std::sync::atomic::AtomicU64::new(0)), + } + } + + /// Check if token is revoked (with local cache) + /// + /// # Performance + /// - Cache hit: <10ns (DashMap lookup) + /// + /// - Cache miss: ~500μs (Redis network latency) + /// - Expected hit rate: >95% + pub async fn check_revoked( + &self, + token_id: &str, + redis: &mut ConnectionManager, + ) -> Result { + // Check cache first + if let Some(entry) = self.cache.get(token_id) { + if entry.cached_at.elapsed() < self.ttl { + // Cache hit - increment counter + self.hits.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + return Ok(entry.is_revoked); + } else { + // Entry expired - remove it + drop(entry); // Release read lock + self.cache.remove(token_id); + } + } + + // Cache miss - check Redis + self.misses + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + + let key = format!("jwt:blacklist:{}", token_id); + let is_revoked: bool = redis + .exists(&key) + .await + .context("Failed to check token revocation status in Redis")?; + + // Update cache + self.cache.insert( + token_id.to_string(), + CachedRevocationResult { + is_revoked, + cached_at: Instant::now(), + }, + ); + + Ok(is_revoked) + } + + /// Invalidate cache entry for a specific token + /// + /// Called when a token is revoked to immediately reflect the change + pub fn invalidate(&self, token_id: &str) { + self.cache.remove(token_id); + } + + /// Clear all cached entries + pub fn clear(&self) { + self.cache.clear(); + } + + /// Get cache statistics + pub fn stats(&self) -> CacheStats { + let hits = self.hits.load(std::sync::atomic::Ordering::Relaxed); + let misses = self.misses.load(std::sync::atomic::Ordering::Relaxed); + let total = hits.saturating_add(misses); + let hit_rate = if total > 0 { + let rate = (hits as f64 / total as f64) * 100.0; + // f64 multiplication is safe, check for valid result + if rate.is_finite() { + rate + } else { + 0.0 + } + } else { + 0.0 + }; + + CacheStats { + hits, + misses, + total, + hit_rate, + entries: self.cache.len(), + } + } + + /// Reset cache statistics + pub fn reset_stats(&self) { + self.hits.store(0, std::sync::atomic::Ordering::Relaxed); + self.misses.store(0, std::sync::atomic::Ordering::Relaxed); + } +} + +/// Cache statistics for monitoring +#[derive(Debug, Clone)] +pub struct CacheStats { + pub hits: u64, + pub misses: u64, + pub total: u64, + pub hit_rate: f64, + pub entries: usize, +} + +/// JWT Revocation Service (Redis-backed with local cache) +#[derive(Clone)] +pub struct RevocationService { + redis: ConnectionManager, + cache: Arc, +} + +impl RevocationService { + /// Create new revocation service with local cache + /// + /// # Arguments + /// * `redis_url` - Redis connection URL + /// + /// * `cache_ttl` - TTL for local cache entries (default: 60s) + pub async fn new(redis_url: &str) -> Result { + Self::new_with_cache_ttl(redis_url, Duration::from_secs(60)).await + } + + /// Create new revocation service with custom cache TTL + pub async fn new_with_cache_ttl(redis_url: &str, cache_ttl: Duration) -> Result { + let client = redis::Client::open(redis_url) + .context("Failed to create Redis client for revocation service")?; + + let redis = ConnectionManager::new(client) + .await + .context("Failed to connect to Redis for revocation service")?; + + Ok(Self { + redis, + cache: Arc::new(LocalRevocationCache::new(cache_ttl)), + }) + } + + /// Check if token is revoked (TARGET: <10ns for cache hits, <500μs for cache misses) + /// + /// PERFORMANCE: Uses local DashMap cache to avoid Redis network latency + pub async fn is_revoked(&self, jti: &Jti) -> Result { + let mut conn = self.redis.clone(); + self.cache.check_revoked(jti.as_str(), &mut conn).await + } + + /// Add token to blacklist (for revocation) + pub async fn revoke_token(&self, jti: &Jti, ttl_seconds: u64) -> Result<()> { + if ttl_seconds == 0 { + return Ok(()); // Already expired + } + + let key = jti.redis_key(); + let mut conn = self.redis.clone(); + + // Set with TTL = remaining token lifetime + // Redis will auto-clean when token would have expired + let _: () = conn + .set_ex(&key, "revoked", ttl_seconds) + .await + .context("Failed to add token to blacklist")?; + + // Invalidate cache entry to immediately reflect revocation + self.cache.invalidate(jti.as_str()); + + Ok(()) + } + + /// Get cache statistics for monitoring + pub fn cache_stats(&self) -> CacheStats { + self.cache.stats() + } + + /// Clear local cache (useful for testing or troubleshooting) + pub fn clear_cache(&self) { + self.cache.clear(); + } + + /// Reset cache statistics + pub fn reset_cache_stats(&self) { + self.cache.reset_stats(); + } +} + +/// High-performance JWT validator with key caching +pub struct JwtService { + /// Cached decoding key (avoids repeated parsing) + decoding_key: Arc, + /// Validation rules + validation: Validation, +} + +impl JwtService { + /// Create new JWT service with cached key + /// + /// PERFORMANCE: Key is parsed once and cached in Arc for <1μs validation + pub fn new(secret: String, issuer: String, audience: String) -> Self { + let decoding_key = Arc::new(DecodingKey::from_secret(secret.as_bytes())); + + let mut validation = Validation::new(Algorithm::HS256); + validation.set_issuer(&[&issuer]); + validation.set_audience(&[&audience]); + validation.validate_exp = true; + validation.validate_nbf = true; + validation.leeway = 0; // Strict for HFT security + + Self { + decoding_key, + validation, + } + } + + /// Validate JWT and extract claims (TARGET: <1μs with cached key) + /// + /// PERFORMANCE: Cached DecodingKey eliminates key parsing overhead + pub fn validate_token(&self, token: &str) -> Result { + // Basic validation before expensive decode + if token.is_empty() || token.len() > 8192 { + return Err(anyhow::anyhow!("Invalid token format")); + } + + let token_data = + decode::(token, &self.decoding_key, &self.validation).map_err(|e| { + error!("JWT decode failed in interceptor: {:?}", e); + let issuer_str = self + .validation + .iss + .as_ref() + .map(|v| v.iter().cloned().collect::>().join(", ")) + .unwrap_or_default(); + let audience_str = self + .validation + .aud + .as_ref() + .map(|v| v.iter().cloned().collect::>().join(", ")) + .unwrap_or_default(); + error!( + "Expected issuer: {}, audience: {}", + issuer_str, audience_str + ); + anyhow::anyhow!("JWT validation failed: {}", e) + })?; + + // Additional security checks + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("System time error")? + .as_secs(); + + if token_data.claims.exp <= now { + return Err(anyhow::anyhow!("Token expired")); + } + + if token_data.claims.jti.is_empty() { + return Err(anyhow::anyhow!("Missing JTI (required for revocation)")); + } + + if token_data.claims.sub.is_empty() { + return Err(anyhow::anyhow!("Missing subject claim")); + } + + // Check if token was issued in the future (clock skew attack) + if token_data.claims.iat > now + 60 { + return Err(anyhow::anyhow!("Token issued in the future")); + } + + // Check token age (max 1 hour from issuance) + let token_age = now + .checked_sub(token_data.claims.iat) + .ok_or_else(|| anyhow::anyhow!("Invalid token timestamp (iat in future)"))?; + + if token_age > 3600 { + return Err(anyhow::anyhow!("Token too old (max age: 1 hour)")); + } + + Ok(token_data.claims) + } +} + +/// High-performance authorization service with permission caching +pub struct AuthzService { + /// Cached user permissions (TARGET: <100ns lookups) + /// + /// PERFORMANCE: DashMap provides concurrent access without RwLock overhead + permission_cache: Arc>>, +} + +impl Default for AuthzService { + fn default() -> Self { + Self::new() + } +} + +impl AuthzService { + pub fn new() -> Self { + Self { + permission_cache: Arc::new(DashMap::new()), + } + } + + /// Check if user has required permission (TARGET: <100ns) + /// + /// PERFORMANCE: In-memory DashMap lookup, no database queries + pub fn has_permission(&self, user_id: &str, permission: &str) -> bool { + if let Some(permissions) = self.permission_cache.get(user_id) { + permissions.contains(&permission.to_string()) + } else { + false + } + } + + /// Cache user permissions for fast lookups + /// + /// Called during user context injection (Layer 7) + pub fn cache_permissions(&self, user_id: String, permissions: Vec) { + self.permission_cache.insert(user_id, permissions); + } + + /// Clear cached permissions (for revocation or permission updates) + pub fn clear_cache(&self, user_id: &str) { + self.permission_cache.remove(user_id); + } +} + +/// High-performance rate limiter with in-memory counters +#[derive(Clone)] +pub struct RateLimiter { + /// Per-user rate limiters (TARGET: <50ns) + /// + /// PERFORMANCE: Governor provides O(1) atomic counter checks + limiters: Arc< + DashMap< + String, + Arc< + GovernorRateLimiter< + String, + DefaultKeyedStateStore, + governor::clock::DefaultClock, + >, + >, + >, + >, + /// Default quota (requests per second) + default_quota: Quota, +} + +impl RateLimiter { + pub fn new(requests_per_second: u32) -> Result { + let default_quota = + Quota::per_second(NonZeroU32::new(requests_per_second).ok_or_else(|| { + format!("Invalid rate limit: {} (must be > 0)", requests_per_second) + })?); + + Ok(Self { + limiters: Arc::new(DashMap::new()), + default_quota, + }) + } + + /// Check if request is allowed (TARGET: <50ns) + /// + /// PERFORMANCE: Atomic counter increment, no locks + pub fn check_rate_limit(&self, user_id: &str) -> bool { + let limiter = self + .limiters + .entry(user_id.to_string()) + .or_insert_with(|| Arc::new(GovernorRateLimiter::dashmap(self.default_quota))); + + limiter.check_key(&user_id.to_string()).is_ok() + } +} + +/// Async audit logger (non-blocking) +pub struct AuditLogger { + /// Audit log buffer (async writes) + enabled: bool, +} + +impl AuditLogger { + pub fn new(enabled: bool) -> Self { + Self { enabled } + } + + /// Log authentication success (non-blocking) + /// + /// PERFORMANCE: Spawns background task, does not block request path + pub fn log_auth_success(&self, user_id: &str, client_ip: Option<&str>) { + if !self.enabled { + return; + } + + let user_id = user_id.to_string(); + let client_ip = client_ip.map(|s| s.to_string()); + + tokio::spawn(async move { + info!( + user_id = %user_id, + client_ip = ?client_ip, + "Authentication successful" + ); + }); + } + + /// Log authentication failure (non-blocking) + pub fn log_auth_failure(&self, reason: &str, client_ip: Option<&str>) { + if !self.enabled { + return; + } + + let reason = reason.to_string(); + let client_ip = client_ip.map(|s| s.to_string()); + + tokio::spawn(async move { + warn!( + reason = %reason, + client_ip = ?client_ip, + "Authentication failed" + ); + }); + } +} + +/// Main authentication interceptor with 6-layer security +#[derive(Clone)] +pub struct AuthInterceptor { + jwt_service: Arc, + revocation_service: Arc, + authz_service: Arc, + rate_limiter: Arc, + audit_logger: Arc, +} + +impl AuthInterceptor { + /// Create new authentication interceptor + pub fn new( + jwt_service: JwtService, + revocation_service: RevocationService, + authz_service: AuthzService, + rate_limiter: RateLimiter, + audit_logger: AuditLogger, + ) -> Self { + Self { + jwt_service: Arc::new(jwt_service), + revocation_service: Arc::new(revocation_service), + authz_service: Arc::new(authz_service), + rate_limiter: Arc::new(rate_limiter), + audit_logger: Arc::new(audit_logger), + } + } + + /// Authenticate request with 6-layer security (TARGET: <10μs total) + /// + /// ## Performance Breakdown (Target vs Actual) + /// - Layer 1 (mTLS): Handled by tonic-tls (0μs in-band) + /// + /// - Layer 2 (Extract JWT): <100ns (header lookup) + /// - Layer 3 (Revocation): <500ns (Redis in-memory) + /// + /// - Layer 4 (JWT Validation): <1μs (cached key) + /// - Layer 5 (Authorization): <100ns (cached permissions) + /// + /// - Layer 6 (Rate Limit): <50ns (atomic counter) + /// - Layer 7 (Context Inject): <100ns (metadata write) + /// + /// - Layer 8 (Audit Log): 0ns (async, non-blocking) + /// **Total**: ~2μs (well under 10μs target) + pub async fn authenticate(&self, mut request: Request) -> Result, Status> { + let start = Instant::now(); + + // Extract client IP for audit logging + let client_ip = self.extract_client_ip(&request); + + // Layer 1: mTLS client certificate + // NOTE: Handled by tonic-tls at transport layer, certificate is already validated + // We can extract it from request extensions if needed for additional checks + + // Layer 2: Extract JWT from authorization header (~100ns) + let bearer_token = self.extract_bearer_token(&request).ok_or_else(|| { + self.audit_logger + .log_auth_failure("missing_token", client_ip.as_deref()); + Status::unauthenticated("Authorization header required") + })?; + + // Layer 4: Validate JWT signature and expiration (<1μs with cached key) + // NOTE: Moved before revocation check for fail-fast on invalid tokens + let claims = self + .jwt_service + .validate_token(&bearer_token) + .map_err(|e| { + self.audit_logger + .log_auth_failure(&format!("invalid_jwt: {}", e), client_ip.as_deref()); + Status::unauthenticated("Invalid or expired token") + })?; + + // Layer 3: Check JWT revocation (<500ns with Redis) + let jti = Jti::from_string(claims.jti.clone()); + let is_revoked = self + .revocation_service + .is_revoked(&jti) + .await + .map_err(|e| { + error!("Revocation check failed: {}", e); + Status::unauthenticated("Authentication service unavailable") + })?; + + if is_revoked { + self.audit_logger + .log_auth_failure("token_revoked", client_ip.as_deref()); + return Err(Status::unauthenticated("Token has been revoked")); + } + + // Layer 5: RBAC permission check (<100ns with cached permissions) + // Cache permissions for this user if not already cached + self.authz_service + .cache_permissions(claims.sub.clone(), claims.permissions.clone()); + + // Basic permission check example (service-specific checks happen in handlers) + if !self.authz_service.has_permission(&claims.sub, "api.access") { + self.audit_logger + .log_auth_failure("insufficient_permissions", client_ip.as_deref()); + return Err(Status::permission_denied("Insufficient permissions")); + } + + // Layer 6: Rate limiting check (<50ns with atomic counters) + if !self.rate_limiter.check_rate_limit(&claims.sub) { + self.audit_logger + .log_auth_failure("rate_limit_exceeded", client_ip.as_deref()); + return Err(Status::resource_exhausted("Rate limit exceeded")); + } + + // Layer 7: Inject user context into request metadata + let user_context = UserContext { + user_id: claims.sub.clone(), + roles: claims.roles.clone(), + permissions: claims.permissions.clone(), + session_id: claims + .session_id + .clone() + .unwrap_or_else(|| Uuid::new_v4().to_string()), + authenticated_at: start, + }; + + // Add context to request extensions for downstream handlers + request.extensions_mut().insert(user_context); + + // CRITICAL: Add x-user-id to metadata for proxy forwarding + // The trading/backtesting/ML proxies expect this header to identify the user + request.metadata_mut().insert( + "x-user-id", + claims + .sub + .parse() + .map_err(|_| Status::internal("Invalid user_id encoding"))?, + ); + + // Layer 8: Async audit logging (non-blocking, 0ns overhead) + self.audit_logger + .log_auth_success(&claims.sub, client_ip.as_deref()); + + let elapsed = start.elapsed(); + debug!( + "Authentication successful for user {} in {:?}", + claims.sub, elapsed + ); + + // Warn if we exceed 10μs target + if elapsed > Duration::from_micros(10) { + warn!( + "Authentication latency exceeded target: {:?} > 10μs", + elapsed + ); + } + + Ok(request) + } + + /// Extract bearer token from Authorization header + fn extract_bearer_token(&self, request: &Request) -> Option { + request + .metadata() + .get("authorization") + .and_then(|auth| auth.to_str().ok()) + .and_then(|auth| { + if auth.starts_with("Bearer ") { + Some(auth[7..].to_string()) + } else { + None + } + }) + } + + /// Extract client IP from request metadata + fn extract_client_ip(&self, request: &Request) -> Option { + request + .metadata() + .get("x-forwarded-for") + .and_then(|ip| ip.to_str().ok()) + .map(|ip| ip.to_string()) + .or_else(|| { + request + .metadata() + .get("x-real-ip") + .and_then(|ip| ip.to_str().ok()) + .map(|ip| ip.to_string()) + }) + } +} + +/// Implement Tonic's Interceptor trait for gRPC integration +impl tonic::service::Interceptor for AuthInterceptor { + fn call(&mut self, request: Request<()>) -> Result, Status> { + // Tonic's Interceptor is synchronous, but we need async authentication + // Use block_in_place to run async code in a blocking context + tokio::task::block_in_place(|| { + tokio::runtime::Handle::current().block_on(self.authenticate(request)) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_jti_generation() { + let jti1 = Jti::new(); + let jti2 = Jti::new(); + + assert_ne!(jti1, jti2); + assert!(!jti1.as_str().is_empty()); + } + + #[test] + fn test_jwt_claims_defaults() { + let claims = JwtClaims { + jti: "test-jti".to_string(), + sub: "user123".to_string(), + iat: 1234567890, + exp: 1234571490, + nbf: Some(1234567890), + iss: "foxhunt".to_string(), + aud: "api-gateway".to_string(), + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some("session-123".to_string()), + }; + + assert_eq!(claims.token_type, "access"); + assert!(claims.session_id.is_some()); + } + + #[tokio::test] + async fn test_jwt_service_validation() { + use jsonwebtoken::{encode, EncodingKey, Header}; + + let secret = "test-secret-must-be-at-least-64-characters-long-for-security-validation-ok" + .to_string(); + let jwt_service = JwtService::new( + secret.clone(), + "test-issuer".to_string(), + "test-audience".to_string(), + ); + + let claims = JwtClaims { + jti: Jti::new().0, + sub: "user123".to_string(), + iat: SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(), + exp: SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() + + 3600, + nbf: Some( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(), + ), + iss: "test-issuer".to_string(), + aud: "test-audience".to_string(), + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(secret.as_bytes()), + ) + .unwrap(); + + let validated = jwt_service.validate_token(&token).unwrap(); + assert_eq!(validated.sub, "user123"); + } + + #[test] + fn test_authz_service_permissions() { + let authz = AuthzService::new(); + + authz.cache_permissions( + "user123".to_string(), + vec!["api.access".to_string(), "trading.submit".to_string()], + ); + + assert!(authz.has_permission("user123", "api.access")); + assert!(authz.has_permission("user123", "trading.submit")); + assert!(!authz.has_permission("user123", "admin.access")); + + authz.clear_cache("user123"); + assert!(!authz.has_permission("user123", "api.access")); + } + + #[test] + fn test_rate_limiter() { + let limiter = RateLimiter::new(10).expect("Valid rate limit"); // 10 requests per second + + // Should allow first request + assert!(limiter.check_rate_limit("user123")); + } + + #[tokio::test] + async fn test_revocation_cache_hit() { + // Create a mock Redis connection manager for testing + // In real tests, you would use a real Redis instance or mock + let cache = LocalRevocationCache::new(Duration::from_secs(60)); + + // Test cache statistics initialization + let stats = cache.stats(); + assert_eq!(stats.hits, 0); + assert_eq!(stats.misses, 0); + assert_eq!(stats.total, 0); + assert_eq!(stats.hit_rate, 0.0); + } + + #[test] + fn test_cache_ttl_expiration() { + let cache = LocalRevocationCache::new(Duration::from_millis(10)); + + // Insert an entry + cache.cache.insert( + "test_token".to_string(), + CachedRevocationResult { + is_revoked: false, + cached_at: Instant::now(), + }, + ); + + // Should be present immediately + assert!(cache.cache.contains_key("test_token")); + + // Wait for TTL to expire + std::thread::sleep(Duration::from_millis(15)); + + // Entry should still exist in DashMap but will be removed on next access + // The check_revoked method handles TTL expiration + assert_eq!(cache.cache.len(), 1); + } + + #[test] + fn test_cache_invalidation() { + let cache = LocalRevocationCache::new(Duration::from_secs(60)); + + // Insert an entry + cache.cache.insert( + "test_token".to_string(), + CachedRevocationResult { + is_revoked: false, + cached_at: Instant::now(), + }, + ); + + assert!(cache.cache.contains_key("test_token")); + + // Invalidate the entry + cache.invalidate("test_token"); + + assert!(!cache.cache.contains_key("test_token")); + } + + #[test] + fn test_cache_clear() { + let cache = LocalRevocationCache::new(Duration::from_secs(60)); + + // Insert multiple entries + for i in 0..10 { + cache.cache.insert( + format!("token_{}", i), + CachedRevocationResult { + is_revoked: false, + cached_at: Instant::now(), + }, + ); + } + + assert_eq!(cache.cache.len(), 10); + + // Clear all entries + cache.clear(); + + assert_eq!(cache.cache.len(), 0); + } + + #[test] + fn test_cache_stats_tracking() { + let cache = LocalRevocationCache::new(Duration::from_secs(60)); + + // Simulate cache hits + cache + .hits + .fetch_add(95, std::sync::atomic::Ordering::Relaxed); + cache + .misses + .fetch_add(5, std::sync::atomic::Ordering::Relaxed); + + let stats = cache.stats(); + assert_eq!(stats.hits, 95); + assert_eq!(stats.misses, 5); + assert_eq!(stats.total, 100); + assert_eq!(stats.hit_rate, 95.0); + + // Reset stats + cache.reset_stats(); + + let stats = cache.stats(); + assert_eq!(stats.hits, 0); + assert_eq!(stats.misses, 0); + } + + #[test] + fn test_cache_concurrent_access() { + use std::sync::Arc; + use std::thread; + + let cache = Arc::new(LocalRevocationCache::new(Duration::from_secs(60))); + + // Insert initial entry + cache.cache.insert( + "shared_token".to_string(), + CachedRevocationResult { + is_revoked: false, + cached_at: Instant::now(), + }, + ); + + let mut handles = vec![]; + + // Spawn 10 threads that all try to access the same cache entry + for _ in 0..10 { + let cache_clone = cache.clone(); + let handle = thread::spawn(move || { + for _ in 0..100 { + let entry = cache_clone.cache.get("shared_token"); + assert!(entry.is_some()); + } + }); + handles.push(handle); + } + + // Wait for all threads to complete + for handle in handles { + handle.join().expect("INVARIANT: Thread should complete successfully"); + } + + // Entry should still exist + assert!(cache.cache.contains_key("shared_token")); + } + + #[test] + fn test_cache_stats_struct() { + let stats = CacheStats { + hits: 950, + misses: 50, + total: 1000, + hit_rate: 95.0, + entries: 100, + }; + + assert_eq!(stats.hits, 950); + assert_eq!(stats.misses, 50); + assert_eq!(stats.total, 1000); + assert_eq!(stats.hit_rate, 95.0); + assert_eq!(stats.entries, 100); + } + + #[test] + fn test_cached_revocation_result() { + let result = CachedRevocationResult { + is_revoked: true, + cached_at: Instant::now(), + }; + + assert!(result.is_revoked); + assert!(result.cached_at.elapsed() < Duration::from_secs(1)); + } + + #[test] + fn test_cache_memory_efficiency() { + let cache = LocalRevocationCache::new(Duration::from_secs(60)); + + // Insert 1000 entries + for i in 0u32..1000 { + cache.cache.insert( + format!("token_{}", i), + CachedRevocationResult { + is_revoked: i.checked_rem(100).expect("Modulo overflow") == 0, // 1% revoked + cached_at: Instant::now(), + }, + ); + } + + assert_eq!(cache.cache.len(), 1000); + + // Verify mix of revoked and valid tokens + let mut revoked_count: u32 = 0; + for i in 0u32..1000 { + if let Some(entry) = cache.cache.get(&format!("token_{}", i)) { + if entry.is_revoked { + revoked_count = revoked_count + .checked_add(1) + .expect("Revoked count overflow"); + } + } + } + + assert_eq!(revoked_count, 10); // 1% of 1000 = 10 + } +} diff --git a/services/api/src/auth/jwt/endpoints.rs b/services/api/src/auth/jwt/endpoints.rs new file mode 100644 index 000000000..3dc3cfe54 --- /dev/null +++ b/services/api/src/auth/jwt/endpoints.rs @@ -0,0 +1,318 @@ +//! JWT Revocation Admin Endpoints +//! +//! Migrated from trading_service to api for centralized JWT management. +//! This module provides HTTP/gRPC endpoints for JWT token revocation management. +//! +//! ## Security +//! - All endpoints require admin permissions +//! - Audit logging for all revocation operations +//! - Rate limiting to prevent abuse + +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tonic::Status; +use tracing::{error, info}; + +use super::revocation::{Jti, JwtRevocationService, RevocationReason, RevocationStatistics}; + +/// Request to revoke the current user's token +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RevokeCurrentTokenRequest { + /// Optional reason for revocation + pub reason: Option, +} + +/// Request to revoke a specific token by JTI +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RevokeTokenRequest { + /// JWT ID to revoke + pub jti: String, + /// Reason for revocation + pub reason: String, + /// User ID who owned the token + pub user_id: String, +} + +/// Request to revoke all tokens for a user +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RevokeUserTokensRequest { + /// User ID whose tokens should be revoked + pub user_id: String, + /// Reason for revocation + pub reason: String, +} + +/// Response for token revocation operations +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RevocationResponse { + /// Whether the operation succeeded + pub success: bool, + /// Number of tokens revoked + pub tokens_revoked: usize, + /// Message describing the result + pub message: String, +} + +/// Authentication context for endpoints +#[derive(Debug, Clone)] +pub struct AuthContext { + /// User ID + pub user_id: String, + /// JWT claims if authenticated via JWT + pub jwt_claims: Option, + /// User permissions + pub permissions: Vec, + /// Client IP address + pub client_ip: Option, +} + +impl AuthContext { + /// Check if user has specific permission + pub fn has_permission(&self, permission: &str) -> bool { + self.permissions.contains(&permission.to_string()) + } +} + +/// Revocation endpoints handler +#[derive(Clone)] +pub struct RevocationEndpoints { + revocation_service: Arc, +} + +impl RevocationEndpoints { + /// Create new revocation endpoints handler + pub fn new(revocation_service: Arc) -> Self { + Self { revocation_service } + } + + /// Revoke the current user's token (user self-service) + pub async fn revoke_current_token( + &self, + auth_context: &AuthContext, + request: RevokeCurrentTokenRequest, + ) -> Result { + // Extract JTI from JWT claims + let jwt_claims = auth_context.jwt_claims.as_ref().ok_or_else(|| { + Status::invalid_argument("Token revocation only supported for JWT authentication") + })?; + + let jti = Jti::from_string(jwt_claims.jti.clone()); + + // Calculate remaining TTL + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|e| Status::internal(format!("System time error: {}", e)))? + .as_secs(); + + let ttl = jwt_claims.exp.saturating_sub(now); + + if ttl == 0 { + return Ok(RevocationResponse { + success: false, + tokens_revoked: 0, + message: "Token already expired".to_string(), + }); + } + + let reason = if let Some(user_reason) = request.reason { + RevocationReason::Other(user_reason) + } else { + RevocationReason::UserLogout + }; + + self.revocation_service + .revoke_token( + &jti, + &auth_context.user_id, + ttl, + reason, + &auth_context.user_id, + auth_context.client_ip.clone(), + ) + .await + .map_err(|e| Status::internal(format!("Failed to revoke token: {}", e)))?; + + info!( + "User {} revoked their own token (jti={})", + auth_context.user_id, jti + ); + + Ok(RevocationResponse { + success: true, + tokens_revoked: 1, + message: "Token revoked successfully".to_string(), + }) + } + + /// Revoke a specific token by `JTI` (admin only) + pub async fn revoke_token_by_jti( + &self, + auth_context: &AuthContext, + request: RevokeTokenRequest, + ) -> Result { + // Check admin permission + if !auth_context.has_permission("admin.revoke_tokens") { + return Err(Status::permission_denied( + "Admin permission required to revoke tokens", + )); + } + + let jti = Jti::from_string(request.jti.clone()); + + // Use standard 1 hour TTL for admin-revoked tokens + // This is reasonable since admin revocations are typically for active threats + // and tokens should expire within a reasonable timeframe + let ttl = 3600u64; // 1 hour (standard JWT access token TTL) + + let reason = match request.reason.as_str() { + "compromised" => RevocationReason::TokenCompromised, + "suspicious" => RevocationReason::SuspiciousActivity, + "admin" => RevocationReason::AdminRevocation, + other => RevocationReason::Other(other.to_string()), + }; + + self.revocation_service + .revoke_token( + &jti, + &request.user_id, + ttl, + reason, + &auth_context.user_id, + auth_context.client_ip.clone(), + ) + .await + .map_err(|e| Status::internal(format!("Failed to revoke token: {}", e)))?; + + info!( + "Admin {} revoked token {} for user {}", + auth_context.user_id, jti, request.user_id + ); + + Ok(RevocationResponse { + success: true, + tokens_revoked: 1, + message: format!("Token {} revoked successfully", jti), + }) + } + + /// Revoke all tokens for a specific user (admin only) + pub async fn revoke_all_user_tokens( + &self, + auth_context: &AuthContext, + request: RevokeUserTokensRequest, + ) -> Result { + // Check admin permission + if !auth_context.has_permission("admin.revoke_tokens") { + return Err(Status::permission_denied( + "Admin permission required to revoke user tokens", + )); + } + + let reason = match request.reason.as_str() { + "account_locked" => RevocationReason::AccountLocked, + "password_change" => RevocationReason::PasswordChange, + "compromised" => RevocationReason::TokenCompromised, + "suspicious" => RevocationReason::SuspiciousActivity, + "admin" => RevocationReason::AdminRevocation, + other => RevocationReason::Other(other.to_string()), + }; + + let revoked_count = self + .revocation_service + .revoke_all_user_tokens(&request.user_id, reason, &auth_context.user_id) + .await + .map_err(|e| Status::internal(format!("Failed to revoke user tokens: {}", e)))?; + + info!( + "Admin {} revoked {} tokens for user {}", + auth_context.user_id, revoked_count, request.user_id + ); + + Ok(RevocationResponse { + success: true, + tokens_revoked: revoked_count, + message: format!( + "Revoked {} tokens for user {}", + revoked_count, request.user_id + ), + }) + } + + /// Get revocation statistics (admin only) + pub async fn get_revocation_stats( + &self, + auth_context: &AuthContext, + ) -> Result { + // Check admin permission + if !auth_context.has_permission("admin.view_stats") { + return Err(Status::permission_denied( + "Admin permission required to view revocation statistics", + )); + } + + let stats = self + .revocation_service + .get_statistics() + .await + .map_err(|e| Status::internal(format!("Failed to get statistics: {}", e)))?; + + Ok(stats) + } + + /// Health check for revocation service + pub async fn health_check(&self) -> Result { + match self.revocation_service.get_statistics().await { + Ok(_) => Ok(true), + Err(e) => { + error!("Revocation service health check failed: {}", e); + Ok(false) + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::super::revocation::RevocationConfig; + use super::*; + + async fn create_test_revocation_service() -> Result> { + let redis_url = std::env::var("TEST_REDIS_URL") + .unwrap_or_else(|_| "redis://localhost:6379/15".to_string()); + let config = RevocationConfig::default(); + let service = JwtRevocationService::new(&redis_url, config).await?; + Ok(Arc::new(service)) + } + + #[tokio::test] + #[ignore = "requires Redis instance (TEST_REDIS_URL)"] + async fn test_revoke_user_tokens_requires_admin() { + let service = match create_test_revocation_service().await { + Ok(s) => s, + Err(e) => { + tracing::warn!("Redis unavailable, skipping test: {:?}", e); + return; // Skip test gracefully + }, + }; + let endpoints = RevocationEndpoints::new(service); + + let auth_context = AuthContext { + user_id: "test_user".to_string(), + jwt_claims: None, + permissions: vec![], // No admin permission + client_ip: None, + }; + + let request = RevokeUserTokensRequest { + user_id: "other_user".to_string(), + reason: "test".to_string(), + }; + + let result = endpoints + .revoke_all_user_tokens(&auth_context, request) + .await; + assert!(result.is_err()); + assert_eq!(result.unwrap_err().code(), tonic::Code::PermissionDenied); + } +} diff --git a/services/api/src/auth/jwt/mod.rs b/services/api/src/auth/jwt/mod.rs new file mode 100644 index 000000000..068ececa7 --- /dev/null +++ b/services/api/src/auth/jwt/mod.rs @@ -0,0 +1,29 @@ +//! JWT Authentication and Revocation Module +//! +//! Migrated from trading_service to api for centralized authentication. +//! +//! This module provides: +//! - JWT token validation and verification +//! - Token revocation via Redis blacklist +//! - Admin endpoints for token management +//! - Enhanced security validation +//! +//! ## Architecture +//! - `service.rs` - JWT validation service +//! - `revocation.rs` - Redis-backed revocation system +//! - `endpoints.rs` - gRPC/HTTP endpoints for revocation + +pub mod endpoints; +pub mod revocation; +pub mod service; + +// Re-export main types +pub use endpoints::{ + AuthContext, RevocationEndpoints, RevocationResponse, RevokeCurrentTokenRequest, + RevokeTokenRequest, RevokeUserTokensRequest, +}; +pub use revocation::{ + EnhancedJwtClaims, Jti, JwtRevocationService, RevocationConfig, RevocationMetadata, + RevocationReason, RevocationStatistics, TokenPair, +}; +pub use service::{JwtClaims, JwtConfig, JwtService}; diff --git a/services/api/src/auth/jwt/revocation.rs b/services/api/src/auth/jwt/revocation.rs new file mode 100644 index 000000000..2846da434 --- /dev/null +++ b/services/api/src/auth/jwt/revocation.rs @@ -0,0 +1,594 @@ +//! JWT Token Revocation System with Redis-backed Blacklist +//! +//! Migrated from trading_service to api for centralized JWT management. +//! This module implements comprehensive JWT session revocation to address the critical +//! vulnerability where compromised tokens remain valid until expiration (CVSS 8.8). +//! +//! ## Features +//! - Redis-backed token blacklist with JTI tracking +//! - Immediate revocation capability for compromised tokens +//! - Refresh token mechanism for session continuity +//! - Admin endpoints for forced revocation +//! - Integration with existing JWT validation flow + +use anyhow::{Context, Result}; +use redis::{aio::ConnectionManager, AsyncCommands}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; +use tracing::{debug, info, warn}; +use uuid::Uuid; + +/// JWT Token ID (JTI) for unique token identification +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct Jti(pub String); + +impl Jti { + /// Generate a new random JTI + pub fn new() -> Self { + Self(Uuid::new_v4().to_string()) + } + + /// Parse `JTI` from string + pub fn from_string(s: String) -> Self { + Self(s) + } + + /// Get `JTI` as string reference + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Get Redis key for this JTI + fn redis_key(&self) -> String { + format!("jwt:blacklist:{}", self.0) + } +} + +impl Default for Jti { + fn default() -> Self { + Self::new() + } +} + +impl std::fmt::Display for Jti { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +/// Enhanced JWT claims with `JTI` and refresh token support +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct EnhancedJwtClaims { + /// JWT ID for revocation tracking + pub jti: String, + /// Subject (user ID) + pub sub: String, + /// Issued at timestamp + pub iat: u64, + /// Expiration timestamp + pub exp: u64, + /// Not before timestamp + pub nbf: u64, + /// Issuer + pub iss: String, + /// Audience + pub aud: String, + /// User roles + pub roles: Vec, + /// Additional permissions + pub permissions: Vec, + /// Token type: "access" or "refresh" + pub token_type: String, + /// Session ID for tracking related tokens + pub session_id: String, +} + +impl EnhancedJwtClaims { + /// Create new access token claims + pub fn new_access_token( + user_id: String, + roles: Vec, + permissions: Vec, + issuer: String, + audience: String, + ttl_seconds: u64, + ) -> Result { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("System time error")? + .as_secs(); + + Ok(Self { + jti: Jti::new().to_string(), + sub: user_id, + iat: now, + exp: now + ttl_seconds, + nbf: now, + iss: issuer, + aud: audience, + roles, + permissions, + token_type: "access".to_string(), + session_id: Uuid::new_v4().to_string(), + }) + } + + /// Create new refresh token claims (longer TTL, limited permissions) + pub fn new_refresh_token( + user_id: String, + issuer: String, + audience: String, + session_id: String, + ttl_seconds: u64, + ) -> Result { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("System time error")? + .as_secs(); + + Ok(Self { + jti: Jti::new().to_string(), + sub: user_id, + iat: now, + exp: now + ttl_seconds, + nbf: now, + iss: issuer, + aud: audience, + roles: vec![], + permissions: vec!["refresh_token".to_string()], + token_type: "refresh".to_string(), + session_id, + }) + } + + /// Get remaining TTL in seconds + pub fn remaining_ttl(&self) -> Result { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("System time error")? + .as_secs(); + + if self.exp > now { + Ok(self.exp - now) + } else { + Ok(0) + } + } +} + +/// Token pair containing access and refresh tokens +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TokenPair { + /// Access token (short-lived, full permissions) + pub access_token: String, + /// Refresh token (long-lived, limited to refresh permission) + pub refresh_token: String, + /// Access token expiration timestamp + pub access_token_expires_at: u64, + /// Refresh token expiration timestamp + pub refresh_token_expires_at: u64, + /// Token type (always "Bearer") + pub token_type: String, +} + +/// Revocation reason for audit logging +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum RevocationReason { + /// User-initiated logout + UserLogout, + /// Admin-forced revocation + AdminRevocation, + /// Suspicious activity detected + SuspiciousActivity, + /// Password change + PasswordChange, + /// Account locked + AccountLocked, + /// Token compromised + TokenCompromised, + /// Session timeout + SessionTimeout, + /// Other reason + Other(String), +} + +impl std::fmt::Display for RevocationReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::UserLogout => write!(f, "user_logout"), + Self::AdminRevocation => write!(f, "admin_revocation"), + Self::SuspiciousActivity => write!(f, "suspicious_activity"), + Self::PasswordChange => write!(f, "password_change"), + Self::AccountLocked => write!(f, "account_locked"), + Self::TokenCompromised => write!(f, "token_compromised"), + Self::SessionTimeout => write!(f, "session_timeout"), + Self::Other(reason) => write!(f, "other:{}", reason), + } + } +} + +/// Revocation metadata stored in Redis +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct RevocationMetadata { + /// User ID who owned the token + user_id: String, + /// Reason for revocation + reason: String, + /// Who initiated the revocation + revoked_by: String, + /// Timestamp of revocation + revoked_at: u64, + /// Client IP if available + client_ip: Option, +} + +impl RevocationMetadata { + /// Public accessor for user_id (for audit logging) + pub fn user_id(&self) -> &str { + &self.user_id + } + + /// Public accessor for reason (for audit logging) + pub fn reason(&self) -> &str { + &self.reason + } + + /// Public accessor for revoked_by (for audit logging) + pub fn revoked_by(&self) -> &str { + &self.revoked_by + } +} + +/// Revocation service configuration +#[derive(Debug, Clone, PartialEq)] +pub struct RevocationConfig { + /// Redis key prefix for blacklist entries + pub redis_prefix: String, + /// Redis key prefix for user session tracking + pub session_prefix: String, + /// Enable audit logging + pub enable_audit_logging: bool, + /// Maximum tokens per user to track for bulk revocation + pub max_tokens_per_user: usize, +} + +impl Default for RevocationConfig { + fn default() -> Self { + Self { + redis_prefix: "jwt:blacklist:".to_string(), + session_prefix: "jwt:user_sessions:".to_string(), + enable_audit_logging: true, + max_tokens_per_user: 100, + } + } +} + +/// Redis-backed JWT revocation service +#[derive(Clone)] +pub struct JwtRevocationService { + /// Redis connection manager + redis: ConnectionManager, + /// Configuration + config: Arc, +} + +impl std::fmt::Debug for JwtRevocationService { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("JwtRevocationService") + .field("config", &self.config) + .finish() + } +} + +impl JwtRevocationService { + /// Create new revocation service with proper timeout handling + /// + /// CRITICAL FIX: The redis crate v0.27.6 does NOT support connection_timeout + /// or response_timeout as URL parameters. These are silently ignored, causing + /// 60+ second OS-level TCP timeouts when Redis is unavailable. + /// + /// Solution: Use tokio::time::timeout() to wrap the async ConnectionManager::new(). + pub async fn new(redis_url: &str, config: RevocationConfig) -> Result { + use tokio::time::{timeout, Duration}; + + let client = redis::Client::open(redis_url) + .context("Failed to create Redis client for JWT revocation")?; + + // CRITICAL: Wrap ConnectionManager::new() with tokio timeout (5s) + let redis = timeout( + Duration::from_secs(5), + ConnectionManager::new(client) + ) + .await + .context("Redis connection timed out after 5s")? + .context("Failed to connect to Redis for JWT revocation")?; + + info!( + "JWT revocation service connected to Redis with timeout (5s): {}", + redis_url + ); + + Ok(Self { + redis, + config: Arc::new(config), + }) + } + + /// Check if a token is revoked (blacklisted) + pub async fn is_revoked(&self, jti: &Jti) -> Result { + let key = jti.redis_key(); + let mut conn = self.redis.clone(); + + let exists: bool = conn + .exists(&key) + .await + .context("Failed to check token revocation status in Redis")?; + + if exists { + debug!("Token {} is blacklisted (revoked)", jti); + } + + Ok(exists) + } + + /// Revoke a single token by JTI + pub async fn revoke_token( + &self, + jti: &Jti, + user_id: &str, + ttl_seconds: u64, + reason: RevocationReason, + revoked_by: &str, + client_ip: Option, + ) -> Result<()> { + if ttl_seconds == 0 { + warn!("Token {} already expired, skipping revocation", jti); + return Ok(()); + } + + let key = jti.redis_key(); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("System time error")? + .as_secs(); + + let metadata = RevocationMetadata { + user_id: user_id.to_string(), + reason: reason.to_string(), + revoked_by: revoked_by.to_string(), + revoked_at: now, + client_ip, + }; + + let metadata_json = + serde_json::to_string(&metadata).context("Failed to serialize revocation metadata")?; + + let mut conn = self.redis.clone(); + + let _: () = conn + .set_ex(&key, metadata_json, ttl_seconds) + .await + .context("Failed to add token to Redis blacklist")?; + + if self.config.enable_audit_logging { + info!( + "Token revoked: jti={} user={} reason={} revoked_by={} ttl={}s", + jti, user_id, reason, revoked_by, ttl_seconds + ); + } + + self.track_user_token(user_id, jti).await?; + Ok(()) + } + + /// Track a token under a user's session list + async fn track_user_token(&self, user_id: &str, jti: &Jti) -> Result<()> { + let key = format!("{}{}", self.config.session_prefix, user_id); + let mut conn = self.redis.clone(); + + let _: () = conn + .sadd(&key, jti.as_str()) + .await + .context("Failed to add token to user session tracking")?; + + let set_size: usize = conn + .scard(&key) + .await + .context("Failed to get user session set size")?; + + if set_size > self.config.max_tokens_per_user { + warn!( + "User {} has {} tracked tokens, trimming oldest", + user_id, set_size + ); + } + + Ok(()) + } + + /// Revoke all tokens for a specific user + pub async fn revoke_all_user_tokens( + &self, + user_id: &str, + reason: RevocationReason, + revoked_by: &str, + ) -> Result { + let key = format!("{}{}", self.config.session_prefix, user_id); + let mut conn = self.redis.clone(); + + let jtis: Vec = conn + .smembers(&key) + .await + .context("Failed to get user's token list from Redis")?; + + if jtis.is_empty() { + info!("No tokens found for user {}, nothing to revoke", user_id); + return Ok(0); + } + + let mut revoked_count = 0; + + for jti_str in &jtis { + let jti = Jti::from_string(jti_str.clone()); + let blacklist_key = jti.redis_key(); + + let exists: bool = conn + .exists(&blacklist_key) + .await + .context("Failed to check if token is already revoked")?; + + if exists { + continue; + } + + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("System time error")? + .as_secs(); + + let metadata = RevocationMetadata { + user_id: user_id.to_string(), + reason: reason.to_string(), + revoked_by: revoked_by.to_string(), + revoked_at: now, + client_ip: None, + }; + + let metadata_json = serde_json::to_string(&metadata) + .context("Failed to serialize revocation metadata")?; + + // Use standard 1 hour TTL for bulk revocations + // This ensures tokens expire within a reasonable timeframe + // and prevents indefinite memory usage in Redis + let ttl = 3600u64; // 1 hour (standard JWT access token TTL) + + let _: () = conn + .set_ex(&blacklist_key, metadata_json, ttl) + .await + .context("Failed to add token to Redis blacklist")?; + + revoked_count += 1; + } + + let _: () = conn + .del(&key) + .await + .context("Failed to delete user token tracking set")?; + + if self.config.enable_audit_logging { + info!( + "Revoked {} tokens for user {} (reason: {}, revoked_by: {})", + revoked_count, user_id, reason, revoked_by + ); + } + + Ok(revoked_count) + } + + /// Get revocation metadata for a token + pub async fn get_revocation_metadata(&self, jti: &Jti) -> Result> { + let key = jti.redis_key(); + let mut conn = self.redis.clone(); + + let metadata_json: Option = conn + .get(&key) + .await + .context("Failed to get revocation metadata from Redis")?; + + match metadata_json { + Some(ref json) => { + let metadata: RevocationMetadata = serde_json::from_str(json) + .context("Failed to deserialize revocation metadata")?; + Ok(Some(metadata)) + }, + None => Ok(None), + } + } + + /// Get statistics about revoked tokens + pub async fn get_statistics(&self) -> Result { + let mut conn = self.redis.clone(); + + let pattern = format!("{}*", self.config.redis_prefix); + let blacklist_count: usize = self.count_keys(&mut conn, &pattern).await?; + + let session_pattern = format!("{}*", self.config.session_prefix); + let active_users: usize = self.count_keys(&mut conn, &session_pattern).await?; + + Ok(RevocationStatistics { + revoked_tokens: blacklist_count, + active_users, + }) + } + + /// Helper to count keys matching a pattern + /// Uses SCAN instead of KEYS to avoid blocking Redis + async fn count_keys(&self, conn: &mut ConnectionManager, pattern: &str) -> Result { + let mut total_count = 0; + let mut cursor: u64 = 0; + + // Use SCAN with cursor to iterate through keys without blocking Redis + loop { + let (new_cursor, keys): (u64, Vec) = redis::cmd("SCAN") + .arg(cursor) + .arg("MATCH") + .arg(pattern) + .arg("COUNT") + .arg(100) + .query_async(conn) + .await + .context("Failed to scan keys in Redis")?; + + total_count += keys.len(); + cursor = new_cursor; + + if cursor == 0 { + break; + } + } + + Ok(total_count) + } +} + +/// Statistics about revoked tokens +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RevocationStatistics { + /// Number of currently revoked tokens + pub revoked_tokens: usize, + /// Number of active users with tracked sessions + pub active_users: usize, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_jti_generation() { + let jti1 = Jti::new(); + let jti2 = Jti::new(); + + assert_ne!(jti1, jti2); + assert!(!jti1.as_str().is_empty()); + } + + #[test] + fn test_enhanced_jwt_claims_creation() { + let claims = EnhancedJwtClaims::new_access_token( + "user123".to_string(), + vec!["trader".to_string()], + vec!["trading.submit_order".to_string()], + "foxhunt".to_string(), + "trading-api".to_string(), + 3600, + ) + .unwrap(); + + assert_eq!(claims.sub, "user123"); + assert_eq!(claims.token_type, "access"); + assert!(!claims.jti.is_empty()); + assert!(claims.remaining_ttl().unwrap() > 3500); + } +} diff --git a/services/api/src/auth/jwt/service.rs b/services/api/src/auth/jwt/service.rs new file mode 100644 index 000000000..53b769d76 --- /dev/null +++ b/services/api/src/auth/jwt/service.rs @@ -0,0 +1,508 @@ +//! JWT Service - Token validation and verification +//! +//! Migrated from trading_service to api for centralized authentication. +//! This service handles: +//! - JWT token validation (signature, expiry, claims) +//! - Token revocation checking via Redis +//! - Enhanced security validation (entropy, length, patterns) + +use anyhow::{Context, Result}; +use jsonwebtoken::{decode, Algorithm, DecodingKey, Validation}; +use secrecy::ExposeSecret; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tracing::{error, info, warn}; + +use super::revocation::{Jti, JwtRevocationService}; + +/// JWT claims structure +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct JwtClaims { + /// JWT ID for revocation tracking (SECURITY: MANDATORY for revocation) + pub jti: String, + /// Subject (user ID) + pub sub: String, + /// Issued at timestamp + pub iat: u64, + /// Expiration timestamp + pub exp: u64, + /// Issuer + pub iss: String, + /// Audience + pub aud: String, + /// User roles + pub roles: Vec, + /// Additional permissions + pub permissions: Vec, + /// Token type: "access" or "refresh" (optional for backward compatibility) + #[serde(default = "default_token_type")] + pub token_type: String, + /// Session ID for tracking related tokens (optional for backward compatibility) + #[serde(default)] + pub session_id: Option, +} + +fn default_token_type() -> String { + "access".to_string() +} + +impl JwtClaims { + /// Convert to EnhancedJwtClaims for revocation tracking + pub fn to_enhanced(&self) -> super::revocation::EnhancedJwtClaims { + super::revocation::EnhancedJwtClaims { + jti: self.jti.clone(), + sub: self.sub.clone(), + iat: self.iat, + exp: self.exp, + nbf: self.iat, // Use iat as nbf if not present + iss: self.iss.clone(), + aud: self.aud.clone(), + roles: self.roles.clone(), + permissions: self.permissions.clone(), + token_type: self.token_type.clone(), + session_id: self + .session_id + .clone() + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()), + } + } +} + +/// JWT Service configuration +#[derive(Debug, Clone)] +pub struct JwtConfig { + /// JWT secret for token verification + pub jwt_secret: String, + /// JWT issuer + pub jwt_issuer: String, + /// JWT audience + pub jwt_audience: String, +} + +impl JwtConfig { + /// Create new JwtConfig with proper error handling + /// + /// Priority: + /// 1. Vault (production) - secret/foxhunt/jwt + /// 2. JWT_SECRET_FILE (file-based secret) + /// 3. JWT_SECRET env var (development fallback) + /// + /// Returns error if JWT secret is not configured or invalid + pub async fn new() -> Result { + // Try loading from Vault first (production) + if let Ok(vault_config) = Self::load_from_vault().await { + info!("✅ JWT configuration loaded from Vault"); + return Ok(vault_config); + } + + // Fallback to legacy file/env loading + warn!("⚠️ Vault unavailable - using legacy JWT_SECRET_FILE/JWT_SECRET (development only)"); + + let jwt_secret = Self::load_jwt_secret() + .map_err(|e| anyhow::anyhow!("Failed to load JWT secret: {}", e))?; + + Ok(Self { + jwt_secret, + jwt_issuer: "foxhunt-api-gateway".to_string(), + jwt_audience: "foxhunt-services".to_string(), + }) + } + + /// Load JWT configuration from HashiCorp Vault (production) + async fn load_from_vault() -> Result { + use config::JwtConfig as VaultJwtConfig; + + let vault_config = VaultJwtConfig::load().await?; + + Ok(Self { + jwt_secret: vault_config.secret().expose_secret().to_string(), + jwt_issuer: vault_config.issuer().to_string(), + jwt_audience: vault_config.audience().to_string(), + }) + } + + /// Securely load JWT secret from file or environment with enhanced validation + /// + /// Priority: 1) JWT_SECRET_FILE path, 2) JWT_SECRET env var + /// + /// SECURITY: Enforces minimum 64-character (512-bit) secrets with entropy validation + fn load_jwt_secret() -> Result { + // Try loading from secure file (recommended for production) + if let Ok(secret_file_path) = std::env::var("JWT_SECRET_FILE") { + match std::fs::read_to_string(&secret_file_path) { + Ok(secret) => { + let trimmed_secret = secret.trim().to_string(); + if let Err(e) = Self::validate_jwt_secret(&trimmed_secret) { + error!( + "JWT secret in file {} failed validation: {}", + secret_file_path, e + ); + } else { + tracing::info!("JWT secret loaded from secure file: {}", secret_file_path); + return Ok(trimmed_secret); + } + }, + Err(e) => { + error!("Failed to read JWT secret file {}: {}", secret_file_path, e); + }, + } + } + + // Fallback to environment variable (less secure, warn user) + if let Ok(secret) = std::env::var("JWT_SECRET") { + // WAVE 196: Relaxed validation for development secrets + // Production should use JWT_SECRET_FILE with proper validation + warn!( + "JWT secret loaded from environment variable - consider using JWT_SECRET_FILE for production" + ); + // WAVE 149 Agent 411: Trim whitespace to match file loading behavior (line 103) + return Ok(secret.trim().to_string()); + } + + Err(anyhow::anyhow!( + "JWT secret not found or invalid. Requirements:\n\ + - Minimum 64 characters (512-bit security)\n\ + - High entropy (mixed case, numbers, symbols)\n\ + - No dictionary words or patterns\n\ + Production setup: JWT_SECRET_FILE=/opt/foxhunt/secrets/jwt_secret\n\ + Generate with: openssl rand -base64 64" + )) + } + + /// Validate JWT secret strength and entropy + /// + /// SECURITY: Enforces enterprise-grade JWT secret requirements + fn validate_jwt_secret(secret: &str) -> Result<()> { + // Length validation - minimum 64 characters (512 bits) + if secret.len() < 64 { + return Err(anyhow::anyhow!( + "JWT secret too short: {} characters (minimum 64 required for 512-bit security)", + secret.len() + )); + } + + // Maximum length check (prevent DoS) + if secret.len() > 1024 { + return Err(anyhow::anyhow!( + "JWT secret too long: {} characters (maximum 1024 for performance)", + secret.len() + )); + } + + // Character set validation - require mixed case, numbers, and symbols + let has_lowercase = secret.chars().any(|c| c.is_ascii_lowercase()); + let has_uppercase = secret.chars().any(|c| c.is_ascii_uppercase()); + let has_digit = secret.chars().any(|c| c.is_ascii_digit()); + let has_symbol = secret.chars().any(|c| !c.is_alphanumeric()); + + if !has_lowercase { + return Err(anyhow::anyhow!("JWT secret must contain lowercase letters")); + } + if !has_uppercase { + return Err(anyhow::anyhow!("JWT secret must contain uppercase letters")); + } + if !has_digit { + return Err(anyhow::anyhow!("JWT secret must contain digits")); + } + if !has_symbol { + return Err(anyhow::anyhow!("JWT secret must contain symbols")); + } + + // Entropy estimation - check for repeated patterns + if Self::has_weak_patterns(secret) { + return Err(anyhow::anyhow!( + "JWT secret contains weak patterns (repeated sequences, dictionary words)" + )); + } + + // Basic entropy check - should have reasonable character distribution + let entropy_score = Self::calculate_entropy(secret); + if entropy_score < 4.0 { + return Err(anyhow::anyhow!( + "JWT secret has low entropy: {:.2} bits/char (minimum 4.0 required)", + entropy_score + )); + } + + Ok(()) + } + + /// Check for weak patterns in JWT secret + fn has_weak_patterns(secret: &str) -> bool { + // Check for repeated characters (more than 3 in a row) + let mut prev_char = '\0'; + let mut repeat_count = 1; + for c in secret.chars() { + if c == prev_char { + repeat_count += 1; + if repeat_count > 3 { + return true; + } + } else { + repeat_count = 1; + prev_char = c; + } + } + + // Check for simple sequential patterns + let bytes = secret.as_bytes(); + for window in bytes.windows(4) { + // windows(4) guarantees 4 elements, use pattern matching for safety + if let [a, b, c, d] = window { + let ascending = a + .checked_add(1) + .map(|a_plus_1| a_plus_1 == *b) + .unwrap_or(false) + && b.checked_add(1) + .map(|b_plus_1| b_plus_1 == *c) + .unwrap_or(false) + && c.checked_add(1) + .map(|c_plus_1| c_plus_1 == *d) + .unwrap_or(false); + + // For descending, wrapping_sub is actually correct here as we're checking sequences + let descending = + a.wrapping_sub(1) == *b && b.wrapping_sub(1) == *c && c.wrapping_sub(1) == *d; + if ascending || descending { + return true; + } + } + } + + // Check for common weak patterns + let weak_patterns = [ + "1234", "abcd", "password", "secret", "admin", "user", "test", "demo", "qwer", "asdf", + "0000", "1111", "aaaa", "bbbb", "cccc", "dddd", "eeee", "ffff", + ]; + + for pattern in &weak_patterns { + if secret.to_lowercase().contains(pattern) { + return true; + } + } + + false + } + + /// Calculate Shannon entropy of the secret + fn calculate_entropy(secret: &str) -> f64 { + use std::collections::HashMap; + + let mut char_counts = HashMap::new(); + let total_chars = secret.len() as f64; + + // Count character frequencies + for c in secret.chars() { + *char_counts.entry(c).or_insert(0) += 1; + } + + // Calculate Shannon entropy + let mut entropy = 0.0; + for count in char_counts.values() { + let probability = *count as f64 / total_chars; + // f64 arithmetic is checked at runtime for infinity/NaN + let entropy_contribution = probability * probability.log2(); + if entropy_contribution.is_finite() { + entropy -= entropy_contribution; + } + } + + entropy + } +} + +/// JWT token validator +pub struct JwtService { + config: Arc, + revocation_service: Option>, +} + +impl JwtService { + /// Create new JWT service + pub fn new(config: JwtConfig) -> Self { + Self { + config: Arc::new(config), + revocation_service: None, + } + } + + /// Set revocation service (call after initialization) + pub fn set_revocation_service(&mut self, service: Arc) { + self.revocation_service = Some(service); + } + + /// Validate JWT token + pub async fn validate_token(&self, token: &str) -> Result { + // SECURITY: Enhanced JWT validation with stronger checks + if token.is_empty() { + return Err(anyhow::anyhow!("JWT token is empty")); + } + + if token.len() > 8192 { + return Err(anyhow::anyhow!("JWT token too long - possible attack")); + } + + let key = DecodingKey::from_secret(self.config.jwt_secret.as_ref()); + let mut validation = Validation::new(Algorithm::HS256); + + // SECURITY: Strict validation settings with clock tolerance + validation.set_issuer(&[&self.config.jwt_issuer]); + validation.set_audience(&[&self.config.jwt_audience]); + validation.validate_exp = true; + validation.validate_nbf = false; // Disable NBF validation (optional claim) + validation.leeway = 10; // 10 second tolerance for clock skew + validation.validate_aud = true; + + let token_data = decode::(token, &key, &validation).map_err(|e| { + error!("JWT decode failed: {:?}", e); + error!( + "Token (first 50 chars): {}...", + &token[..50.min(token.len())] + ); + error!( + "Expected issuer: {}, audience: {}", + self.config.jwt_issuer, self.config.jwt_audience + ); + error!( + "Validation settings: exp={}, nbf={}, aud={}, leeway={}", + validation.validate_exp, + validation.validate_nbf, + validation.validate_aud, + validation.leeway + ); + anyhow::anyhow!("Invalid JWT token: {}", e) + })?; + + // SECURITY: Check token revocation BEFORE other validations + if let Some(revocation_service) = &self.revocation_service { + let jti = Jti::from_string(token_data.claims.jti.clone()); + + let is_revoked = revocation_service + .is_revoked(&jti) + .await + .context("Failed to check token revocation status")?; + + if is_revoked { + // Get revocation metadata for detailed error message + if let Ok(Some(metadata)) = revocation_service.get_revocation_metadata(&jti).await { + error!( + "Revoked token attempted: jti={} user={} reason={} revoked_by={}", + jti, + metadata.user_id(), + metadata.reason(), + metadata.revoked_by() + ); + } + return Err(anyhow::anyhow!("JWT token has been revoked")); + } + } + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|e| anyhow::anyhow!("System time error: {}", e))? + .as_secs(); + + // SECURITY: Additional expiration check with buffer + if token_data.claims.exp <= now { + return Err(anyhow::anyhow!("JWT token expired")); + } + + // SECURITY: Check token age (max 1 hour) + let token_age = now + .checked_sub(token_data.claims.iat) + .ok_or_else(|| anyhow::anyhow!("Invalid token timestamp (iat in future)"))?; + if token_age > 3600 { + return Err(anyhow::anyhow!("JWT token too old")); + } + + // SECURITY: Validate claims structure + if token_data.claims.sub.is_empty() { + return Err(anyhow::anyhow!("JWT subject claim is empty")); + } + + // SECURITY: Validate JTI is present (required for revocation) + if token_data.claims.jti.is_empty() { + return Err(anyhow::anyhow!( + "JWT must contain jti claim for revocation support" + )); + } + + if token_data.claims.roles.is_empty() { + return Err(anyhow::anyhow!("JWT must contain at least one role")); + } + + Ok(token_data.claims) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_jwt_config_new_with_valid_secret() { + // WAVE G23: Test isolation - save/restore env state to avoid race conditions + let original_jwt_secret = std::env::var("JWT_SECRET").ok(); + + // Set a high-entropy test JWT secret + std::env::set_var( + "JWT_SECRET", + "Kx7mP@9nR!2sW#5vY$8bC&3fG*6jH^1kL%4pQ+7tZ-0uN~9dM=5eV(8xS)2wT!6yA#4zB", + ); + + let config = JwtConfig::new() + .await + .expect("Should create config with valid JWT_SECRET"); + + assert_eq!(config.jwt_issuer, "foxhunt-api-gateway"); + assert_eq!(config.jwt_audience, "foxhunt-services"); + assert!(config.jwt_secret.len() >= 64); + + // WAVE G23: Restore original env state + match original_jwt_secret { + Some(val) => std::env::set_var("JWT_SECRET", val), + None => std::env::remove_var("JWT_SECRET"), + } + } + + #[tokio::test] + async fn test_jwt_config_new_priority_vault_over_env() { + // WAVE G23: Test isolation - save/restore env state to avoid race conditions + let original_jwt_secret = std::env::var("JWT_SECRET").ok(); + + // Test that Vault takes precedence over env vars (production behavior) + // Set env vars that would be used as fallback + std::env::set_var( + "JWT_SECRET", + "EnvVarSecret_Kx7mP@9nR!2sW#5vY$8bC&3fG*6jH^1kL%4pQ+7tZ-0uN~9dM=5eV(8xS)2wT!6yA#4zB", + ); + + // In dev environment, Vault is available, so it should succeed + // Either from Vault (production) or env var fallback (development) + let result = JwtConfig::new().await; + + match result { + Ok(config) => { + // Verify config was loaded successfully + assert_eq!(config.jwt_issuer, "foxhunt-api-gateway"); + assert_eq!(config.jwt_audience, "foxhunt-services"); + assert!(config.jwt_secret.len() >= 64); + + // If Vault is available, the secret will be from Vault (not our env var) + // If Vault is unavailable, it will use the env var as fallback + // Both cases are valid and test the priority system + } + Err(e) => { + // If both Vault AND env vars fail, report failure without panicking + eprintln!("JwtConfig::new() failed with both Vault and env var: {e}"); + assert!(false, "JwtConfig::new() should succeed with either Vault or env var available: {e}"); + } + } + + // WAVE G23: Restore original env state + match original_jwt_secret { + Some(val) => std::env::set_var("JWT_SECRET", val), + None => std::env::remove_var("JWT_SECRET"), + } + } +} diff --git a/services/api/src/auth/mfa/backup_codes.rs b/services/api/src/auth/mfa/backup_codes.rs new file mode 100644 index 000000000..194eded8b --- /dev/null +++ b/services/api/src/auth/mfa/backup_codes.rs @@ -0,0 +1,353 @@ +//! Backup Codes for MFA Recovery +//! +//! Provides one-time use backup codes for account recovery when TOTP is unavailable. +//! Implements secure generation, storage, and validation of backup codes. + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use rand::Rng; +use secrecy::{ExposeSecret, Secret}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use sqlx::PgPool; +use std::sync::Arc; +use tracing::{debug, info, warn}; +use uuid::Uuid; + +/// Backup code with display format +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BackupCode { + /// The actual code (should be kept secret) + #[serde(skip_serializing)] + pub code: Secret, + /// First 4 characters as hint for user reference + pub hint: String, + /// Display format (e.g., "ABCD-EFGH-IJKL") + pub display: String, +} + +impl BackupCode { + /// Create new backup code from random string + fn new(code: String) -> Self { + let hint = code.chars().take(4).collect(); + let display = format_backup_code(&code); + + Self { + code: Secret::new(code), + hint, + display, + } + } + + /// Get the code value (for verification purposes only) + pub fn expose(&self) -> &str { + self.code.expose_secret() + } +} + +/// Backup code generator +pub struct BackupCodeGenerator { + /// Code length (default: 16 characters) + code_length: usize, + /// Character set for codes (alphanumeric, excluding ambiguous characters) + charset: Vec, +} + +impl BackupCodeGenerator { + /// Create new backup code generator + pub fn new() -> Self { + // Exclude ambiguous characters: 0, O, 1, I, l + let charset: Vec = "23456789ABCDEFGHJKLMNPQRSTUVWXYZ".chars().collect(); + + Self { + code_length: 16, + charset, + } + } + + /// Generate specified number of backup codes + pub fn generate_codes(&self, count: usize) -> Result> { + if count == 0 || count > 20 { + return Err(anyhow::anyhow!("Code count must be between 1 and 20")); + } + + let mut codes = Vec::with_capacity(count); + let mut rng = rand::thread_rng(); + + for _ in 0..count { + let code: String = (0..self.code_length) + .map(|_| { + let idx = rng.gen_range(0..self.charset.len()); + self.charset[idx] + }) + .collect(); + + codes.push(BackupCode::new(code)); + } + + info!("Generated {} backup codes", count); + Ok(codes) + } + + /// Generate single backup code + pub fn generate_single_code(&self) -> Result { + let mut codes = self.generate_codes(1)?; + codes + .pop() + .ok_or_else(|| anyhow::anyhow!("Failed to generate backup code")) + } +} + +impl Default for BackupCodeGenerator { + fn default() -> Self { + Self::new() + } +} + +/// Backup code validator +pub struct BackupCodeValidator { + db_pool: Arc, +} + +impl BackupCodeValidator { + /// Create new backup code validator + pub fn new(db_pool: Arc) -> Self { + Self { db_pool } + } + + /// Validate backup code for a user + pub async fn validate( + &self, + user_id: Uuid, + backup_code: &str, + ip_address: Option<&str>, + ) -> Result { + // Normalize code (remove spaces, hyphens, convert to uppercase) + let normalized_code = normalize_backup_code(backup_code); + + // Validate format + if !is_valid_backup_code_format(&normalized_code) { + debug!("Invalid backup code format"); + return Ok(false); + } + + let ip: Option = ip_address.and_then(|s| s.parse().ok()); + + // Convert IpAddr to String for database binding + let ip_string: Option = ip.map(|addr| addr.to_string()); + + // Use database function for validation + let is_valid = sqlx::query_scalar::<_, bool>("SELECT validate_backup_code($1, $2, $3)") + .bind(user_id) + .bind(&normalized_code) + .bind(ip_string) + .fetch_one(&*self.db_pool) + .await + .context("Failed to validate backup code")?; + + if is_valid { + info!("Backup code successfully validated for user: {}", user_id); + } else { + warn!("Invalid backup code attempt for user: {}", user_id); + } + + Ok(is_valid) + } + + /// Get remaining backup codes count for a user + pub async fn get_remaining_count(&self, user_id: Uuid) -> Result { + let count = sqlx::query_scalar::<_, i32>( + r#" + SELECT COUNT(*)::int + FROM mfa_backup_codes + WHERE user_id = $1 AND is_used = false AND expires_at > NOW() + "#, + ) + .bind(user_id) + .fetch_one(&*self.db_pool) + .await + .context("Failed to get backup codes count")?; + + Ok(count) + } + + /// Check if user needs to regenerate backup codes + pub async fn needs_regeneration(&self, user_id: Uuid) -> Result { + let remaining = self.get_remaining_count(user_id).await?; + + // Warn if less than 3 codes remaining + Ok(remaining < 3) + } + + /// Get backup code usage history for a user + pub async fn get_usage_history(&self, user_id: Uuid) -> Result> { + use sqlx::Row; + + let results = sqlx::query( + r#" + SELECT + id, code_hint as hint, is_used, used_at, + used_from_ip::text as used_from_ip_str, expires_at, created_at + FROM mfa_backup_codes + WHERE user_id = $1 + ORDER BY created_at DESC + "#, + ) + .bind(user_id) + .fetch_all(&*self.db_pool) + .await + .context("Failed to fetch backup code usage history")?; + + let history = results + .into_iter() + .map(|r| BackupCodeUsage { + id: r.get("id"), + hint: r.get("hint"), + is_used: r.get("is_used"), + used_at: r.get("used_at"), + used_from_ip_str: r.get("used_from_ip_str"), + expires_at: r.get("expires_at"), + created_at: r.get("created_at"), + }) + .collect(); + + Ok(history) + } +} + +/// Backup code usage record +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BackupCodeUsage { + pub id: Uuid, + pub hint: String, + pub is_used: bool, + pub used_at: Option>, + pub used_from_ip_str: Option, + pub expires_at: DateTime, + pub created_at: DateTime, +} + +/// Format backup code for display (e.g., "ABCD-EFGH-IJKL-MNOP") +fn format_backup_code(code: &str) -> String { + code.chars() + .collect::>() + .chunks(4) + .map(|chunk| chunk.iter().collect::()) + .collect::>() + .join("-") +} + +/// Normalize backup code (remove formatting, uppercase) +fn normalize_backup_code(code: &str) -> String { + code.chars() + .filter(|c| c.is_alphanumeric()) + .map(|c| c.to_ascii_uppercase()) + .collect() +} + +/// Validate backup code format +fn is_valid_backup_code_format(code: &str) -> bool { + // Must be 16 characters + if code.len() != 16 { + return false; + } + + // Must be alphanumeric uppercase + code.chars() + .all(|c| c.is_ascii_uppercase() || c.is_ascii_digit()) +} + +/// Hash backup code for secure storage (SHA-256) +pub fn hash_backup_code(code: &str) -> String { + let mut hasher = Sha256::new(); + hasher.update(code.as_bytes()); + format!("{:x}", hasher.finalize()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_generate_backup_codes() { + let generator = BackupCodeGenerator::new(); + let codes = generator.generate_codes(10).unwrap(); + + assert_eq!(codes.len(), 10); + + for code in &codes { + // Check code length + assert_eq!(code.code.expose_secret().len(), 16); + + // Check hint + assert_eq!(code.hint.len(), 4); + + // Check display format (contains hyphens) + assert!(code.display.contains('-')); + } + } + + #[test] + fn test_generate_codes_invalid_count() { + let generator = BackupCodeGenerator::new(); + + assert!(generator.generate_codes(0).is_err()); + assert!(generator.generate_codes(21).is_err()); + } + + #[test] + fn test_format_backup_code() { + let code = "ABCDEFGHIJKLMNOP"; + let formatted = format_backup_code(code); + + assert_eq!(formatted, "ABCD-EFGH-IJKL-MNOP"); + } + + #[test] + fn test_normalize_backup_code() { + assert_eq!( + normalize_backup_code("ABCD-EFGH-IJKL-MNOP"), + "ABCDEFGHIJKLMNOP" + ); + assert_eq!( + normalize_backup_code("abcd efgh ijkl mnop"), + "ABCDEFGHIJKLMNOP" + ); + assert_eq!(normalize_backup_code(" A-B-C-D "), "ABCD"); + } + + #[test] + fn test_is_valid_backup_code_format() { + assert!(is_valid_backup_code_format("ABCDEFGH23456789")); + assert!(is_valid_backup_code_format("2345678923456789")); + + assert!(!is_valid_backup_code_format("ABCDEFGH2345678")); // Too short + assert!(!is_valid_backup_code_format("ABCDEFGH234567890")); // Too long + assert!(!is_valid_backup_code_format("abcdefgh23456789")); // Lowercase + assert!(!is_valid_backup_code_format("ABCDEFGH2345678!")); // Special char + } + + #[test] + fn test_hash_backup_code() { + let code = "ABCDEFGH23456789"; + let hash = hash_backup_code(code); + + // SHA-256 produces 64 hex characters + assert_eq!(hash.len(), 64); + + // Hash should be deterministic + assert_eq!(hash_backup_code(code), hash); + + // Different codes should have different hashes + assert_ne!(hash_backup_code("DIFFERENT23456789"), hash); + } + + #[test] + fn test_backup_code_new() { + let code = BackupCode::new("ABCDEFGH23456789".to_string()); + + assert_eq!(code.code.expose_secret(), "ABCDEFGH23456789"); + assert_eq!(code.hint, "ABCD"); + assert_eq!(code.display, "ABCD-EFGH-2345-6789"); + } +} diff --git a/services/api/src/auth/mfa/enrollment.rs b/services/api/src/auth/mfa/enrollment.rs new file mode 100644 index 000000000..7ed00d46b --- /dev/null +++ b/services/api/src/auth/mfa/enrollment.rs @@ -0,0 +1,214 @@ +//! MFA Enrollment Flow +//! +//! Handles the enrollment process for setting up multi-factor authentication. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use uuid::Uuid; + +/// Enrollment session for MFA setup +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EnrollmentSession { + /// Session ID + pub session_id: Uuid, + /// User ID + pub user_id: Uuid, + /// QR code URI (otpauth://) + pub qr_code_uri: String, + /// QR code as PNG image (base64 or bytes) + pub qr_code_png: Vec, + /// Manual entry key for authenticator apps + pub manual_entry_key: String, + /// Session expiration time + pub expires_at: DateTime, +} + +/// MFA enrollment state +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MfaEnrollment { + /// User ID + pub user_id: Uuid, + /// Enrollment status + pub status: EnrollmentStatus, + /// Current session (if active) + pub session: Option, + /// Number of verification attempts + pub verification_attempts: u32, + /// Maximum allowed attempts + pub max_attempts: u32, +} + +impl MfaEnrollment { + /// Create new enrollment for user + pub fn new(user_id: Uuid) -> Self { + Self { + user_id, + status: EnrollmentStatus::NotStarted, + session: None, + verification_attempts: 0, + max_attempts: 3, + } + } + + /// Start enrollment with session + pub fn start(&mut self, session: EnrollmentSession) { + self.status = EnrollmentStatus::InProgress; + self.session = Some(session); + self.verification_attempts = 0; + } + + /// Complete enrollment + pub fn complete(&mut self) { + self.status = EnrollmentStatus::Completed; + self.session = None; + } + + /// Mark enrollment as failed + pub fn fail(&mut self, reason: String) { + self.status = EnrollmentStatus::Failed(reason); + self.session = None; + } + + /// Increment verification attempts + pub fn increment_attempts(&mut self) { + self.verification_attempts += 1; + } + + /// Check if max attempts reached + pub fn is_max_attempts_reached(&self) -> bool { + self.verification_attempts >= self.max_attempts + } + + /// Check if session is expired + pub fn is_expired(&self) -> bool { + if let Some(session) = &self.session { + return session.expires_at < Utc::now(); + } + false + } +} + +/// Enrollment status +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum EnrollmentStatus { + /// Not yet started + NotStarted, + /// Enrollment in progress + InProgress, + /// Successfully completed + Completed, + /// Failed with reason + Failed(String), +} + +/// Enrollment errors +#[derive(Debug, Error)] +pub enum EnrollmentError { + #[error("Enrollment session not found")] + SessionNotFound, + + #[error("Enrollment session expired")] + SessionExpired, + + #[error("Maximum verification attempts exceeded")] + MaxAttemptsExceeded, + + #[error("Invalid verification code")] + InvalidCode, + + #[error("Enrollment already completed")] + AlreadyCompleted, + + #[error("User already has MFA enabled")] + AlreadyEnabled, + + #[error("Database error: {0}")] + DatabaseError(String), + + #[error("Encryption error: {0}")] + EncryptionError(String), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_enrollment_lifecycle() { + let user_id = Uuid::new_v4(); + let mut enrollment = MfaEnrollment::new(user_id); + + // Initial state + assert_eq!(enrollment.status, EnrollmentStatus::NotStarted); + assert!(enrollment.session.is_none()); + + // Start enrollment + let session = EnrollmentSession { + session_id: Uuid::new_v4(), + user_id, + qr_code_uri: "otpauth://totp/test".to_string(), + qr_code_png: vec![], + manual_entry_key: "TEST123".to_string(), + expires_at: Utc::now() + chrono::Duration::minutes(15), + }; + + enrollment.start(session); + assert_eq!(enrollment.status, EnrollmentStatus::InProgress); + assert!(enrollment.session.is_some()); + + // Complete enrollment + enrollment.complete(); + assert_eq!(enrollment.status, EnrollmentStatus::Completed); + assert!(enrollment.session.is_none()); + } + + #[test] + fn test_verification_attempts() { + let user_id = Uuid::new_v4(); + let mut enrollment = MfaEnrollment::new(user_id); + + assert_eq!(enrollment.verification_attempts, 0); + assert!(!enrollment.is_max_attempts_reached()); + + enrollment.increment_attempts(); + enrollment.increment_attempts(); + enrollment.increment_attempts(); + + assert_eq!(enrollment.verification_attempts, 3); + assert!(enrollment.is_max_attempts_reached()); + } + + #[test] + fn test_session_expiration() { + let user_id = Uuid::new_v4(); + let mut enrollment = MfaEnrollment::new(user_id); + + // Session in future (not expired) + let session = EnrollmentSession { + session_id: Uuid::new_v4(), + user_id, + qr_code_uri: "otpauth://totp/test".to_string(), + qr_code_png: vec![], + manual_entry_key: "TEST123".to_string(), + expires_at: Utc::now() + chrono::Duration::hours(1), + }; + + enrollment.start(session); + assert!(!enrollment.is_expired()); + + // Session in past (expired) + let expired_session = EnrollmentSession { + session_id: Uuid::new_v4(), + user_id, + qr_code_uri: "otpauth://totp/test".to_string(), + qr_code_png: vec![], + manual_entry_key: "TEST123".to_string(), + expires_at: Utc::now() - chrono::Duration::hours(1), + }; + + enrollment.start(expired_session); + assert!(enrollment.is_expired()); + } +} diff --git a/services/api/src/auth/mfa/mod.rs b/services/api/src/auth/mfa/mod.rs new file mode 100644 index 000000000..09156e92d --- /dev/null +++ b/services/api/src/auth/mfa/mod.rs @@ -0,0 +1,579 @@ +//! Multi-Factor Authentication (MFA) Module +//! +//! Implements TOTP-based multi-factor authentication for financial system security. +//! CRITICAL SECURITY: Addresses CVSS 9.1 vulnerability by enforcing MFA for all users. +//! +//! Features: +//! - TOTP (Time-Based One-Time Password) implementation (RFC 6238) +//! - QR code generation for authenticator app enrollment +//! - Backup codes for account recovery (10 codes, one-time use) +//! - Encrypted secret storage with secure key management +//! - Rate limiting and account lockout protection +//! - Comprehensive audit logging +//! +//! Security Standards: +//! - RFC 6238: TOTP Algorithm +//! - NIST SP 800-63B: Digital Identity Guidelines +//! - PCI DSS: Multi-factor authentication requirements +//! - SOX: Access control and authentication + +pub mod backup_codes; +pub mod enrollment; +pub mod qr_code; +pub mod totp; +pub mod verification; + +pub use backup_codes::{BackupCode, BackupCodeGenerator, BackupCodeValidator}; +pub use enrollment::{EnrollmentError, EnrollmentSession, MfaEnrollment}; +pub use qr_code::{QrCodeError, QrCodeGenerator}; +pub use totp::{TotpConfig, TotpGenerator, TotpVerifier}; +pub use verification::{MfaVerification, VerificationError, VerificationResult}; + +use anyhow::{Context, Result}; +use chrono::{DateTime, Utc}; +use secrecy::ExposeSecret; +use serde::{Deserialize, Serialize}; +use sqlx::PgPool; +use std::sync::Arc; +use tracing::{info, warn}; +use uuid::Uuid; + +// Re-export Secret types from secrecy crate +pub use secrecy::SecretString; + +/// MFA configuration for a user +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MfaConfig { + pub id: Uuid, + pub user_id: Uuid, + pub is_enabled: bool, + pub is_verified: bool, + pub enrolled_at: Option>, + pub verified_at: Option>, + pub last_used_at: Option>, + pub backup_codes_remaining: i32, + pub failed_verification_attempts: i32, + pub last_failed_attempt_at: Option>, + pub locked_until: Option>, +} + +/// MFA method types +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MfaMethod { + /// Time-based One-Time Password (TOTP) + Totp, + /// Backup recovery code + BackupCode, + /// Trusted device (future enhancement) + TrustedDevice, +} + +impl std::fmt::Display for MfaMethod { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + MfaMethod::Totp => write!(f, "totp"), + MfaMethod::BackupCode => write!(f, "backup_code"), + MfaMethod::TrustedDevice => write!(f, "trusted_device"), + } + } +} + +/// MFA manager - central coordinator for all MFA operations +#[derive(Clone)] +pub struct MfaManager { + db_pool: Arc, + totp_generator: Arc, + totp_verifier: Arc, + backup_code_generator: Arc, + backup_code_validator: Arc, + qr_code_generator: Arc, +} + +impl MfaManager { + /// Create new MFA manager + pub fn new(db_pool: PgPool, _encryption_key: String) -> Result { + let db_pool = Arc::new(db_pool); + + let totp_generator = Arc::new(TotpGenerator::new()); + let totp_verifier = Arc::new(TotpVerifier::new()); + let backup_code_generator = Arc::new(BackupCodeGenerator::new()); + let backup_code_validator = Arc::new(BackupCodeValidator::new(Arc::clone(&db_pool))); + let qr_code_generator = Arc::new(QrCodeGenerator::new()); + + Ok(Self { + db_pool, + totp_generator, + totp_verifier, + backup_code_generator, + backup_code_validator, + qr_code_generator, + }) + } + + /// Check if MFA is required for a user + pub async fn is_mfa_required(&self, user_id: Uuid) -> Result { + let result = sqlx::query_scalar::<_, bool>("SELECT is_mfa_required($1)") + .bind(user_id) + .fetch_one(&*self.db_pool) + .await + .context("Failed to check if MFA is required")?; + + Ok(result) + } + + /// Get MFA configuration for a user + pub async fn get_mfa_config(&self, user_id: Uuid) -> Result> { + use sqlx::Row; + + let result = sqlx::query( + r#" + SELECT + id, user_id, is_enabled, is_verified, + enrolled_at, verified_at, last_used_at, + backup_codes_remaining, failed_verification_attempts, + last_failed_attempt_at, locked_until + FROM mfa_config + WHERE user_id = $1 + "#, + ) + .bind(user_id) + .fetch_optional(&*self.db_pool) + .await + .context("Failed to fetch MFA config")?; + + Ok(result.map(|r| MfaConfig { + id: r.get("id"), + user_id: r.get("user_id"), + is_enabled: r.get("is_enabled"), + is_verified: r.get("is_verified"), + enrolled_at: r.get("enrolled_at"), + verified_at: r.get("verified_at"), + last_used_at: r.get("last_used_at"), + backup_codes_remaining: r.get("backup_codes_remaining"), + failed_verification_attempts: r.get("failed_verification_attempts"), + last_failed_attempt_at: r.get("last_failed_attempt_at"), + locked_until: r.get("locked_until"), + })) + } + + /// Check if user's MFA is locked due to failed attempts + pub async fn is_mfa_locked(&self, user_id: Uuid) -> Result { + if let Some(config) = self.get_mfa_config(user_id).await? { + if let Some(locked_until) = config.locked_until { + return Ok(locked_until > Utc::now()); + } + } + Ok(false) + } + + /// Start MFA enrollment for a user + pub async fn start_enrollment( + &self, + user_id: Uuid, + issuer: &str, + account_name: &str, + ) -> Result { + info!("Starting MFA enrollment for user: {}", user_id); + + // Generate TOTP secret + let secret = self.totp_generator.generate_secret()?; + + // Create QR code data + let qr_uri = self + .totp_generator + .generate_qr_uri(&secret, issuer, account_name)?; + + // Encrypt the secret for database storage using pgcrypto + let encrypted_secret = self.encrypt_totp_secret(secret.expose_secret()).await?; + + // Generate QR code image + let qr_code_png = self.qr_code_generator.generate_png(&qr_uri)?; + + // Create enrollment session + let session_id = Uuid::new_v4(); + let expires_at = Utc::now() + chrono::Duration::minutes(15); + + sqlx::query( + r#" + INSERT INTO mfa_enrollment_sessions ( + id, user_id, temp_totp_secret_encrypted, qr_code_data, + is_active, expires_at + ) VALUES ($1, $2, $3, $4, true, $5) + "#, + ) + .bind(session_id) + .bind(user_id) + .bind(encrypted_secret) + .bind(&qr_uri) + .bind(expires_at) + .execute(&*self.db_pool) + .await + .context("Failed to create enrollment session")?; + + info!("MFA enrollment session created: {}", session_id); + + Ok(EnrollmentSession { + session_id, + user_id, + qr_code_uri: qr_uri, + qr_code_png, + manual_entry_key: secret.expose_secret().to_string(), + expires_at, + }) + } + + /// Complete MFA enrollment by verifying first TOTP code + pub async fn complete_enrollment( + &self, + session_id: Uuid, + user_id: Uuid, + totp_code: &str, + ) -> Result> { + info!("Completing MFA enrollment for user: {}", user_id); + + // Get enrollment session + use sqlx::Row; + + let session = sqlx::query( + r#" + SELECT + temp_totp_secret_encrypted, + is_active, + expires_at, + verification_attempts + FROM mfa_enrollment_sessions + WHERE id = $1 AND user_id = $2 + "#, + ) + .bind(session_id) + .bind(user_id) + .fetch_optional(&*self.db_pool) + .await + .context("Failed to fetch enrollment session")? + .ok_or_else(|| anyhow::anyhow!("Enrollment session not found"))?; + + // Extract session fields + let temp_totp_secret_encrypted: Vec = session.get("temp_totp_secret_encrypted"); + let is_active: bool = session.get("is_active"); + let expires_at: DateTime = session.get("expires_at"); + let verification_attempts: i32 = session.get("verification_attempts"); + + // Validate session + if !is_active { + return Err(anyhow::anyhow!("Enrollment session is not active")); + } + if expires_at < Utc::now() { + return Err(anyhow::anyhow!("Enrollment session has expired")); + } + if verification_attempts >= 3 { + return Err(anyhow::anyhow!("Maximum verification attempts exceeded")); + } + + // Decrypt secret and verify TOTP code using pgcrypto + let secret = self + .decrypt_totp_secret(&temp_totp_secret_encrypted) + .await?; + let is_valid = self.totp_verifier.verify(&secret, totp_code, 1)?; + + if !is_valid { + // Increment verification attempts + sqlx::query( + "UPDATE mfa_enrollment_sessions SET verification_attempts = verification_attempts + 1 WHERE id = $1" + ) + .bind(session_id) + .execute(&*self.db_pool) + .await?; + + return Err(anyhow::anyhow!("Invalid TOTP code")); + } + + // Generate backup codes + let backup_codes = self.backup_code_generator.generate_codes(10)?; + + // Create MFA config + let config_id = Uuid::new_v4(); + let now = Utc::now(); + + sqlx::query( + r#" + INSERT INTO mfa_config ( + id, user_id, totp_secret_encrypted, totp_algorithm, totp_digits, totp_period, + is_enabled, is_verified, enrolled_at, verified_at, backup_codes_remaining + ) VALUES ($1, $2, $3, 'SHA1', 6, 30, true, true, $4, $4, 10) + ON CONFLICT (user_id) DO UPDATE SET + totp_secret_encrypted = EXCLUDED.totp_secret_encrypted, + is_enabled = true, + is_verified = true, + verified_at = $4, + backup_codes_remaining = 10 + "#, + ) + .bind(config_id) + .bind(user_id) + .bind(&temp_totp_secret_encrypted) + .bind(now) + .execute(&*self.db_pool) + .await + .context("Failed to create MFA config")?; + + // Store backup codes + self.store_backup_codes(user_id, &backup_codes).await?; + + // Mark enrollment session as completed + sqlx::query( + "UPDATE mfa_enrollment_sessions SET is_active = false, completed_at = NOW() WHERE id = $1" + ) + .bind(session_id) + .execute(&*self.db_pool) + .await?; + + info!( + "MFA enrollment completed successfully for user: {}", + user_id + ); + + Ok(backup_codes) + } + + /// Verify TOTP code for authentication + pub async fn verify_totp( + &self, + user_id: Uuid, + totp_code: &str, + ip_address: Option, + ) -> Result { + // Check if account is locked + if self.is_mfa_locked(user_id).await? { + warn!("MFA verification attempted for locked account: {}", user_id); + return Err(anyhow::anyhow!( + "Account is locked due to too many failed attempts" + )); + } + + // Get MFA config + let config = self + .get_mfa_config(user_id) + .await? + .ok_or_else(|| anyhow::anyhow!("MFA not configured for user"))?; + + if !config.is_enabled { + return Err(anyhow::anyhow!("MFA is not enabled for user")); + } + + // Get encrypted secret + let encrypted_secret = sqlx::query_scalar::<_, Vec>( + "SELECT totp_secret_encrypted FROM mfa_config WHERE user_id = $1", + ) + .bind(user_id) + .fetch_one(&*self.db_pool) + .await + .context("Failed to fetch TOTP secret")?; + + // Decrypt and verify using pgcrypto + let secret = self.decrypt_totp_secret(&encrypted_secret).await?; + let is_valid = self.totp_verifier.verify(&secret, totp_code, 1)?; + + // Record attempt + self.record_verification_attempt( + user_id, + MfaMethod::Totp, + is_valid, + ip_address, + None, + if is_valid { + None + } else { + Some("INVALID_TOTP_CODE".to_string()) + }, + ) + .await?; + + Ok(is_valid) + } + + /// Verify backup code for authentication + pub async fn verify_backup_code( + &self, + user_id: Uuid, + backup_code: &str, + ip_address: Option, + ) -> Result { + let is_valid = self + .backup_code_validator + .validate(user_id, backup_code, ip_address.as_deref()) + .await?; + + Ok(is_valid) + } + + /// Record MFA verification attempt + async fn record_verification_attempt( + &self, + user_id: Uuid, + method: MfaMethod, + success: bool, + ip_address: Option, + user_agent: Option, + error_code: Option, + ) -> Result { + let ip: Option = ip_address.as_ref().and_then(|s| s.parse().ok()); + + let log_id = sqlx::query_scalar::<_, Uuid>( + r#" + SELECT record_mfa_attempt($1, $2, $3, $4, $5, NULL, $6) + "#, + ) + .bind(user_id) + .bind(method.to_string()) + .bind(success) + .bind(ip.map(|addr| addr.to_string())) + .bind(user_agent) + .bind(error_code) + .fetch_one(&*self.db_pool) + .await + .context("Failed to record MFA attempt")?; + + Ok(log_id) + } + + /// Store backup codes in database + async fn store_backup_codes(&self, user_id: Uuid, codes: &[BackupCode]) -> Result<()> { + let expires_at = Utc::now() + chrono::Duration::days(365); + + for code in codes { + let code_id = Uuid::new_v4(); + let code_hash = self.hash_backup_code(code.code.expose_secret()); + + sqlx::query( + r#" + INSERT INTO mfa_backup_codes ( + id, user_id, code_hash, code_hint, expires_at + ) VALUES ($1, $2, $3, $4, $5) + "#, + ) + .bind(code_id) + .bind(user_id) + .bind(&code_hash) + .bind(&code.hint) + .bind(expires_at) + .execute(&*self.db_pool) + .await + .context("Failed to store backup code")?; + } + + Ok(()) + } + + /// Encrypt TOTP secret for storage using Postgre`SQL` pgcrypto + /// + /// Wave 121 Agent 1: Implemented proper encryption using pgcrypto AES-256-CBC + /// + /// Resolves CRITICAL security blocker (plaintext TOTP secrets) + async fn encrypt_totp_secret(&self, secret: &str) -> Result> { + let encrypted: Vec = sqlx::query_scalar("SELECT encrypt_mfa_secret($1)") + .bind(secret) + .fetch_one(&*self.db_pool) + .await + .context("Failed to encrypt TOTP secret using pgcrypto")?; + + Ok(encrypted) + } + + /// Decrypt TOTP secret from storage using Postgre`SQL` pgcrypto + /// + /// Wave 121 Agent 1: Implemented proper decryption using pgcrypto AES-256-CBC + /// + /// Resolves CRITICAL security blocker (plaintext TOTP secrets) + async fn decrypt_totp_secret(&self, encrypted: &[u8]) -> Result { + let decrypted: String = sqlx::query_scalar("SELECT decrypt_mfa_secret($1)") + .bind(encrypted) + .fetch_one(&*self.db_pool) + .await + .context("Failed to decrypt TOTP secret using pgcrypto")?; + + Ok(decrypted) + } + + /// Hash backup code for secure storage + fn hash_backup_code(&self, code: &str) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(code.as_bytes()); + format!("{:x}", hasher.finalize()) + } + + /// Disable MFA for a user (admin only) + pub async fn disable_mfa(&self, user_id: Uuid) -> Result<()> { + warn!( + "Disabling MFA for user: {} - This should only be done by administrators", + user_id + ); + + sqlx::query( + "UPDATE mfa_config SET is_enabled = false, updated_at = NOW() WHERE user_id = $1", + ) + .bind(user_id) + .execute(&*self.db_pool) + .await + .context("Failed to disable MFA")?; + + // Invalidate all backup codes + sqlx::query( + "UPDATE mfa_backup_codes SET is_used = true WHERE user_id = $1 AND is_used = false", + ) + .bind(user_id) + .execute(&*self.db_pool) + .await?; + + Ok(()) + } + + /// Get backup codes status for a user + pub async fn get_backup_codes_status(&self, user_id: Uuid) -> Result { + use sqlx::Row; + + let result = sqlx::query( + r#" + SELECT + COUNT(*) FILTER (WHERE is_used = false) as remaining, + COUNT(*) FILTER (WHERE is_used = true) as used, + MIN(expires_at) FILTER (WHERE is_used = false) as earliest_expiry + FROM mfa_backup_codes + WHERE user_id = $1 + "#, + ) + .bind(user_id) + .fetch_one(&*self.db_pool) + .await + .context("Failed to fetch backup codes status")?; + + Ok(BackupCodesStatus { + remaining: u32::try_from(result.get::("remaining")) + .map_err(|_| anyhow::anyhow!("Remaining backup codes count exceeds u32 range"))?, + used: u32::try_from(result.get::("used")) + .map_err(|_| anyhow::anyhow!("Used backup codes count exceeds u32 range"))?, + earliest_expiry: result.get("earliest_expiry"), + }) + } +} + +/// Backup codes status +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BackupCodesStatus { + pub remaining: u32, + pub used: u32, + pub earliest_expiry: Option>, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_mfa_method_display() { + assert_eq!(MfaMethod::Totp.to_string(), "totp"); + assert_eq!(MfaMethod::BackupCode.to_string(), "backup_code"); + assert_eq!(MfaMethod::TrustedDevice.to_string(), "trusted_device"); + } +} diff --git a/services/api/src/auth/mfa/qr_code.rs b/services/api/src/auth/mfa/qr_code.rs new file mode 100644 index 000000000..20b3d8038 --- /dev/null +++ b/services/api/src/auth/mfa/qr_code.rs @@ -0,0 +1,185 @@ +//! QR Code Generator for TOTP Enrollment +//! +//! Generates QR codes for TOTP secret enrollment in authenticator apps. + +use anyhow::Result; +use qrcode::{render::svg, QrCode}; +use thiserror::Error; + +/// QR code generation errors +#[derive(Debug, Error)] +pub enum QrCodeError { + #[error("QR code generation failed: {0}")] + GenerationFailed(String), + + #[error("Invalid URI format: {0}")] + InvalidUri(String), + + #[error("Rendering failed: {0}")] + RenderingFailed(String), +} + +/// QR code generator for TOTP enrollment +pub struct QrCodeGenerator { + /// Image size in pixels + image_size: u32, + /// Error correction level + error_correction: qrcode::EcLevel, +} + +impl QrCodeGenerator { + /// Create new QR code generator with default settings + pub fn new() -> Self { + Self { + image_size: 256, + error_correction: qrcode::EcLevel::M, // Medium error correction + } + } + + /// Create QR code generator with custom size + pub fn with_size(size: u32) -> Self { + Self { + image_size: size, + error_correction: qrcode::EcLevel::M, + } + } + + /// Generate PNG image from otpauth:// URI + pub fn generate_png(&self, uri: &str) -> Result> { + // Validate URI format + if !uri.starts_with("otpauth://") { + return Err( + QrCodeError::InvalidUri("URI must start with otpauth://".to_string()).into(), + ); + } + + // Create QR code + let code = QrCode::with_error_correction_level(uri, self.error_correction) + .map_err(|e| QrCodeError::GenerationFailed(e.to_string()))?; + + // Render to PNG + let image = code + .render::>() + .min_dimensions(self.image_size, self.image_size) + .build(); + + // Convert to PNG bytes + let mut png_bytes = Vec::new(); + image + .write_to( + &mut std::io::Cursor::new(&mut png_bytes), + image::ImageFormat::Png, + ) + .map_err(|e| QrCodeError::RenderingFailed(e.to_string()))?; + + Ok(png_bytes) + } + + /// Generate SVG image from otpauth:// URI + pub fn generate_svg(&self, uri: &str) -> Result { + // Validate URI format + if !uri.starts_with("otpauth://") { + return Err( + QrCodeError::InvalidUri("URI must start with otpauth://".to_string()).into(), + ); + } + + // Create QR code + let code = QrCode::with_error_correction_level(uri, self.error_correction) + .map_err(|e| QrCodeError::GenerationFailed(e.to_string()))?; + + // Render to SVG + let svg = code + .render() + .min_dimensions(self.image_size, self.image_size) + .dark_color(svg::Color("#000000")) + .light_color(svg::Color("#ffffff")) + .build(); + + Ok(svg) + } + + /// Generate data URL for inline display (base64 encoded PNG) + pub fn generate_data_url(&self, uri: &str) -> Result { + let png_bytes = self.generate_png(uri)?; + let base64_encoded = + base64::Engine::encode(&base64::engine::general_purpose::STANDARD, &png_bytes); + + Ok(format!("data:image/png;base64,{}", base64_encoded)) + } + + /// Set image size + pub fn set_size(&mut self, size: u32) { + self.image_size = size; + } + + /// Set error correction level + pub fn set_error_correction(&mut self, level: qrcode::EcLevel) { + self.error_correction = level; + } +} + +impl Default for QrCodeGenerator { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_generate_png() { + let generator = QrCodeGenerator::new(); + let uri = + "otpauth://totp/FoxhuntHFT:user@example.com?secret=JBSWY3DPEHPK3PXP&issuer=FoxhuntHFT"; + + let png_bytes = generator.generate_png(uri).unwrap(); + + // PNG should have magic bytes + assert_eq!(&png_bytes[0..8], b"\x89PNG\r\n\x1a\n"); + + // Should be a valid size + assert!(png_bytes.len() > 100); + } + + #[test] + fn test_generate_svg() { + let generator = QrCodeGenerator::new(); + let uri = + "otpauth://totp/FoxhuntHFT:user@example.com?secret=JBSWY3DPEHPK3PXP&issuer=FoxhuntHFT"; + + let svg = generator.generate_svg(uri).unwrap(); + + // SVG should contain proper XML + assert!(svg.contains("")); + } + + #[test] + fn test_generate_data_url() { + let generator = QrCodeGenerator::new(); + let uri = + "otpauth://totp/FoxhuntHFT:user@example.com?secret=JBSWY3DPEHPK3PXP&issuer=FoxhuntHFT"; + + let data_url = generator.generate_data_url(uri).unwrap(); + + assert!(data_url.starts_with("data:image/png;base64,")); + } + + #[test] + fn test_invalid_uri() { + let generator = QrCodeGenerator::new(); + let invalid_uri = "https://example.com/invalid"; + + assert!(generator.generate_png(invalid_uri).is_err()); + assert!(generator.generate_svg(invalid_uri).is_err()); + } + + #[test] + fn test_custom_size() { + let generator = QrCodeGenerator::with_size(512); + assert_eq!(generator.image_size, 512); + } +} diff --git a/services/api/src/auth/mfa/totp.rs b/services/api/src/auth/mfa/totp.rs new file mode 100644 index 000000000..64169df75 --- /dev/null +++ b/services/api/src/auth/mfa/totp.rs @@ -0,0 +1,431 @@ +//! TOTP (Time-Based One-Time Password) Implementation +//! +//! Implements RFC 6238 TOTP algorithm for multi-factor authentication. +//! Uses HMAC-based One-Time Password (HOTP) as defined in RFC 4226. + +use anyhow::Result; +use base32::Alphabet; +use chrono::Utc; +use hmac::{Hmac, Mac}; +use rand::Rng; +use serde::{Deserialize, Serialize}; +use sha1::Sha1; +use tracing::warn; + +// Use secrecy::Secret from workspace dependencies +use secrecy::{ExposeSecret, SecretString}; + +type HmacSha1 = Hmac; + +/// TOTP configuration parameters (RFC 6238) +#[derive(Debug, Clone, Serialize)] +pub struct TotpConfig { + /// Secret key (Base32 encoded) + #[serde(skip)] + pub secret: SecretString, + /// Number of digits in TOTP code (6 or 8) + pub digits: u32, + /// Time step in seconds (usually 30) + pub period: u64, + /// Algorithm (SHA1, SHA256, SHA512) + pub algorithm: TotpAlgorithm, +} + +impl Default for TotpConfig { + fn default() -> Self { + Self { + secret: SecretString::new(String::new()), + digits: 6_u32, + period: 30_u64, + algorithm: TotpAlgorithm::SHA1, + } + } +} + +/// TOTP algorithm types +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum TotpAlgorithm { + SHA1, + SHA256, + SHA512, +} + +impl std::fmt::Display for TotpAlgorithm { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TotpAlgorithm::SHA1 => write!(f, "SHA1"), + TotpAlgorithm::SHA256 => write!(f, "SHA256"), + TotpAlgorithm::SHA512 => write!(f, "SHA512"), + } + } +} + +/// TOTP generator for creating secrets and QR codes +pub struct TotpGenerator { + config: TotpConfig, +} + +impl TotpGenerator { + /// Create new TOTP generator with default configuration + pub fn new() -> Self { + Self { + config: TotpConfig::default(), + } + } + + /// Generate a random TOTP secret (Base32 encoded, 160 bits) + pub fn generate_secret(&self) -> Result { + let mut rng = rand::thread_rng(); + let mut secret_bytes = [0u8; 20_usize]; // 160 bits for SHA1 + rng.fill(&mut secret_bytes); + + // Encode to Base32 (RFC 4648) + let secret_base32 = base32::encode(Alphabet::Rfc4648 { padding: false }, &secret_bytes); + + Ok(SecretString::new(secret_base32)) + } + + /// Generate QR code URI for authenticator apps (otpauth:// format) + pub fn generate_qr_uri( + &self, + secret: &SecretString, + issuer: &str, + account_name: &str, + ) -> Result { + // Format: otpauth://totp/ISSUER:ACCOUNT?secret=SECRET&issuer=ISSUER&algorithm=SHA1&digits=6&period=30 + let uri = format!( + "otpauth://totp/{}:{}?secret={}&issuer={}&algorithm={}&digits={}&period={}", + urlencoding::encode(issuer), + urlencoding::encode(account_name), + secret.expose_secret(), + urlencoding::encode(issuer), + self.config.algorithm, + self.config.digits, + self.config.period + ); + + Ok(uri) + } + + /// Generate TOTP code for current time (for testing purposes only) + pub fn generate_code(&self, secret: &str) -> Result { + let time = u64::try_from(Utc::now().timestamp()) + .map_err(|_| anyhow::anyhow!("Negative timestamp cannot be converted to u64"))?; + + self.generate_code_at_time(secret, time) + } + + /// Generate TOTP code for specific timestamp + pub fn generate_code_at_time(&self, secret: &str, time: u64) -> Result { + let counter = time / self.config.period; + self.generate_hotp(secret, counter) + } + + /// Generate HOTP code (RFC 4226) + fn generate_hotp(&self, secret: &str, counter: u64) -> Result { + // Validate secret is not empty + if secret.is_empty() { + return Err(anyhow::anyhow!("Secret cannot be empty")); + } + + // Decode Base32 secret + let secret_bytes = base32::decode(Alphabet::Rfc4648 { padding: false }, secret) + .ok_or_else(|| anyhow::anyhow!("Invalid Base32 secret"))?; + + // Ensure decoded bytes are not empty + if secret_bytes.is_empty() { + return Err(anyhow::anyhow!("Decoded secret cannot be empty")); + } + + // Create HMAC-SHA1 + let mut mac = HmacSha1::new_from_slice(&secret_bytes) + .map_err(|e| anyhow::anyhow!("HMAC initialization failed: {}", e))?; + + // Counter as 8-byte big-endian + let counter_bytes = counter.to_be_bytes(); + mac.update(&counter_bytes); + + // Compute HMAC + let result = mac.finalize(); + let hmac_result = result.into_bytes(); + + // Dynamic truncation (RFC 4226 section 5.3) + let offset = usize::from(hmac_result[19_usize] & 0x0f); + let truncated_hash = u32::from_be_bytes([ + hmac_result[offset] & 0x7f, + hmac_result[offset + 1_usize], + hmac_result[offset + 2_usize], + hmac_result[offset + 3_usize], + ]); + + // Generate code with specified digits + let code = truncated_hash % 10u32.pow(self.config.digits); + let code_str = format!( + "{:0width$}", + code, + width = usize::try_from(self.config.digits).unwrap_or(6) + ); + + Ok(code_str) + } +} + +impl Default for TotpGenerator { + fn default() -> Self { + Self::new() + } +} + +/// TOTP verifier for validating codes +pub struct TotpVerifier { + config: TotpConfig, +} + +impl TotpVerifier { + /// Create new TOTP verifier with default configuration + pub fn new() -> Self { + Self { + config: TotpConfig::default(), + } + } + + /// Verify TOTP code with time drift tolerance + /// + /// # Arguments + /// * `secret` - Base32-encoded secret + /// + /// * `code` - TOTP code to verify + /// * `drift_tolerance` - Number of time steps to check before/after current time (0-2 recommended) + pub fn verify(&self, secret: &str, code: &str, drift_tolerance: u64) -> Result { + let current_time = u64::try_from(Utc::now().timestamp()) + .map_err(|_| anyhow::anyhow!("Negative timestamp cannot be converted to u64"))?; + + self.verify_at_time(secret, code, current_time, drift_tolerance) + } + + /// Verify TOTP code at specific time with drift tolerance + pub fn verify_at_time( + &self, + secret: &str, + code: &str, + time: u64, + drift_tolerance: u64, + ) -> Result { + // Validate code format + if code.len() != usize::try_from(self.config.digits).unwrap_or(6) { + return Ok(false); + } + + if !code.chars().all(|c| c.is_ascii_digit()) { + return Ok(false); + } + + let generator = TotpGenerator::new(); + let current_counter = time / self.config.period; + + // Check current time and drift windows + for offset in 0..=drift_tolerance { + // Check current + offset + if offset > 0 { + let forward_counter = current_counter + offset; + let forward_code = generator.generate_hotp(secret, forward_counter)?; + if constant_time_compare(code, &forward_code) { + return Ok(true); + } + } + + // Check current - offset + if offset > 0 && current_counter >= offset { + let backward_counter = current_counter - offset; + let backward_code = generator.generate_hotp(secret, backward_counter)?; + if constant_time_compare(code, &backward_code) { + return Ok(true); + } + } + + // Check exact current counter (offset == 0) + if offset == 0 { + let current_code = generator.generate_hotp(secret, current_counter)?; + if constant_time_compare(code, ¤t_code) { + return Ok(true); + } + } + } + + Ok(false) + } + + /// Get current time step counter + pub fn current_counter(&self) -> u64 { + let current_time = u64::try_from(Utc::now().timestamp()).unwrap_or_else(|_| { + warn!("Negative timestamp, using 0"); + 0 + }); + current_time / self.config.period + } + + /// Get time remaining in current period (seconds) + pub fn time_remaining(&self) -> u64 { + let current_time = u64::try_from(Utc::now().timestamp()).unwrap_or_else(|_| { + warn!("Negative timestamp, using 0"); + 0 + }); + self.config.period - (current_time % self.config.period) + } +} + +impl Default for TotpVerifier { + fn default() -> Self { + Self::new() + } +} + +/// Constant-time string comparison to prevent timing attacks +fn constant_time_compare(a: &str, b: &str) -> bool { + // Reject empty strings immediately (security: empty codes should never validate) + if a.is_empty() || b.is_empty() { + return false; + } + + if a.len() != b.len() { + return false; + } + + let mut result = 0u8; + for (x, y) in a.bytes().zip(b.bytes()) { + result |= x ^ y; + } + + result == 0 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_generate_secret() { + let generator = TotpGenerator::new(); + let secret = generator.generate_secret().unwrap(); + + // Base32 encoded secret should be non-empty + assert!(!secret.expose_secret().is_empty()); + + // Should be valid Base32 + assert!( + base32::decode(Alphabet::Rfc4648 { padding: false }, secret.expose_secret()).is_some() + ); + } + + #[test] + fn test_generate_qr_uri() { + let generator = TotpGenerator::new(); + let secret = SecretString::new("JBSWY3DPEHPK3PXP".into()); + let uri = generator + .generate_qr_uri(&secret, "FoxhuntHFT", "user@example.com") + .unwrap(); + + assert!(uri.starts_with("otpauth://totp/")); + assert!(uri.contains("secret=JBSWY3DPEHPK3PXP")); + assert!(uri.contains("issuer=FoxhuntHFT")); + assert!(uri.contains("digits=6")); + assert!(uri.contains("period=30")); + } + + #[test] + fn test_generate_and_verify_totp() { + let generator = TotpGenerator::new(); + let verifier = TotpVerifier::new(); + + let secret = "JBSWY3DPEHPK3PXP"; // Test secret + let current_time = 1234567890u64; + + // Generate code + let code = generator + .generate_code_at_time(secret, current_time) + .unwrap(); + assert_eq!(code.len(), 6); + + // Verify code + assert!(verifier + .verify_at_time(secret, &code, current_time, 1) + .unwrap()); + + // Verify invalid code + assert!(!verifier + .verify_at_time(secret, "000000", current_time, 1) + .unwrap()); + } + + #[test] + fn test_totp_drift_tolerance() { + let generator = TotpGenerator::new(); + let verifier = TotpVerifier::new(); + + let secret = "JBSWY3DPEHPK3PXP"; + let current_time = 1234567890u64; + let period = 30u64; + + // Generate code for current time + let code = generator + .generate_code_at_time(secret, current_time) + .unwrap(); + + // Should verify within drift tolerance (±1 period) + assert!(verifier + .verify_at_time(secret, &code, current_time + period, 1) + .unwrap()); + assert!(verifier + .verify_at_time(secret, &code, current_time - period, 1) + .unwrap()); + + // Should fail outside drift tolerance + assert!(!verifier + .verify_at_time(secret, &code, current_time + period * 2_u64, 1) + .unwrap()); + } + + #[test] + fn test_constant_time_compare() { + assert!(constant_time_compare("123456", "123456")); + assert!(!constant_time_compare("123456", "123457")); + assert!(!constant_time_compare("123456", "12345")); + + // Security test: Empty strings should NEVER validate + assert!(!constant_time_compare("", "")); + assert!(!constant_time_compare("", "123456")); + assert!(!constant_time_compare("123456", "")); + } + + #[test] + fn test_verifier_time_remaining() { + let verifier = TotpVerifier::new(); + let remaining = verifier.time_remaining(); + + // Should be between 0 and 30 seconds + assert!(remaining > 0 && remaining <= 30); + } + + #[test] + fn test_invalid_totp_code_format() { + let verifier = TotpVerifier::new(); + let secret = "JBSWY3DPEHPK3PXP"; + let current_time = 1234567890u64; + + // Wrong length + assert!(!verifier + .verify_at_time(secret, "12345", current_time, 1) + .unwrap()); + assert!(!verifier + .verify_at_time(secret, "1234567", current_time, 1) + .unwrap()); + + // Non-numeric + assert!(!verifier + .verify_at_time(secret, "12345a", current_time, 1) + .unwrap()); + assert!(!verifier + .verify_at_time(secret, "abcdef", current_time, 1) + .unwrap()); + } +} diff --git a/services/api/src/auth/mfa/verification.rs b/services/api/src/auth/mfa/verification.rs new file mode 100644 index 000000000..4245f0366 --- /dev/null +++ b/services/api/src/auth/mfa/verification.rs @@ -0,0 +1,180 @@ +//! MFA Verification Flow +//! +//! Handles verification of MFA codes during authentication. + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use uuid::Uuid; + +/// MFA verification request +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MfaVerification { + /// User ID + pub user_id: Uuid, + /// Verification method + pub method: VerificationMethod, + /// Code to verify + pub code: String, + /// Client IP address (for audit) + pub ip_address: Option, + /// User agent (for audit) + pub user_agent: Option, +} + +/// Verification method +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VerificationMethod { + /// TOTP code from authenticator app + Totp, + /// Backup recovery code + BackupCode, + /// Trusted device (future enhancement) + TrustedDevice, +} + +/// Verification result +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VerificationResult { + /// Whether verification succeeded + pub success: bool, + /// User ID + pub user_id: Uuid, + /// Verification method used + pub method: VerificationMethod, + /// Timestamp of verification + pub timestamp: DateTime, + /// Additional metadata + pub metadata: VerificationMetadata, +} + +/// Verification metadata +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VerificationMetadata { + /// Number of failed attempts + pub failed_attempts: i32, + /// Whether account is locked + pub is_locked: bool, + /// Lock expiration (if locked) + pub locked_until: Option>, + /// Remaining backup codes (if applicable) + pub backup_codes_remaining: Option, + /// Time drift in TOTP verification (if applicable) + pub totp_drift: Option, +} + +impl VerificationResult { + /// Create successful verification result + pub fn success( + user_id: Uuid, + method: VerificationMethod, + metadata: VerificationMetadata, + ) -> Self { + Self { + success: true, + user_id, + method, + timestamp: Utc::now(), + metadata, + } + } + + /// Create failed verification result + pub fn failure( + user_id: Uuid, + method: VerificationMethod, + metadata: VerificationMetadata, + ) -> Self { + Self { + success: false, + user_id, + method, + timestamp: Utc::now(), + metadata, + } + } +} + +/// Verification errors +#[derive(Debug, Error)] +pub enum VerificationError { + #[error("MFA not configured for user")] + NotConfigured, + + #[error("MFA is not enabled for user")] + NotEnabled, + + #[error("Account is locked due to too many failed attempts")] + AccountLocked, + + #[error("Invalid verification code")] + InvalidCode, + + #[error("Verification code has expired")] + CodeExpired, + + #[error("Backup code already used")] + BackupCodeUsed, + + #[error("No backup codes remaining")] + NoBackupCodesRemaining, + + #[error("Database error: {0}")] + DatabaseError(String), + + #[error("Internal error: {0}")] + InternalError(String), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_verification_result_success() { + let user_id = Uuid::new_v4(); + let metadata = VerificationMetadata { + failed_attempts: 0, + is_locked: false, + locked_until: None, + backup_codes_remaining: Some(10), + totp_drift: None, + }; + + let result = VerificationResult::success(user_id, VerificationMethod::Totp, metadata); + + assert!(result.success); + assert_eq!(result.user_id, user_id); + assert_eq!(result.method, VerificationMethod::Totp); + assert_eq!(result.metadata.failed_attempts, 0); + } + + #[test] + fn test_verification_result_failure() { + let user_id = Uuid::new_v4(); + let metadata = VerificationMetadata { + failed_attempts: 3, + is_locked: false, + locked_until: None, + backup_codes_remaining: Some(10), + totp_drift: None, + }; + + let result = VerificationResult::failure(user_id, VerificationMethod::Totp, metadata); + + assert!(!result.success); + assert_eq!(result.metadata.failed_attempts, 3); + } + + #[test] + fn test_verification_method_serialization() { + let method = VerificationMethod::Totp; + let json = serde_json::to_string(&method).expect("INVARIANT: Serialization should succeed for valid types"); + assert_eq!(json, "\"totp\""); + + let method = VerificationMethod::BackupCode; + let json = serde_json::to_string(&method).expect("INVARIANT: Serialization should succeed for valid types"); + assert_eq!(json, "\"backup_code\""); + } +} diff --git a/services/api/src/auth/mod.rs b/services/api/src/auth/mod.rs new file mode 100644 index 000000000..b1912a138 --- /dev/null +++ b/services/api/src/auth/mod.rs @@ -0,0 +1,33 @@ +//! Authentication module for API Gateway +//! +//! Provides comprehensive 6-layer authentication for all incoming gRPC requests: +//! 1. mTLS client certificate validation (tonic-tls) +//! 2. JWT extraction from Authorization header +//! 3. JWT revocation check (Redis-backed) +//! 4. JWT signature and expiration validation +//! 5. RBAC permission check (cached) +//! 6. Rate limiting (in-memory atomic counters) +//! 7. User context injection (gRPC metadata) +//! 8. Async audit logging (non-blocking) +//! +//! Performance targets: +//! - Total overhead: <10μs per request +//! - JWT validation: <1μs (cached decoding key) +//! - Revocation check: <500ns (Redis in same AZ) +//! - Authorization: <100ns (in-memory cache) +//! - Rate limiting: <50ns (atomic counters) + +pub mod interceptor; +pub mod jwt; +#[cfg(feature = "mfa")] +pub mod mfa; +pub mod mtls; + +// Re-export core authentication types +pub use interceptor::{ + AuditLogger, AuthInterceptor, AuthzService, CacheStats, Jti, JwtClaims, JwtService, + RateLimiter, RevocationService, UserContext, +}; + +// Re-export mTLS types +pub use mtls::{ApiGatewayTlsConfig, TlsInterceptor, TlsProtocolVersion}; diff --git a/services/api/src/auth/mtls/mod.rs b/services/api/src/auth/mtls/mod.rs new file mode 100644 index 000000000..0e5a5a0ca --- /dev/null +++ b/services/api/src/auth/mtls/mod.rs @@ -0,0 +1,77 @@ +//! Mutual TLS (mTLS) and X.509 Certificate Validation Module +//! +//! This module provides comprehensive mTLS support for the API Gateway: +//! +//! ## 6-Layer Security Validation +//! +//! 1. **Certificate Expiry Check** - Validates certificate is within valid time period +//! 2. **Revocation Check** - CRL and OCSP certificate revocation status +//! 3. **Certificate Chain Verification** - Validates signature chain to CA +//! 4. **Extended Key Usage** - Ensures certificate has TLS Client Authentication purpose +//! 5. **Signature Verification** - Validates certificate cryptographic signature +//! 6. **Hostname Verification** - Validates Subject Alternative Names (SAN) +//! +//! ## Architecture +//! +//! - **validator.rs**: X509CertificateValidator with 6-layer validation +//! - **revocation.rs**: CRL/OCSP revocation checking +//! - **tls_config.rs**: TLS server configuration and identity extraction +//! +//! ## Usage +//! +//! ```rust,no_run +//! use api::auth::mtls::{ApiGatewayTlsConfig, TlsInterceptor}; +//! use config::manager::ConfigManager; +//! use std::sync::Arc; +//! +//! # async fn example() -> anyhow::Result<()> { +//! // Load TLS configuration from config crate +//! let config_manager = ConfigManager::new().await?; +//! let tls_config = ApiGatewayTlsConfig::from_config(&config_manager).await?; +//! let tls_config = Arc::new(tls_config); +//! +//! // Create TLS interceptor for gRPC requests +//! let interceptor = TlsInterceptor::new(tls_config.clone()); +//! +//! // Convert to tonic ServerTlsConfig +//! let server_tls = tls_config.to_server_tls_config(); +//! # Ok(()) +//! # } +//! ``` +//! +//! ## TLS 1.3 Enforcement +//! +//! The API Gateway enforces TLS 1.3 by default for maximum security and performance. +//! TLS 1.2 is supported but not recommended for production use. +//! +//! ## Certificate Requirements +//! +//! Client certificates must: +//! - Be issued by a trusted CA +//! - Have Extended Key Usage with TLS Client Authentication (1.3.6.1.5.5.7.3.2) +//! - Have valid Organizational Unit (OU): trading, admin, analytics, risk, or compliance +//! - Not be marked as a CA certificate (Basic Constraints: CA=false) +//! - Not be revoked (if revocation checking enabled) +//! - Be within valid time period (not before/not after) +//! +//! ## Role-Based Access Control (RBAC) +//! +//! User roles are derived from certificate Organizational Unit (OU): +//! - **admin**: Full system access with configuration permissions +//! - **trading**: Order submission, modification, and cancellation +//! - **analytics**: Data analysis and backtesting +//! - **risk**: Position viewing and risk limit management +//! - **compliance**: Audit reports and regulatory compliance +//! + +pub mod revocation; +pub mod tls_config; +pub mod validator; + +// Re-export primary types +pub use validator::{ClientIdentity, UserRole, X509CertificateValidator}; + +pub use revocation::RevocationChecker; + +pub use common::tls::TlsProtocolVersion; +pub use tls_config::{ApiGatewayTlsConfig, TlsInterceptor}; diff --git a/services/api/src/auth/mtls/revocation.rs b/services/api/src/auth/mtls/revocation.rs new file mode 100644 index 000000000..02a97f761 --- /dev/null +++ b/services/api/src/auth/mtls/revocation.rs @@ -0,0 +1,791 @@ +//! Certificate Revocation Checking (CRL and OCSP) +//! +//! Provides certificate revocation status checking via: +//! - OCSP (Online Certificate Status Protocol) - RFC 6960 (primary) +//! - CRL (Certificate Revocation List) - RFC 5280 (fallback) + +use anyhow::{anyhow, Context, Result}; +use lru::LruCache; +use once_cell::sync::Lazy; +use prometheus::{register_histogram, register_int_counter, Histogram, IntCounter}; +use std::{ + num::NonZeroUsize, + sync::Arc, + time::{Duration, Instant}, +}; +use tokio::sync::RwLock; +use tracing::{debug, error, info, warn}; +use x509_parser::{ + certificate::X509Certificate, + extensions::{GeneralName, ParsedExtension}, + oid_registry::asn1_rs::oid, + prelude::FromDer, + revocation_list::CertificateRevocationList, + time::ASN1Time, +}; + +// --- Prometheus Metrics --- +// SAFETY: Prometheus metric registration fails only on duplicate names, +// which is a fatal misconfiguration. We abort rather than panic to avoid +// unwinding through FFI boundaries and to satisfy deny(clippy::unwrap_used). +static OCSP_REQUESTS_TOTAL: Lazy = Lazy::new(|| { + register_int_counter!("ocsp_requests_total", "Total OCSP requests sent").unwrap_or_else(|e| { + eprintln!("FATAL: failed to register ocsp_requests_total metric: {e}"); + std::process::abort() + }) +}); +static OCSP_CACHE_HITS: Lazy = Lazy::new(|| { + register_int_counter!("ocsp_cache_hits_total", "Total OCSP cache hits").unwrap_or_else(|e| { + eprintln!("FATAL: failed to register ocsp_cache_hits_total metric: {e}"); + std::process::abort() + }) +}); +static OCSP_CACHE_MISSES: Lazy = Lazy::new(|| { + register_int_counter!("ocsp_cache_misses_total", "Total OCSP cache misses").unwrap_or_else( + |e| { + eprintln!("FATAL: failed to register ocsp_cache_misses_total metric: {e}"); + std::process::abort() + }, + ) +}); +static OCSP_REVOKED_TOTAL: Lazy = Lazy::new(|| { + register_int_counter!( + "ocsp_revoked_certs_total", + "Total certificates found to be revoked via OCSP" + ) + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register ocsp_revoked_certs_total metric: {e}"); + std::process::abort() + }) +}); +static OCSP_REQUEST_FAILURES: Lazy = Lazy::new(|| { + register_int_counter!("ocsp_request_failures_total", "Total failed OCSP requests") + .unwrap_or_else(|e| { + eprintln!("FATAL: failed to register ocsp_request_failures_total metric: {e}"); + std::process::abort() + }) +}); +static OCSP_RESPONSE_VALIDATION_FAILURES: Lazy = Lazy::new(|| { + register_int_counter!( + "ocsp_response_validation_failures_total", + "Total OCSP response validation failures" + ) + .unwrap_or_else(|e| { + eprintln!( + "FATAL: failed to register ocsp_response_validation_failures_total metric: {e}" + ); + std::process::abort() + }) +}); +static OCSP_REQUEST_LATENCY: Lazy = Lazy::new(|| { + register_histogram!("ocsp_request_latency_seconds", "Latency of OCSP requests").unwrap_or_else( + |e| { + eprintln!("FATAL: failed to register ocsp_request_latency_seconds metric: {e}"); + std::process::abort() + }, + ) +}); + +// --- OCSP Cache Implementation --- + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OcspStatus { + Good, + Revoked, + Unknown, +} + +#[derive(Debug, Clone)] +struct OcspCacheEntry { + status: OcspStatus, + timestamp: Instant, +} + +#[derive(Debug, Clone)] +struct OcspCache { + cache: Arc>>, + ttl: Duration, +} + +impl OcspCache { + fn new(capacity: NonZeroUsize, ttl: Duration) -> Self { + Self { + cache: Arc::new(RwLock::new(LruCache::new(capacity))), + ttl, + } + } + + async fn get(&self, key: &str) -> Option { + let mut cache = self.cache.write().await; + if let Some(entry) = cache.get(key) { + if entry.timestamp.elapsed() < self.ttl { + OCSP_CACHE_HITS.inc(); + return Some(entry.status); + } + } + OCSP_CACHE_MISSES.inc(); + None + } + + /// Store OCSP status in cache with TTL + async fn put(&self, key: String, status: OcspStatus) { + let mut cache = self.cache.write().await; + let entry = OcspCacheEntry { + status, + timestamp: Instant::now(), + }; + cache.put(key, entry); + } +} + +/// Configuration for the RevocationChecker +#[derive(Debug, Clone)] +pub struct RevocationConfig { + pub crl_url: Option, + pub ocsp_responder_url: Option, + pub ocsp_cache_ttl: Duration, + pub ocsp_cache_capacity: NonZeroUsize, +} + +/// Certificate revocation checker +#[derive(Debug, Clone)] +pub struct RevocationChecker { + config: RevocationConfig, + ocsp_cache: OcspCache, + http_client: reqwest::Client, +} + +impl RevocationChecker { + /// Create new revocation checker with configuration + pub fn new(config: RevocationConfig) -> Result { + let http_client = reqwest::Client::builder() + .timeout(Duration::from_secs(10)) + .build() + .context("Failed to create HTTP client for revocation checking")?; + + let ocsp_cache = OcspCache::new(config.ocsp_cache_capacity, config.ocsp_cache_ttl); + + Ok(Self { + config, + ocsp_cache, + http_client, + }) + } + + /// Check certificate revocation status, prioritizing OCSP and falling back to CRL. + /// A valid issuer certificate is required for OCSP. + pub async fn check_revocation( + &self, + cert: &X509Certificate<'_>, + issuer: &X509Certificate<'_>, + ) -> Result<()> { + // --- OCSP Check (Primary) --- + let ocsp_urls = self.extract_ocsp_urls(cert); + + if !ocsp_urls.is_empty() { + for ocsp_url in &ocsp_urls { + match self.check_ocsp_revocation(cert, issuer, ocsp_url).await { + Ok(is_revoked) => { + if is_revoked { + OCSP_REVOKED_TOTAL.inc(); + return Err(anyhow!( + "Certificate has been revoked (OCSP check against: {})", + ocsp_url + )); + } + info!("Certificate OCSP check passed: {}", ocsp_url); + return Ok(()); // Success, not revoked + }, + Err(e) => { + OCSP_REQUEST_FAILURES.inc(); + warn!("OCSP check failed for {}: {}", ocsp_url, e); + // Continue to next OCSP URL + }, + } + } + } + + // --- CRL Check (Fallback) --- + if let Some(crl_url) = &self.config.crl_url { + if !ocsp_urls.is_empty() { + info!("OCSP checks failed/incomplete, falling back to CRL check."); + } + match self.check_crl_revocation(cert, crl_url).await { + Ok(is_revoked) => { + if is_revoked { + return Err(anyhow!( + "Certificate has been revoked (CRL check against: {})", + crl_url + )); + } + info!("Certificate CRL check passed: {}", crl_url); + return Ok(()); + }, + Err(e) => { + warn!("CRL check failed for {}: {}", crl_url, e); + // If both OCSP and CRL fail, the entire check fails. + return Err(e).context("All revocation checks (OCSP and CRL) failed"); + }, + } + } + + if ocsp_urls.is_empty() && self.config.crl_url.is_none() { + warn!("Revocation checking is enabled, but no OCSP or CRL URLs are available for certificate with SN: {:X}", cert.serial); + // In strict mode, this would be an error. For now, we allow it with a warning. + return Ok(()); + } + + // If we attempted OCSP and it failed, and there's no CRL to fall back to, fail closed. + if !ocsp_urls.is_empty() { + return Err(anyhow!( + "All OCSP checks failed/incomplete and no CRL fallback is configured." + )); + } + + Ok(()) + } + + /// Extracts OCSP responder URLs from the certificate's AIA extension, + /// with a fallback to the configured URL. + fn extract_ocsp_urls(&self, cert: &X509Certificate<'_>) -> Vec { + let mut urls = Vec::new(); + + // OID for Authority Information Access (1.3.6.1.5.5.7.1.1) + let aia_oid = oid!(1.3.6 .1 .5 .5 .7 .1 .1); + // OID for OCSP access method (1.3.6.1.5.5.7.48.1) + let ocsp_oid = oid!(1.3.6 .1 .5 .5 .7 .48 .1); + + // Extract OCSP URLs from Authority Information Access extension + for ext in cert.extensions() { + if ext.oid == aia_oid { + if let ParsedExtension::AuthorityInfoAccess(aia) = ext.parsed_extension() { + for desc in &aia.accessdescs { + if desc.access_method == ocsp_oid { + if let GeneralName::URI(uri) = &desc.access_location { + urls.push(uri.to_string()); + } + } + } + } + } + } + + if let Some(fallback_url) = &self.config.ocsp_responder_url { + if !urls.contains(fallback_url) { + urls.push(fallback_url.clone()); + } + } + urls + } + + /// Check certificate against CRL (Certificate Revocation List) + async fn check_crl_revocation( + &self, + cert: &X509Certificate<'_>, + crl_url: &str, + ) -> Result { + debug!("Checking certificate revocation via CRL: {}", crl_url); + + let crl_response = self + .http_client + .get(crl_url) + .send() + .await + .context("Failed to download CRL")?; + + if !crl_response.status().is_success() { + return Err(anyhow!( + "Failed to download CRL: received status {}", + crl_response.status() + )); + } + + let crl_bytes = crl_response + .bytes() + .await + .context("Failed to read CRL response")?; + + let (_, crl) = CertificateRevocationList::from_der(&crl_bytes) + .map_err(|e| anyhow!("Failed to parse CRL: {}", e))?; + + // Validate CRL validity period + let now = ASN1Time::now(); + let last_update = crl.last_update(); + if now < last_update { + warn!( + "CRL not yet valid: thisUpdate ({}) is in the future", + last_update + ); + return Err(anyhow!( + "CRL not yet valid: thisUpdate ({}) is in the future", + last_update + )); + } + + if let Some(next_update) = crl.next_update() { + if now > next_update { + warn!( + "CRL has expired: nextUpdate ({}) is in the past", + next_update + ); + return Err(anyhow!( + "CRL has expired: nextUpdate ({}) is in the past", + next_update + )); + } + debug!( + "CRL validity period OK: thisUpdate={}, nextUpdate={}", + last_update, next_update + ); + } else { + warn!("CRL does not contain a nextUpdate field; assuming still valid"); + } + + // Validate CRL issuer matches the certificate issuer + let crl_issuer = crl.issuer(); + let cert_issuer = cert.issuer(); + if crl_issuer != cert_issuer { + warn!( + "CRL issuer mismatch: CRL issuer='{}', certificate issuer='{}'", + crl_issuer, cert_issuer + ); + return Err(anyhow!( + "CRL issuer '{}' does not match certificate issuer '{}'", + crl_issuer, + cert_issuer + )); + } + debug!("CRL issuer validation passed: {}", crl_issuer); + + // Note: CRL cryptographic signature verification requires the x509-parser + // "verify" feature (backed by ring). If enabled, call crl.verify_signature() + // with the issuer's public key. Currently not enabled in this build. + + for revoked_cert in crl.iter_revoked_certificates() { + if revoked_cert.raw_serial() == cert.raw_serial() { + error!( + "Certificate REVOKED! Serial: {:X}, Revocation date: {:?}", + cert.serial, revoked_cert.revocation_date + ); + return Ok(true); + } + } + + Ok(false) + } + + /// Check certificate via OCSP (Online Certificate Status Protocol) - RFC 6960 + /// + /// Builds an OCSP request using SHA-1 hashes of the issuer name and public key + /// (per RFC 6960 Section 4.1.1), sends it via HTTP POST to the responder, and + /// parses the DER-encoded response to determine revocation status. + async fn check_ocsp_revocation( + &self, + cert: &X509Certificate<'_>, + issuer: &X509Certificate<'_>, + ocsp_url: &str, + ) -> Result { + debug!( + "OCSP check requested for certificate SN {:X} against {}", + cert.serial, ocsp_url + ); + OCSP_REQUESTS_TOTAL.inc(); + let _timer = OCSP_REQUEST_LATENCY.start_timer(); + + let cache_key = format!("{:X}", cert.serial); + + // Check cache first + if let Some(status) = self.ocsp_cache.get(&cache_key).await { + debug!("OCSP cache hit for SN: {}", cache_key); + return match status { + OcspStatus::Good => Ok(false), + OcspStatus::Revoked => Ok(true), + OcspStatus::Unknown => Err(anyhow!("Cached OCSP status is Unknown")), + }; + } + debug!("OCSP cache miss for SN: {}", cache_key); + + // Build the OCSP request DER bytes + let ocsp_request_der = Self::build_ocsp_request(cert, issuer) + .context("Failed to build OCSP request")?; + + // Send OCSP request via HTTP POST (RFC 6960 Appendix A) + let response = self + .http_client + .post(ocsp_url) + .header("Content-Type", "application/ocsp-request") + .header("Accept", "application/ocsp-response") + .body(ocsp_request_der) + .send() + .await + .context("Failed to send OCSP request")?; + + if !response.status().is_success() { + return Err(anyhow!( + "OCSP responder returned HTTP status {}", + response.status() + )); + } + + let response_bytes = response + .bytes() + .await + .context("Failed to read OCSP response body")?; + + if response_bytes.is_empty() { + return Err(anyhow!("OCSP responder returned empty response")); + } + + // Parse the OCSP response status and certificate status + let status = Self::parse_ocsp_response(&response_bytes, cert) + .context("Failed to parse OCSP response")?; + + // Cache the result + self.ocsp_cache.put(cache_key.clone(), status).await; + debug!("OCSP result cached for SN {}: {:?}", cache_key, status); + + match status { + OcspStatus::Good => Ok(false), + OcspStatus::Revoked => Ok(true), + OcspStatus::Unknown => Err(anyhow!( + "OCSP responder returned Unknown status for certificate SN {:X}", + cert.serial + )), + } + } + + /// Encode ASN.1 DER length bytes. + fn der_encode_length(len: usize) -> Vec { + if len < 128 { + vec![len as u8] + } else { + let be_bytes: Vec = len.to_be_bytes().into_iter().skip_while(|b| *b == 0).collect(); + let mut result = vec![0x80 | be_bytes.len() as u8]; + result.extend(be_bytes); + result + } + } + + /// Wrap content bytes in an ASN.1 SEQUENCE (tag 0x30). + fn wrap_in_sequence(content: &[u8]) -> Vec { + let mut result = vec![0x30u8]; + result.extend(Self::der_encode_length(content.len())); + result.extend_from_slice(content); + result + } + + /// Build a DER-encoded OCSP request for the given certificate and issuer. + /// + /// Per RFC 6960 Section 4.1.1, the CertID is constructed from: + /// - SHA-1 hash of the issuer's distinguished name (DER encoding) + /// - SHA-1 hash of the issuer's public key (bit string value) + /// - The certificate's serial number + fn build_ocsp_request( + cert: &X509Certificate<'_>, + issuer: &X509Certificate<'_>, + ) -> Result> { + use ocsp::common::asn1::{CertId, Oid}; + use ocsp::oid::ALGO_SHA1_DOT; + use ocsp::request::OneReq; + use sha1::{Digest, Sha1}; + + // SHA-1 hash of the issuer's distinguished name (DER-encoded) + let issuer_name_der = issuer.subject().as_raw(); + let issuer_name_hash: Vec = Sha1::digest(issuer_name_der).to_vec(); + + // SHA-1 hash of the issuer's public key (BIT STRING value, excluding tag/length) + let issuer_pubkey_data = &issuer.public_key().subject_public_key.data; + let issuer_key_hash: Vec = Sha1::digest(issuer_pubkey_data).to_vec(); + + // Certificate serial number as raw bytes + let serial_bytes = cert.raw_serial().to_vec(); + + // Build the CertId using SHA-1 algorithm OID + let sha1_oid = Oid::new_from_dot(ALGO_SHA1_DOT) + .map_err(|e| anyhow!("Failed to create SHA-1 OID: {:?}", e))?; + let cert_id = CertId::new(sha1_oid, &issuer_name_hash, &issuer_key_hash, &serial_bytes); + + // Build the single request (no per-request extensions) + let one_req = OneReq { + certid: cert_id, + one_req_ext: None, + }; + + // DER-encode the OneReq + let one_req_der = one_req + .to_der() + .map_err(|e| anyhow!("Failed to DER-encode OneReq: {:?}", e))?; + + // Wrap in requestList SEQUENCE, then TBSRequest SEQUENCE, then OCSPRequest SEQUENCE + let request_list = Self::wrap_in_sequence(&one_req_der); + let tbs_request = Self::wrap_in_sequence(&request_list); + let ocsp_request = Self::wrap_in_sequence(&tbs_request); + + debug!( + "Built OCSP request ({} bytes) for certificate SN {:X}", + ocsp_request.len(), + cert.serial + ); + Ok(ocsp_request) + } + + /// Parse a DER-encoded OCSP response to extract the certificate status. + /// + /// The OCSP response structure (RFC 6960 Section 4.2.1): + /// + /// OCSPResponse ::= SEQUENCE { + /// responseStatus OCSPResponseStatus, + /// responseBytes [0] EXPLICIT ResponseBytes OPTIONAL } + /// + /// We extract the response status first, then scan for the certificate + /// status tag within the response data: + /// - 0x80 0x00 = Good + /// - 0xa1 ... = Revoked (with RevokedInfo) + /// - 0x82 0x00 = Unknown + fn parse_ocsp_response(response_bytes: &[u8], cert: &X509Certificate<'_>) -> Result { + if response_bytes.len() < 5 { + return Err(anyhow!( + "OCSP response too short ({} bytes)", + response_bytes.len() + )); + } + + let first_byte = response_bytes.first().copied().unwrap_or(0); + if first_byte != 0x30 { + return Err(anyhow!( + "OCSP response does not start with SEQUENCE tag (got 0x{:02x})", + first_byte + )); + } + + let resp_status = Self::find_response_status(response_bytes)?; + + match resp_status { + 0 => debug!("OCSP response status: successful"), + 1 => return Err(anyhow!("OCSP responder: malformed request")), + 2 => return Err(anyhow!("OCSP responder: internal error")), + 3 => return Err(anyhow!("OCSP responder: try later")), + 5 => return Err(anyhow!("OCSP responder: signature required")), + 6 => return Err(anyhow!("OCSP responder: unauthorized")), + _ => return Err(anyhow!("OCSP responder: unknown status {}", resp_status)), + } + + let serial_bytes = cert.raw_serial(); + let cert_status = Self::find_cert_status_in_response(response_bytes, serial_bytes)?; + + debug!("OCSP certificate status for SN {:X}: {:?}", cert.serial, cert_status); + Ok(cert_status) + } + + /// Extract the OCSPResponseStatus ENUMERATED value from the response. + fn find_response_status(data: &[u8]) -> Result { + let content = Self::parse_tlv_content(data) + .context("Failed to parse OCSP response outer SEQUENCE")?; + + if content.len() < 3 { + return Err(anyhow!("OCSP response content too short for status")); + } + let tag = content.first().copied().unwrap_or(0); + if tag != 0x0a { + return Err(anyhow!("Expected ENUMERATED tag (0x0a) but got 0x{:02x}", tag)); + } + let len = content.get(1).copied().unwrap_or(0) as usize; + if len != 1 || content.len() < 3 { + return Err(anyhow!("Invalid ENUMERATED length in OCSP response status")); + } + content + .get(2) + .copied() + .ok_or_else(|| anyhow!("Missing OCSP response status value")) + } + + /// Parse a DER TLV (Tag-Length-Value) and return the value (content) portion. + fn parse_tlv_content(data: &[u8]) -> Result<&[u8]> { + if data.len() < 2 { + return Err(anyhow!("TLV data too short")); + } + let len_byte = data.get(1).copied().unwrap_or(0); + if len_byte & 0x80 == 0 { + let content_start = 2; + let content_len = len_byte as usize; + if data.len() < content_start + content_len { + return Err(anyhow!("TLV data truncated (short form)")); + } + Ok(data.get(content_start..content_start + content_len).unwrap_or(&[])) + } else { + let num_len_bytes = (len_byte & 0x7f) as usize; + if num_len_bytes == 0 || num_len_bytes > 4 { + return Err(anyhow!("Invalid long-form length byte count: {}", num_len_bytes)); + } + let content_start = 2 + num_len_bytes; + if data.len() < content_start { + return Err(anyhow!("TLV data truncated (long form header)")); + } + let mut content_len: usize = 0; + for i in 0..num_len_bytes { + content_len = (content_len << 8) + | data + .get(2 + i) + .copied() + .ok_or_else(|| anyhow!("TLV long form length truncated"))? + as usize; + } + if data.len() < content_start + content_len { + return Err(anyhow!("TLV data truncated (long form content)")); + } + Ok(data.get(content_start..content_start + content_len).unwrap_or(&[])) + } + } + + /// Search the OCSP response body for the certificate status associated with our serial. + fn find_cert_status_in_response(data: &[u8], serial: &[u8]) -> Result { + // Build the DER-encoded serial INTEGER to search for + let mut serial_der = vec![0x02u8]; // INTEGER tag + if serial.len() < 128 { + serial_der.push(serial.len() as u8); + } else { + return Err(anyhow!("Serial number too long for OCSP response parsing")); + } + serial_der.extend_from_slice(serial); + + // Find the serial in the response data + let serial_pos = Self::find_subsequence(data, &serial_der).ok_or_else(|| { + OCSP_RESPONSE_VALIDATION_FAILURES.inc(); + anyhow!("Certificate serial number not found in OCSP response") + })?; + + // After the serial INTEGER, scan forward for the cert status tag + let search_start = serial_pos + serial_der.len(); + let search_window = data.get(search_start..).unwrap_or(&[]); + let max_scan = std::cmp::min(search_window.len(), 128); + + for i in 0..max_scan { + let tag = search_window.get(i).copied().unwrap_or(0); + match tag { + // Good: context-specific [0] IMPLICIT NULL = 0x80 0x00 + 0x80 => { + if search_window.get(i + 1).copied().unwrap_or(0xFF) == 0x00 { + return Ok(OcspStatus::Good); + } + } + // Revoked: context-specific [1] CONSTRUCTED = 0xa1 with nonzero length + 0xa1 => { + if search_window.get(i + 1).copied().unwrap_or(0) > 0 { + return Ok(OcspStatus::Revoked); + } + } + // Unknown: context-specific [2] IMPLICIT NULL = 0x82 0x00 + 0x82 => { + if search_window.get(i + 1).copied().unwrap_or(0xFF) == 0x00 { + return Ok(OcspStatus::Unknown); + } + } + _ => {} + } + } + + OCSP_RESPONSE_VALIDATION_FAILURES.inc(); + Err(anyhow!("Could not determine certificate status from OCSP response")) + } + + /// Find a subsequence (needle) within a byte slice (haystack). + fn find_subsequence(haystack: &[u8], needle: &[u8]) -> Option { + if needle.is_empty() || needle.len() > haystack.len() { + return None; + } + haystack.windows(needle.len()).position(|window| window == needle) + } + + /// Get cache statistics for health monitoring + pub fn get_cache_stats(&self) -> CacheStats { + CacheStats { + total_requests: OCSP_REQUESTS_TOTAL.get(), + cache_hits: OCSP_CACHE_HITS.get(), + cache_misses: OCSP_CACHE_MISSES.get(), + revoked_certs: OCSP_REVOKED_TOTAL.get(), + request_failures: OCSP_REQUEST_FAILURES.get(), + validation_failures: OCSP_RESPONSE_VALIDATION_FAILURES.get(), + } + } +} + +/// Cache statistics for monitoring +#[derive(Debug, Clone)] +pub struct CacheStats { + pub total_requests: u64, + pub cache_hits: u64, + pub cache_misses: u64, + pub revoked_certs: u64, + pub request_failures: u64, + pub validation_failures: u64, +} + +impl CacheStats { + /// Calculate cache hit rate (0.0 to 1.0) + pub fn hit_rate(&self) -> f64 { + let total_lookups = self.cache_hits + self.cache_misses; + if total_lookups == 0 { + 0.0 + } else { + self.cache_hits as f64 / total_lookups as f64 + } + } + + /// Calculate failure rate (0.0 to 1.0) + pub fn failure_rate(&self) -> f64 { + if self.total_requests == 0 { + 0.0 + } else { + (self.request_failures + self.validation_failures) as f64 / self.total_requests as f64 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::num::NonZeroUsize; + + #[test] + fn test_revocation_checker_creation() { + let config = RevocationConfig { + crl_url: Some("http://crl.example.com/ca.crl".to_string()), + ocsp_responder_url: Some("http://ocsp.example.com".to_string()), + ocsp_cache_ttl: Duration::from_secs(1800), + ocsp_cache_capacity: NonZeroUsize::new(1000).unwrap(), + }; + let checker = RevocationChecker::new(config).unwrap(); + assert!(checker.config.crl_url.is_some()); + assert!(checker.config.ocsp_responder_url.is_some()); + } + + #[test] + fn test_cache_stats() { + let stats = CacheStats { + total_requests: 100, + cache_hits: 75, + cache_misses: 25, + revoked_certs: 2, + request_failures: 5, + validation_failures: 1, + }; + + assert_eq!(stats.hit_rate(), 0.75); + assert_eq!(stats.failure_rate(), 0.06); + } + + #[test] + fn test_cache_stats_zero_requests() { + let stats = CacheStats { + total_requests: 0, + cache_hits: 0, + cache_misses: 0, + revoked_certs: 0, + request_failures: 0, + validation_failures: 0, + }; + + assert_eq!(stats.hit_rate(), 0.0); + assert_eq!(stats.failure_rate(), 0.0); + } +} diff --git a/services/api/src/auth/mtls/tls_config.rs b/services/api/src/auth/mtls/tls_config.rs new file mode 100644 index 000000000..59fe41bdf --- /dev/null +++ b/services/api/src/auth/mtls/tls_config.rs @@ -0,0 +1,288 @@ +//! TLS Server Configuration for API Gateway with mutual TLS +//! +//! This module provides enterprise-grade TLS configuration: +//! - Mutual TLS (mTLS) for all gRPC connections +//! - Static certificate management via config crate +//! - Client certificate validation and authentication +//! - TLS 1.3 enforcement for maximum security +//! - Performance optimized for HFT requirements + +use anyhow::{Context, Result}; +use common::tls::TlsProtocolVersion; +use config::manager::ConfigManager; +use config::structures::TlsConfig; +use std::sync::Arc; +use tonic::transport::{Certificate, Identity, ServerTlsConfig}; +use tracing::info; + +use super::validator::{ClientIdentity, X509CertificateValidator}; + +/// TLS configuration for the API Gateway +#[derive(Debug, Clone)] +pub struct ApiGatewayTlsConfig { + /// Server certificate and private key + pub server_identity: Identity, + /// CA certificate for client verification + pub ca_certificate: Certificate, + /// Parsed CA certificate (PEM bytes) for OCSP validation + pub ca_cert_pem: Vec, + /// Require client certificates + pub require_client_cert: bool, + /// TLS protocol version (1.2 or 1.3) + pub protocol_version: TlsProtocolVersion, + /// Certificate validator with 6-layer validation + pub validator: Arc, +} + +impl ApiGatewayTlsConfig { + /// Create TLS configuration from certificate files + pub async fn from_files( + cert_path: &str, + key_path: &str, + ca_cert_path: &str, + require_client_cert: bool, + enable_revocation_check: bool, + crl_url: Option, + ) -> Result { + info!("Loading TLS certificates from filesystem"); + + // Read server certificate and key + let cert_pem = tokio::fs::read_to_string(cert_path) + .await + .with_context(|| format!("Failed to read certificate file: {}", cert_path))?; + + let key_pem = tokio::fs::read_to_string(key_path) + .await + .with_context(|| format!("Failed to read private key file: {}", key_path))?; + + // Combine certificate and key for server identity + let server_identity = Identity::from_pem(cert_pem, key_pem); + + // Read CA certificate for client verification + let ca_pem = tokio::fs::read_to_string(ca_cert_path) + .await + .with_context(|| format!("Failed to read CA certificate file: {}", ca_cert_path))?; + + let ca_cert_pem = ca_pem.as_bytes().to_vec(); + let ca_certificate = Certificate::from_pem(ca_pem); + + // Create validator with 6-layer validation + let validator = Arc::new(X509CertificateValidator::new( + enable_revocation_check, + crl_url, + )); + + info!( + "TLS certificates loaded successfully - mTLS: {}, Revocation: {}", + require_client_cert, enable_revocation_check + ); + + Ok(Self { + server_identity, + ca_certificate, + ca_cert_pem, + require_client_cert, + protocol_version: TlsProtocolVersion::Tls13, + validator, + }) + } + + /// Create TLS configuration from config crate + pub async fn from_config(config_manager: &ConfigManager) -> Result { + info!("Loading TLS certificates from configuration"); + + // Get the service config which contains settings as JSON + let service_config = config_manager.get_config(); + + // Extract TLS config from the settings JSON field + let tls_config: TlsConfig = serde_json::from_value( + service_config + .settings + .get("tls") + .cloned() + .unwrap_or(serde_json::json!({})), + ) + .unwrap_or_default(); + + // Use environment variable for CA cert path with fallback to /tmp + let ca_cert_path = std::env::var("TLS_CA_PATH") + .unwrap_or_else(|_| "/tmp/foxhunt/certs/ca.crt".to_string()); + + Self::from_files( + &tls_config.cert_path, + &tls_config.key_path, + tls_config.ca_cert_path.as_deref().unwrap_or(&ca_cert_path), + true, // Always require mTLS for API Gateway + false, // Default disabled for compatibility + None, // No CRL URL by default + ) + .await + } + + /// Convert to tonic ServerTlsConfig + pub fn to_server_tls_config(&self) -> ServerTlsConfig { + let mut tls_config = ServerTlsConfig::new().identity(self.server_identity.clone()); + + if self.require_client_cert { + tls_config = tls_config.client_ca_root(self.ca_certificate.clone()); + } + + tls_config + } + + /// Validate client certificate and extract identity + pub fn validate_client_certificate(&self, cert_chain: &[u8]) -> Result { + // Parse the X.509 certificate from PEM format + let (_, pem) = x509_parser::pem::parse_x509_pem(cert_chain) + .map_err(|e| anyhow::anyhow!("Failed to parse PEM certificate: {}", e))?; + + let cert = pem + .parse_x509() + .map_err(|e| anyhow::anyhow!("Failed to parse X.509 certificate: {}", e))?; + + // Comprehensive certificate validation using 6-layer validator + let client_identity = self.validator.extract_and_validate_certificate(&cert)?; + + tracing::info!( + "Client certificate validated: CN={}, OU={}", + client_identity.common_name, + client_identity.organizational_unit + ); + + Ok(client_identity) + } + + /// Validate client certificate with async revocation checking + pub async fn validate_client_certificate_async( + &self, + cert_chain: &[u8], + ) -> Result { + // Parse the X.509 client certificate from PEM format + let (_, pem) = x509_parser::pem::parse_x509_pem(cert_chain) + .map_err(|e| anyhow::anyhow!("Failed to parse PEM certificate: {}", e))?; + + let cert = pem + .parse_x509() + .map_err(|e| anyhow::anyhow!("Failed to parse X.509 certificate: {}", e))?; + + // Parse the CA certificate (issuer) for OCSP validation + let (_, ca_pem) = x509_parser::pem::parse_x509_pem(&self.ca_cert_pem) + .map_err(|e| anyhow::anyhow!("Failed to parse CA PEM certificate: {}", e))?; + + let ca_cert = ca_pem + .parse_x509() + .map_err(|e| anyhow::anyhow!("Failed to parse CA X.509 certificate: {}", e))?; + + // Comprehensive certificate validation (layers 1-5) + let client_identity = self.validator.extract_and_validate_certificate(&cert)?; + + // SECURITY CHECK 6: Async revocation checking (OCSP + CRL) + self.validator + .check_revocation_status_async(&cert, &ca_cert) + .await?; + + tracing::info!( + "Client certificate validated (with revocation check): CN={}, OU={}", + client_identity.common_name, + client_identity.organizational_unit + ); + + Ok(client_identity) + } +} + +/// TLS interceptor for gRPC requests +#[derive(Clone)] +pub struct TlsInterceptor { + tls_config: Arc, +} + +impl TlsInterceptor { + /// Create new TLS interceptor + pub fn new(tls_config: Arc) -> Self { + Self { tls_config } + } + + /// Extract and validate client certificate from request + pub fn extract_client_identity( + &self, + request: &tonic::Request, + ) -> Result { + // Get TLS info from request metadata + let tls_info = request + .extensions() + .get::>() + .ok_or_else(|| anyhow::anyhow!("No TLS connection info found"))?; + + // Extract client certificate if present + if let Some(cert_der) = tls_info + .peer_certs() + .and_then(|certs| certs.first().cloned()) + { + // Convert DER to PEM for processing + let cert_pem = self.der_to_pem(&cert_der)?; + self.tls_config.validate_client_certificate(&cert_pem) + } else { + Err(anyhow::anyhow!("No client certificate provided")) + } + } + + /// Extract and validate client certificate with async revocation checking + pub async fn extract_client_identity_async( + &self, + request: &tonic::Request, + ) -> Result { + // Get TLS info from request metadata + let tls_info = request + .extensions() + .get::>() + .ok_or_else(|| anyhow::anyhow!("No TLS connection info found"))?; + + // Extract client certificate if present + if let Some(cert_der) = tls_info + .peer_certs() + .and_then(|certs| certs.first().cloned()) + { + // Convert DER to PEM for processing + let cert_pem = self.der_to_pem(&cert_der)?; + self.tls_config + .validate_client_certificate_async(&cert_pem) + .await + } else { + Err(anyhow::anyhow!("No client certificate provided")) + } + } + + /// Convert DER certificate to PEM format + fn der_to_pem(&self, der_bytes: &[u8]) -> Result> { + use base64::{engine::general_purpose, Engine as _}; + + let b64_cert = general_purpose::STANDARD.encode(der_bytes); + let pem_cert = format!( + "-----BEGIN CERTIFICATE-----\n{}\n-----END CERTIFICATE-----\n", + b64_cert + .chars() + .collect::>() + .chunks(64) + .map(|chunk| chunk.iter().collect::()) + .collect::>() + .join("\n") + ); + + Ok(pem_cert.into_bytes()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_tls_protocol_version() { + let tls13 = TlsProtocolVersion::Tls13; + let tls12 = TlsProtocolVersion::Tls12; + + assert_ne!(tls13, tls12); + assert_eq!(tls13, TlsProtocolVersion::Tls13); + } +} diff --git a/services/api/src/auth/mtls/validator.rs b/services/api/src/auth/mtls/validator.rs new file mode 100644 index 000000000..e2d5130eb --- /dev/null +++ b/services/api/src/auth/mtls/validator.rs @@ -0,0 +1,555 @@ +//! X.509 Certificate Validator with 6-layer security validation +//! +//! Comprehensive certificate validation including: +//! 1. Certificate expiry check +//! 2. Revocation check (CRL + OCSP) +//! 3. Certificate chain verification +//! 4. Extended key usage validation +//! 5. Signature verification +//! 6. Hostname verification + +use anyhow::Result; +pub use common::tls::{ClientIdentity, UserRole}; +use tracing::{debug, info, warn}; +use x509_parser::certificate::X509Certificate; +use x509_parser::extensions::{GeneralName, ParsedExtension}; + +use super::revocation::{RevocationChecker, RevocationConfig}; +use std::num::NonZeroUsize; +use std::time::Duration; + +/// X.509 Certificate Validator with 6 security layers +#[derive(Debug, Clone)] +pub struct X509CertificateValidator { + /// Enable certificate revocation checking + pub enable_revocation_check: bool, + /// Revocation checker instance + pub revocation_checker: Option, +} + +impl X509CertificateValidator { + /// Create new validator with optional revocation checking + pub fn new(enable_revocation_check: bool, crl_url: Option) -> Self { + let revocation_checker = if enable_revocation_check { + let config = RevocationConfig { + crl_url, + ocsp_responder_url: None, + ocsp_cache_ttl: Duration::from_secs(1800), // 30 minutes + ocsp_cache_capacity: NonZeroUsize::new(1000) + .unwrap_or(NonZeroUsize::MIN), + }; + RevocationChecker::new(config).ok() + } else { + None + }; + + Self { + enable_revocation_check, + revocation_checker, + } + } + + /// Extract and validate certificate with comprehensive security checks + pub fn extract_and_validate_certificate( + &self, + cert: &X509Certificate<'_>, + ) -> Result { + // SECURITY CHECK 1: Certificate Validity Period (Expiration) + self.validate_certificate_expiration(cert)?; + + // SECURITY CHECK 2: Certificate Purpose (Extended Key Usage) + self.validate_certificate_purpose(cert)?; + + // SECURITY CHECK 3: Certificate Chain of Trust (Basic Constraints) + self.validate_certificate_constraints(cert)?; + + // SECURITY CHECK 4: Critical Extensions Validation + self.validate_critical_extensions(cert)?; + + // SECURITY CHECK 5: Subject Alternative Names (if present) + self.validate_subject_alternative_names(cert)?; + + // SECURITY CHECK 6: Certificate Revocation Status (CRL/OCSP) + // Note: Revocation checking is async and must be called separately + // via check_revocation_status_async() + + // Extract identity information from Subject DN + let subject = cert.subject(); + + // Extract Common Name (CN) + let common_name = subject + .iter_common_name() + .next() + .and_then(|cn| cn.as_str().ok()) + .ok_or_else(|| anyhow::anyhow!("Certificate missing Common Name (CN)"))? + .to_string(); + + // Extract Organizational Unit (OU) - required for RBAC + let organizational_unit = subject + .iter_organizational_unit() + .next() + .and_then(|ou| ou.as_str().ok()) + .ok_or_else(|| anyhow::anyhow!("Certificate missing Organizational Unit (OU)"))? + .to_string(); + + // Extract Serial Number + let serial_number = format!("{:X}", cert.serial); + + // Extract Issuer CN + let issuer = cert + .issuer() + .iter_common_name() + .next() + .and_then(|cn| cn.as_str().ok()) + .unwrap_or("Unknown Issuer") + .to_string(); + + // SECURITY: Validate organizational unit is in allowed list + let allowed_ous = ["trading", "admin", "analytics", "risk", "compliance"]; + if !allowed_ous.contains(&organizational_unit.as_str()) { + return Err(anyhow::anyhow!( + "Organizational Unit '{}' is not authorized for access. Allowed: {:?}", + organizational_unit, + allowed_ous + )); + } + + // SECURITY: Validate common name format (prevent injection attacks) + if !common_name + .chars() + .all(|c| c.is_alphanumeric() || c == '.' || c == '-' || c == '_') + { + return Err(anyhow::anyhow!( + "Common Name contains invalid characters: {}", + common_name + )); + } + + Ok(ClientIdentity { + common_name, + organizational_unit, + serial_number, + issuer, + }) + } + + /// SECURITY CHECK 1: Validate certificate expiration + fn validate_certificate_expiration(&self, cert: &X509Certificate<'_>) -> Result<()> { + let validity = cert.validity(); + + // Get current time + let now: i64 = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|e| anyhow::anyhow!("System time error: {}", e))? + .as_secs() + .try_into() + .map_err(|_| anyhow::anyhow!("Timestamp exceeds i64 range"))?; + // Check not before + let not_before = validity.not_before.timestamp(); + if now < not_before { + return Err(anyhow::anyhow!( + "Certificate not yet valid. Valid from: {}", + validity.not_before + )); + } + + // Check not after + let not_after = validity.not_after.timestamp(); + if now > not_after { + return Err(anyhow::anyhow!( + "Certificate expired. Expired on: {}", + validity.not_after + )); + } + + // SECURITY: Warn if certificate expires soon (within 30 days) + let thirty_days_secs = 30 * 24 * 3600; + if not_after - now < thirty_days_secs { + let days_remaining = (not_after - now) / (24 * 3600); + warn!( + "Certificate expires soon! Days remaining: {}. Expiration: {}", + days_remaining, validity.not_after + ); + } + + Ok(()) + } + + /// SECURITY CHECK 2: Validate certificate purpose via Extended Key Usage + fn validate_certificate_purpose(&self, cert: &X509Certificate<'_>) -> Result<()> { + // Look for Extended Key Usage extension + let mut has_client_auth = false; + let mut has_eku_extension = false; + + for ext in cert.extensions() { + if let ParsedExtension::ExtendedKeyUsage(eku) = ext.parsed_extension() { + has_eku_extension = true; + + // Check for TLS Client Authentication (OID: 1.3.6.1.5.5.7.3.2) + has_client_auth = eku.client_auth; + + if has_client_auth { + debug!("Certificate has TLS Client Authentication purpose"); + } else { + warn!( + "Certificate Extended Key Usage present but missing Client Auth. Purposes: {:?}", + eku + ); + } + } + } + + // SECURITY: Require Extended Key Usage with Client Auth for mTLS + if has_eku_extension && !has_client_auth { + return Err(anyhow::anyhow!( + "Certificate does not have TLS Client Authentication purpose (Extended Key Usage)" + )); + } + + // If no EKU extension, we allow it (some CAs don't set this for client certs) + // but log a warning for security awareness + if !has_eku_extension { + warn!( + "Certificate missing Extended Key Usage extension - certificate purpose cannot be verified" + ); + } + + Ok(()) + } + + /// SECURITY CHECK 3: Validate Basic Constraints (ensure not a CA certificate) + fn validate_certificate_constraints(&self, cert: &X509Certificate<'_>) -> Result<()> { + for ext in cert.extensions() { + if let ParsedExtension::BasicConstraints(bc) = ext.parsed_extension() { + // SECURITY: Client certificates should NOT be CA certificates + if bc.ca { + return Err(anyhow::anyhow!( + "Client certificate has CA flag set - this is a CA certificate, not a client certificate" + )); + } + + debug!("Certificate Basic Constraints validated: ca={}", bc.ca); + } + } + + Ok(()) + } + + /// SECURITY CHECK 4: Validate all critical extensions are recognized + fn validate_critical_extensions(&self, cert: &X509Certificate<'_>) -> Result<()> { + // List of recognized critical extensions (OIDs) + let recognized_critical = [ + "2.5.29.15", // Key Usage + "2.5.29.19", // Basic Constraints + "2.5.29.37", // Extended Key Usage + "2.5.29.17", // Subject Alternative Name + "2.5.29.32", // Certificate Policies + "2.5.29.35", // Authority Key Identifier + "2.5.29.14", // Subject Key Identifier + ]; + + for ext in cert.extensions() { + if ext.critical { + let oid_str = ext.oid.to_id_string(); + + // Check if this critical extension is recognized + if !recognized_critical.contains(&oid_str.as_str()) { + return Err(anyhow::anyhow!( + "Certificate contains unrecognized critical extension: {} - cannot safely process", + oid_str + )); + } + + debug!("Recognized critical extension: {}", oid_str); + } + } + + Ok(()) + } + + /// SECURITY CHECK 5: Validate Subject Alternative Names (if present) + fn validate_subject_alternative_names(&self, cert: &X509Certificate<'_>) -> Result<()> { + for ext in cert.extensions() { + if let ParsedExtension::SubjectAlternativeName(san) = ext.parsed_extension() { + // Extract and validate SAN entries + let mut san_entries = Vec::new(); + + for name in &san.general_names { + match name { + GeneralName::DNSName(dns) => { + san_entries.push(format!("DNS:{}", dns)); + + // SECURITY: Validate DNS name format + if !Self::is_valid_dns_name(dns) { + return Err(anyhow::anyhow!( + "Invalid DNS name in Subject Alternative Name: {}", + dns + )); + } + }, + GeneralName::RFC822Name(email) => { + san_entries.push(format!("Email:{}", email)); + }, + GeneralName::IPAddress(ip) => { + san_entries.push(format!("IP:{:?}", ip)); + }, + GeneralName::URI(uri) => { + san_entries.push(format!("URI:{}", uri)); + }, + _ => { + debug!("Other SAN type: {:?}", name); + }, + } + } + + if !san_entries.is_empty() { + debug!("Certificate Subject Alternative Names: {:?}", san_entries); + } + } + } + + Ok(()) + } + + /// Validate DNS name format (prevent injection attacks) + fn is_valid_dns_name(name: &str) -> bool { + // DNS name validation: alphanumeric, dots, hyphens, underscores + // Max 253 characters total, max 63 characters per label + if name.is_empty() || name.len() > 253 { + return false; + } + + for label in name.split('.') { + if label.is_empty() || label.len() > 63 { + return false; + } + + // Check valid characters: alphanumeric, hyphen, underscore + // Cannot start or end with hyphen + if !label + .chars() + .all(|c| c.is_alphanumeric() || c == '-' || c == '_') + { + return false; + } + + if label.starts_with('-') || label.ends_with('-') { + return false; + } + } + + true + } + + /// SECURITY CHECK 6: Check certificate revocation status via CRL or OCSP + /// + /// This is an async operation and must be called separately. + /// The issuer certificate is required for OCSP validation. + pub async fn check_revocation_status_async( + &self, + cert: &X509Certificate<'_>, + issuer: &X509Certificate<'_>, + ) -> Result<()> { + if !self.enable_revocation_check { + return Ok(()); + } + + if let Some(ref checker) = self.revocation_checker { + checker.check_revocation(cert, issuer).await + } else { + warn!("Revocation checking enabled but no checker configured"); + Ok(()) + } + } + + /// Validate certificate chain of trust against CA certificate + /// + /// Performs cryptographic signature verification of the client certificate + /// against the CA's public key using the `ring` crate (via x509-parser's verify feature). + /// + /// Validation steps: + /// 1. Parse both client and CA certificates from PEM + /// 2. Verify the client certificate's issuer CN matches the CA's subject CN + /// 3. Verify the CA certificate's validity period (not expired) + /// 4. Verify the CA has the cA Basic Constraint (is actually a CA) + /// 5. Cryptographically verify the client cert's signature using the CA's public key + pub fn validate_certificate_chain( + &self, + client_cert_pem: &[u8], + ca_cert_pem: &[u8], + ) -> Result<()> { + // Parse client certificate + let (_, client_pem) = x509_parser::pem::parse_x509_pem(client_cert_pem) + .map_err(|e| anyhow::anyhow!("Failed to parse client certificate PEM: {}", e))?; + + let client_cert = client_pem + .parse_x509() + .map_err(|e| anyhow::anyhow!("Failed to parse client X.509 certificate: {}", e))?; + + // Parse CA certificate + let (_, ca_pem) = x509_parser::pem::parse_x509_pem(ca_cert_pem) + .map_err(|e| anyhow::anyhow!("Failed to parse CA certificate PEM: {}", e))?; + + let ca_cert = ca_pem + .parse_x509() + .map_err(|e| anyhow::anyhow!("Failed to parse CA X.509 certificate: {}", e))?; + + // --- Step 1: Verify issuer CN matches CA subject CN --- + let client_issuer_cn = client_cert + .issuer() + .iter_common_name() + .next() + .and_then(|cn| cn.as_str().ok()) + .ok_or_else(|| anyhow::anyhow!("Client certificate missing issuer CN"))?; + + let ca_subject_cn = ca_cert + .subject() + .iter_common_name() + .next() + .and_then(|cn| cn.as_str().ok()) + .ok_or_else(|| anyhow::anyhow!("CA certificate missing subject CN"))?; + + if client_issuer_cn != ca_subject_cn { + return Err(anyhow::anyhow!( + "Certificate chain broken: client issuer CN '{}' does not match CA subject CN '{}'", + client_issuer_cn, + ca_subject_cn + )); + } + + debug!( + "Issuer CN match verified: client issuer='{}', CA subject='{}'", + client_issuer_cn, ca_subject_cn + ); + + // --- Step 2: Verify CA certificate is not expired --- + let ca_validity = ca_cert.validity(); + let now: i64 = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|e| anyhow::anyhow!("System time error: {}", e))? + .as_secs() + .try_into() + .map_err(|_| anyhow::anyhow!("Timestamp exceeds i64 range"))?; + + if now < ca_validity.not_before.timestamp() { + return Err(anyhow::anyhow!( + "CA certificate not yet valid. Valid from: {}", + ca_validity.not_before + )); + } + + if now > ca_validity.not_after.timestamp() { + return Err(anyhow::anyhow!( + "CA certificate expired. Expired on: {}", + ca_validity.not_after + )); + } + + // --- Step 3: Verify CA has Basic Constraints with cA=true --- + let mut ca_has_ca_flag = false; + for ext in ca_cert.extensions() { + if let ParsedExtension::BasicConstraints(bc) = ext.parsed_extension() { + if bc.ca { + ca_has_ca_flag = true; + } + } + } + + if !ca_has_ca_flag { + return Err(anyhow::anyhow!( + "CA certificate does not have Basic Constraints cA=true -- not a valid CA" + )); + } + + // --- Step 4: Cryptographic signature verification --- + // Use the CA's public key to verify the client certificate's signature. + // x509-parser's verify_signature uses ring internally to: + // - Extract the signature algorithm OID from the client cert + // - Map it to the appropriate ring verification algorithm + // (RSA-PKCS1-SHA256/384/512, ECDSA-P256/P384, Ed25519) + // - Extract the CA's SubjectPublicKeyInfo + // - Verify the signature over the client cert's TBS (to-be-signed) data + let ca_public_key = ca_cert.public_key(); + client_cert + .verify_signature(Some(ca_public_key)) + .map_err(|e| { + anyhow::anyhow!( + "Certificate signature verification failed: {} \ + (algorithm OID: {}, CA subject: '{}')", + e, + client_cert.signature_algorithm.algorithm, + ca_subject_cn + ) + })?; + + info!( + "Certificate chain verified: client cert signed by CA '{}' \ + (algorithm: {})", + ca_subject_cn, client_cert.signature_algorithm.algorithm + ); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_client_identity_authorization() { + let trading_identity = ClientIdentity { + common_name: "trader1.trading.foxhunt.internal".to_string(), + organizational_unit: "trading".to_string(), + serial_number: "12345".to_string(), + issuer: "Foxhunt Trading CA".to_string(), + }; + + assert!(trading_identity.is_authorized_for_trading()); + assert!(trading_identity.is_authorized_for_readonly()); + assert_eq!(trading_identity.get_role(), UserRole::Trader); + + let readonly_identity = ClientIdentity { + common_name: "analyst1.analytics.foxhunt.internal".to_string(), + organizational_unit: "analytics".to_string(), + serial_number: "12346".to_string(), + issuer: "Foxhunt Trading CA".to_string(), + }; + + assert!(!readonly_identity.is_authorized_for_trading()); + assert!(readonly_identity.is_authorized_for_readonly()); + assert_eq!(readonly_identity.get_role(), UserRole::Analyst); + } + + #[test] + fn test_user_role_permissions() { + let trader = UserRole::Trader; + let permissions = trader.get_permissions(); + + assert!(permissions.contains(&"trading.submit_order")); + assert!(permissions.contains(&"trading.cancel_order")); + assert!(!permissions.contains(&"system.configure")); + + let readonly = UserRole::ReadOnly; + let readonly_permissions = readonly.get_permissions(); + + assert!(!readonly_permissions.contains(&"trading.submit_order")); + assert!(readonly_permissions.contains(&"analytics.view_data")); + } + + #[test] + fn test_dns_name_validation() { + assert!(X509CertificateValidator::is_valid_dns_name("example.com")); + assert!(X509CertificateValidator::is_valid_dns_name( + "sub.example.com" + )); + assert!(X509CertificateValidator::is_valid_dns_name( + "test-server.internal" + )); + + assert!(!X509CertificateValidator::is_valid_dns_name("")); + assert!(!X509CertificateValidator::is_valid_dns_name("-invalid.com")); + assert!(!X509CertificateValidator::is_valid_dns_name("invalid-.com")); + assert!(!X509CertificateValidator::is_valid_dns_name("invalid..com")); + } +} diff --git a/services/api/src/config/authz.rs b/services/api/src/config/authz.rs new file mode 100644 index 000000000..11f5a7292 --- /dev/null +++ b/services/api/src/config/authz.rs @@ -0,0 +1,381 @@ +/// RBAC Authorization Service with Permission Caching +/// +/// Provides role-based access control with sub-100ns cached permission checks. +/// +/// Supports hot-reload via Postgre`SQL` NOTIFY/LISTEN for permission changes. +use anyhow::{Context, Result}; +use dashmap::DashMap; +use sqlx::PgPool; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::RwLock; +use tracing::{debug, error, info, warn}; +use uuid::Uuid; + +/// Performance metrics for RBAC operations +#[derive(Debug, Clone)] +pub struct AuthzMetrics { + pub cache_hits: u64, + pub cache_misses: u64, + pub avg_check_time_ns: u64, + pub last_reload_time: Option, +} + +/// Permission check result +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PermissionResult { + Allowed, + Denied, + NotFound, +} + +/// Cached user permissions +#[derive(Debug, Clone)] +struct UserPermissions { + permissions: HashSet, + loaded_at: Instant, +} + +/// Cached role permissions (stored for cache invalidation) +#[derive(Debug, Clone)] +struct RolePermissions; + +/// RBAC Authorization Service +pub struct AuthzService { + db_pool: Arc, + + // Cache: user_id -> Set - Lock-free with DashMap + user_permissions_cache: Arc>, + + // Cache: role_name -> Set - Lock-free with DashMap + role_permissions_cache: Arc>, + + // Performance metrics + metrics: Arc>, + + // Cache TTL (5 minutes default) + cache_ttl: Duration, +} + +impl AuthzService { + /// Create a new AuthzService instance + pub fn new(db_pool: Arc) -> Self { + Self { + db_pool, + user_permissions_cache: Arc::new(DashMap::new()), + role_permissions_cache: Arc::new(DashMap::new()), + metrics: Arc::new(RwLock::new(AuthzMetrics { + cache_hits: 0, + cache_misses: 0, + avg_check_time_ns: 0, + last_reload_time: None, + })), + cache_ttl: Duration::from_secs(300), // 5 minutes + } + } + + /// Create with custom cache TTL + pub fn with_cache_ttl(db_pool: Arc, cache_ttl: Duration) -> Self { + let mut service = Self::new(db_pool); + service.cache_ttl = cache_ttl; + service + } + + /// Check if user has permission for an endpoint + /// + /// This is the hot path - optimized for <8ns cached checks with lock-free DashMap + pub async fn check_permission( + &self, + user_id: &Uuid, + endpoint: &str, + ) -> Result { + let start = Instant::now(); + + // 1. Check cache first (fast path - lock-free DashMap access) + if let Some(user_perms_ref) = self.user_permissions_cache.get(user_id) { + // Check if cache entry is still valid + if user_perms_ref.loaded_at.elapsed() < self.cache_ttl { + let has_permission = user_perms_ref.permissions.contains(endpoint); + + // Update metrics + let mut metrics = self.metrics.write().await; + metrics.cache_hits += 1; + let elapsed_nanos = start.elapsed().as_nanos().try_into().unwrap_or_else(|_| { + warn!("Elapsed time exceeds u64::MAX nanoseconds"); + u64::MAX + }); + self.update_avg_time(&mut metrics, elapsed_nanos); + + debug!( + user_id = %user_id, + endpoint = endpoint, + result = has_permission, + duration_ns = start.elapsed().as_nanos(), + "Permission check (cache hit - lock-free)" + ); + + return Ok(if has_permission { + PermissionResult::Allowed + } else { + PermissionResult::Denied + }); + } + } + + // 2. Cache miss - load from database + let user_perms = self.load_user_permissions(user_id).await?; + + // 3. Check permission + let has_permission = user_perms.permissions.contains(endpoint); + + // 4. Update cache (lock-free insert) + self.user_permissions_cache.insert(*user_id, user_perms); + + // 5. Update metrics + { + let mut metrics = self.metrics.write().await; + metrics.cache_misses += 1; + let elapsed_nanos = start.elapsed().as_nanos().try_into().unwrap_or_else(|_| { + warn!("Elapsed time exceeds u64::MAX nanoseconds"); + u64::MAX + }); + self.update_avg_time(&mut metrics, elapsed_nanos); + } + + debug!( + user_id = %user_id, + endpoint = endpoint, + result = has_permission, + duration_ns = start.elapsed().as_nanos(), + "Permission check (cache miss)" + ); + + Ok(if has_permission { + PermissionResult::Allowed + } else { + PermissionResult::Denied + }) + } + + /// Load user permissions from database + async fn load_user_permissions(&self, user_id: &Uuid) -> Result { + #[derive(sqlx::FromRow)] + struct PermissionRow { + endpoint: String, + } + + let rows = sqlx::query_as::<_, PermissionRow>( + r#" + SELECT DISTINCT p.endpoint + FROM permissions p + JOIN role_permissions rp ON p.id = rp.permission_id + JOIN user_roles ur ON rp.role_id = ur.role_id + WHERE ur.user_id = $1 + "#, + ) + .bind(user_id) + .fetch_all(self.db_pool.as_ref()) + .await + .context("Failed to load user permissions from database")?; + + let permissions: HashSet = rows.into_iter().map(|row| row.endpoint).collect(); + + debug!( + user_id = %user_id, + permission_count = permissions.len(), + "Loaded user permissions from database" + ); + + Ok(UserPermissions { + permissions, + loaded_at: Instant::now(), + }) + } + + /// Reload all permissions from database + /// + /// Called on Postgre`SQL` NOTIFY for permission changes + pub async fn reload_permissions(&self) -> Result<()> { + info!("Reloading all permissions from database"); + let start = Instant::now(); + + // 1. Load all role-permission mappings + let role_perms = self.load_all_role_permissions().await?; + + // 2. Update role permissions cache (lock-free DashMap operations) + self.role_permissions_cache.clear(); + for (role_name, _permissions) in role_perms { + self.role_permissions_cache.insert( + role_name, + RolePermissions, + ); + } + + // 3. Clear user permissions cache (will be reloaded on demand) + self.user_permissions_cache.clear(); + + // 4. Update metrics + { + let mut metrics = self.metrics.write().await; + metrics.last_reload_time = Some(Instant::now()); + } + + info!( + duration_ms = start.elapsed().as_millis(), + "Permission reload complete (lock-free)" + ); + + Ok(()) + } + + /// Load all role-permission mappings from database + async fn load_all_role_permissions(&self) -> Result>> { + #[derive(sqlx::FromRow)] + struct RolePermissionRow { + role_name: String, + endpoint: String, + } + + let rows = sqlx::query_as::<_, RolePermissionRow>( + r#" + SELECT r.name as role_name, p.endpoint + FROM roles r + JOIN role_permissions rp ON r.id = rp.role_id + JOIN permissions p ON rp.permission_id = p.id + ORDER BY r.name, p.endpoint + "#, + ) + .fetch_all(self.db_pool.as_ref()) + .await + .context("Failed to load role permissions from database")?; + + let mut role_perms: HashMap> = HashMap::new(); + + for row in rows { + role_perms + .entry(row.role_name) + .or_default() + .insert(row.endpoint); + } + + debug!( + role_count = role_perms.len(), + "Loaded role permissions from database" + ); + + Ok(role_perms) + } + + /// Get current metrics + pub async fn get_metrics(&self) -> AuthzMetrics { + self.metrics.read().await.clone() + } + + /// Invalidate user permissions cache entry + pub async fn invalidate_user(&self, user_id: &Uuid) { + self.user_permissions_cache.remove(user_id); + debug!(user_id = %user_id, "Invalidated user permissions cache (lock-free)"); + } + + /// Invalidate all caches + pub async fn invalidate_all(&self) { + self.user_permissions_cache.clear(); + self.role_permissions_cache.clear(); + info!("Invalidated all permission caches (lock-free)"); + } + + /// Preload permissions for common users on startup + pub async fn preload_common_users(&self, user_ids: &[Uuid]) -> Result<()> { + info!(user_count = user_ids.len(), "Preloading user permissions"); + + for user_id in user_ids { + if let Err(e) = self.load_user_permissions(user_id).await { + warn!( + user_id = %user_id, + error = %e, + "Failed to preload user permissions" + ); + } + } + + Ok(()) + } + + /// Update average time metric with exponential moving average + fn update_avg_time(&self, metrics: &mut AuthzMetrics, duration_ns: u64) { + const ALPHA: f64 = 0.1; // EMA smoothing factor + + if metrics.avg_check_time_ns == 0 { + metrics.avg_check_time_ns = duration_ns; + } else { + metrics.avg_check_time_ns = (ALPHA * duration_ns as f64 + + (1.0 - ALPHA) * metrics.avg_check_time_ns as f64) + as u64; + } + } + + /// Start listening for Postgre`SQL` NOTIFY events + pub async fn start_notify_listener(self: Arc) -> Result<()> { + info!("Starting PostgreSQL NOTIFY listener for permission changes"); + + let mut listener = sqlx::postgres::PgListener::connect_with(self.db_pool.as_ref()) + .await + .context("Failed to create PostgreSQL listener")?; + + listener + .listen("permission_changes") + .await + .context("Failed to listen on permission_changes channel")?; + + // Spawn background task to handle notifications + tokio::spawn(async move { + loop { + match listener.recv().await { + Ok(notification) => { + info!( + channel = notification.channel(), + payload = notification.payload(), + "Received permission change notification" + ); + + if let Err(e) = self.reload_permissions().await { + error!(error = %e, "Failed to reload permissions on NOTIFY"); + } + }, + Err(e) => { + error!(error = %e, "Error receiving PostgreSQL notification"); + // Wait before retrying + tokio::time::sleep(Duration::from_secs(5)).await; + }, + } + } + }); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_permission_result() { + assert_eq!(PermissionResult::Allowed, PermissionResult::Allowed); + assert_ne!(PermissionResult::Allowed, PermissionResult::Denied); + } + + #[test] + fn test_metrics_creation() { + let metrics = AuthzMetrics { + cache_hits: 0, + cache_misses: 0, + avg_check_time_ns: 0, + last_reload_time: None, + }; + + assert_eq!(metrics.cache_hits, 0); + assert_eq!(metrics.cache_misses, 0); + } +} diff --git a/services/api/src/config/endpoints.rs b/services/api/src/config/endpoints.rs new file mode 100644 index 000000000..6f95c525b --- /dev/null +++ b/services/api/src/config/endpoints.rs @@ -0,0 +1,291 @@ +//! gRPC endpoints for configuration management + +use crate::config::ConfigurationManager; +use crate::error::GatewayConfigError; +use futures::Stream; +use std::pin::Pin; +use tonic::{Request, Response, Status}; +use tracing::{debug, info, instrument}; + +use crate::config_backend::{ + config_service_server::{ConfigService, ConfigServiceServer}, + ConfigChangeEvent, ConfigDataType, ConfigurationCategory, ConfigurationSetting, + DeleteConfigurationRequest, DeleteConfigurationResponse, ExportConfigurationRequest, + ExportConfigurationResponse, GetConfigSchemaRequest, GetConfigSchemaResponse, + GetConfigurationHistoryRequest, GetConfigurationHistoryResponse, GetConfigurationRequest, + GetConfigurationResponse, ImportConfigurationRequest, ImportConfigurationResponse, + ListCategoriesRequest, ListCategoriesResponse, RollbackConfigurationRequest, + RollbackConfigurationResponse, StreamConfigChangesRequest, UpdateConfigSchemaRequest, + UpdateConfigSchemaResponse, UpdateConfigurationRequest, UpdateConfigurationResponse, + ValidateConfigurationRequest, ValidateConfigurationResponse, +}; + +/// gRPC service implementation for gateway-local configuration management. +/// +/// Implements the `config.ConfigService` trait from config.proto. +/// Handles GetConfiguration, UpdateConfiguration, and ListCategories locally; +/// all other RPCs return `Status::unimplemented`. +pub struct GatewayConfigService { + manager: std::sync::Arc>, +} + +impl GatewayConfigService { + /// Creates a new GatewayConfigService + pub fn new(manager: ConfigurationManager) -> Self { + Self { + manager: std::sync::Arc::new(tokio::sync::RwLock::new(manager)), + } + } + + /// Returns the gRPC server instance + pub fn into_server(self) -> ConfigServiceServer { + ConfigServiceServer::new(self) + } +} + +#[tonic::async_trait] +impl ConfigService for GatewayConfigService { + type StreamConfigChangesStream = + Pin> + Send>>; + + #[instrument(skip_all)] + async fn get_configuration( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let category = req.category.unwrap_or_default(); + let key = req.key.unwrap_or_default(); + debug!("GetConfiguration: {}/{}", category, key); + + if category.is_empty() && key.is_empty() { + // List all configs + let manager = self.manager.read().await; + let configs = manager + .list_configs(None) + .await + .map_err(|e| Status::internal(format!("{}", e)))?; + + let settings: Result, Status> = configs + .into_iter() + .map(|c| config_to_setting(&c)) + .collect(); + + return Ok(Response::new(GetConfigurationResponse { + settings: settings?, + })); + } + + let manager = self.manager.read().await; + + if !key.is_empty() { + // Get specific config + let config = manager + .get_config(&category, &key) + .await + .map_err(|e| match e { + GatewayConfigError::NotFound { .. } => Status::not_found(format!("{}", e)), + _ => Status::internal(format!("{}", e)), + })?; + + Ok(Response::new(GetConfigurationResponse { + settings: vec![config_to_setting(&config)?], + })) + } else { + // List by category + let configs = manager + .list_configs(Some(&category)) + .await + .map_err(|e| Status::internal(format!("{}", e)))?; + + let settings: Result, Status> = configs + .into_iter() + .map(|c| config_to_setting(&c)) + .collect(); + + Ok(Response::new(GetConfigurationResponse { + settings: settings?, + })) + } + } + + #[instrument(skip_all)] + async fn update_configuration( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + info!( + "UpdateConfiguration: {}/{} by {}", + req.category, req.key, req.changed_by + ); + + let new_value: serde_json::Value = serde_json::from_str(&req.value) + .map_err(|e| Status::invalid_argument(format!("Invalid JSON: {}", e)))?; + + let manager = self.manager.read().await; + manager + .update_config(&req.category, &req.key, new_value, &req.changed_by) + .await + .map_err(|e| match e { + GatewayConfigError::Validation(msg) => Status::invalid_argument(msg), + GatewayConfigError::NotFound { .. } => Status::not_found(format!("{}", e)), + _ => Status::internal(format!("{}", e)), + })?; + + Ok(Response::new(UpdateConfigurationResponse { + success: true, + message: format!("Successfully updated {}/{}", req.category, req.key), + validation_result: None, + timestamp: chrono::Utc::now().timestamp(), + })) + } + + #[instrument(skip_all)] + async fn delete_configuration( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "Delete not supported by gateway config", + )) + } + + #[instrument(skip_all)] + async fn list_categories( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + debug!("ListCategories: {:?}", req.parent_category); + + let manager = self.manager.read().await; + let configs = manager + .list_configs(req.parent_category.as_deref()) + .await + .map_err(|e| Status::internal(format!("{}", e)))?; + + // Extract unique categories from configs + let mut seen = std::collections::HashSet::new(); + let categories = configs + .iter() + .filter_map(|c| { + if seen.insert(c.service_scope.clone()) { + Some(ConfigurationCategory { + id: 0, + name: c.service_scope.clone(), + description: String::new(), + parent_id: None, + display_order: 0, + icon: None, + created_at: c.created_at.timestamp(), + children: vec![], + }) + } else { + None + } + }) + .collect(); + + Ok(Response::new(ListCategoriesResponse { categories })) + } + + async fn stream_config_changes( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "Streaming not supported by gateway config", + )) + } + + async fn validate_configuration( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "Validation not supported by gateway config", + )) + } + + async fn get_configuration_history( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "History not supported by gateway config", + )) + } + + async fn rollback_configuration( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "Rollback not supported by gateway config", + )) + } + + async fn export_configuration( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "Export not supported by gateway config", + )) + } + + async fn import_configuration( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "Import not supported by gateway config", + )) + } + + async fn get_config_schema( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "Schema not supported by gateway config", + )) + } + + async fn update_config_schema( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "Schema update not supported by gateway config", + )) + } +} + +/// Convert a ConfigItem (from ConfigurationManager) to a ConfigurationSetting proto message +fn config_to_setting(c: &crate::config::ConfigItem) -> Result { + Ok(ConfigurationSetting { + id: 0, + category: c.service_scope.clone(), + key: c.config_key.clone(), + value: serde_json::to_string(&c.config_value) + .map_err(|e| Status::internal(format!("Serialization error: {}", e)))?, + data_type: ConfigDataType::String as i32, + hot_reload: false, + description: c.description.clone().unwrap_or_default(), + default_value: None, + required: false, + sensitive: false, + validation_rule: None, + environment_override: None, + min_value: None, + max_value: None, + enum_values: None, + depends_on: vec![], + tags: vec![], + display_order: 0, + created_at: c.created_at.timestamp(), + modified_at: c.updated_at.timestamp(), + }) +} diff --git a/services/api/src/config/manager.rs b/services/api/src/config/manager.rs new file mode 100644 index 000000000..ee1e74387 --- /dev/null +++ b/services/api/src/config/manager.rs @@ -0,0 +1,322 @@ +//! Configuration management with PostgreSQL NOTIFY/LISTEN hot-reload and Redis caching + +use crate::config::validator::ConfigValidator; +use crate::error::{GatewayConfigError, GatewayConfigResult}; +use chrono::{DateTime, Utc}; +use redis::aio::ConnectionManager; +use serde_json::Value; +use sqlx::PgPool; +use std::sync::Arc; +use tokio::sync::RwLock; +use tracing::{debug, info}; +use uuid::Uuid; + +/// Configuration item from database +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, sqlx::FromRow)] +pub struct ConfigItem { + pub id: Uuid, + pub service_scope: String, + pub config_key: String, + pub config_value: Value, + pub data_type: String, + pub validation_rules: Option, + pub description: Option, + pub is_active: bool, + pub created_at: DateTime, + pub updated_at: DateTime, + pub updated_by: Option, +} + +/// Centralized configuration manager with hot-reload and caching +pub struct ConfigurationManager { + /// Postgre`SQL` connection pool + db_pool: Arc, + /// Redis connection manager for caching + redis: Arc>, + /// Configuration validator + validator: Arc>, + /// Postgre`SQL` NOTIFY listener + listener: Option, +} + +impl ConfigurationManager { + /// Creates a new ConfigurationManager + /// + /// # Arguments + /// * `db_pool` - Postgre`SQL` connection pool + /// + /// * `redis` - Redis connection manager + pub async fn new(db_pool: PgPool, redis: ConnectionManager) -> GatewayConfigResult { + Ok(Self { + db_pool: Arc::new(db_pool), + redis: Arc::new(RwLock::new(redis)), + validator: Arc::new(RwLock::new(ConfigValidator::new())), + listener: None, + }) + } + + /// Starts listening for configuration changes via Postgre`SQL` NOTIFY + pub async fn start_listening(&mut self) -> GatewayConfigResult<()> { + let mut listener = sqlx::postgres::PgListener::connect_with(&self.db_pool).await?; + + // Listen to global config updates channel + listener.listen("config_updates_global").await?; + + info!("Started listening for configuration updates on 'config_updates_global'"); + + self.listener = Some(listener); + Ok(()) + } + + /// Processes configuration change notifications + pub async fn handle_notifications(&mut self) -> GatewayConfigResult<()> { + // Collect notifications first to avoid borrow checker issues + let mut invalidations = Vec::new(); + + if let Some(listener) = &mut self.listener { + while let Some(notification) = listener.try_recv().await? { + let payload = notification.payload(); + debug!("Received config notification: {}", payload); + + // Parse notification payload + if let Ok(payload_json) = serde_json::from_str::(payload) { + if let (Some(service_scope), Some(config_key)) = ( + payload_json.get("service_scope").and_then(|v| v.as_str()), + payload_json.get("config_key").and_then(|v| v.as_str()), + ) { + invalidations.push((service_scope.to_string(), config_key.to_string())); + } + } + } + } + + // Now invalidate caches without holding the listener borrow + for (service_scope, config_key) in invalidations { + self.invalidate_cache(&service_scope, &config_key).await?; + info!("Invalidated cache for {}/{}", service_scope, config_key); + } + + Ok(()) + } + + /// Retrieves a configuration value + /// + /// # Arguments + /// * `service_scope` - Service scope (e.g., "trading", "global") + /// + /// * `config_key` - Configuration key + /// + /// # Returns + /// + /// Configuration item if found + pub async fn get_config( + &self, + service_scope: &str, + config_key: &str, + ) -> GatewayConfigResult { + // Try Redis cache first + if let Some(cached) = self.get_from_cache(service_scope, config_key).await? { + debug!("Cache hit for {}/{}", service_scope, config_key); + return Ok(cached); + } + + // Cache miss - load from PostgreSQL + debug!("Cache miss for {}/{}", service_scope, config_key); + let config = self.load_from_db(service_scope, config_key).await?; + + // Update Redis cache + self.set_in_cache(&config).await?; + + Ok(config) + } + + /// Updates a configuration value with validation and audit logging + /// + /// # Arguments + /// * `service_scope` - Service scope + /// + /// * `config_key` - Configuration key + /// * `new_value` - New configuration value + /// + /// * `updated_by` - User ID making the change + pub async fn update_config( + &self, + service_scope: &str, + config_key: &str, + new_value: Value, + updated_by: &str, + ) -> GatewayConfigResult<()> { + // Load current configuration for validation rules and audit + let current = self.load_from_db(service_scope, config_key).await?; + + // Validate new value + { + let mut validator = self.validator.write().await; + validator.validate( + &new_value, + ¤t.data_type, + current.validation_rules.as_ref(), + )?; + } + + // Start transaction + let mut tx = self.db_pool.begin().await?; + + // Update config_settings + sqlx::query( + r#" + UPDATE config_settings + SET config_value = $1, updated_by = $2, updated_at = NOW() + WHERE service_scope = $3 AND config_key = $4 + "#, + ) + .bind(&new_value) + .bind(updated_by) + .bind(service_scope) + .bind(config_key) + .execute(&mut *tx) + .await?; + + // Insert audit log entry + sqlx::query( + r#" + INSERT INTO config_audit_log (action, service_scope, config_key, old_value, new_value, changed_by) + VALUES ('UPDATE', $1, $2, $3, $4, $5) + "#, + ) + .bind(service_scope) + .bind(config_key) + .bind(¤t.config_value) + .bind(&new_value) + .bind(updated_by) + .execute(&mut *tx) + .await?; + + // Commit transaction (this triggers NOTIFY via database trigger) + tx.commit().await?; + + info!( + "Updated configuration {}/{} by {}", + service_scope, config_key, updated_by + ); + + Ok(()) + } + + /// Lists all configurations for a service scope + /// + /// # Arguments + /// * `service_scope` - Service scope (None for all scopes) + pub async fn list_configs(&self, service_scope: Option<&str>) -> GatewayConfigResult> { + let configs = if let Some(scope) = service_scope { + sqlx::query_as::<_, ConfigItem>( + r#" + SELECT * FROM config_settings + WHERE service_scope = $1 AND is_active = TRUE + ORDER BY config_key + "#, + ) + .bind(scope) + .fetch_all(&*self.db_pool) + .await? + } else { + sqlx::query_as::<_, ConfigItem>( + r#" + SELECT * FROM config_settings + WHERE is_active = TRUE + ORDER BY service_scope, config_key + "#, + ) + .fetch_all(&*self.db_pool) + .await? + }; + + Ok(configs) + } + + /// Loads configuration from Postgre`SQL` database + async fn load_from_db( + &self, + service_scope: &str, + config_key: &str, + ) -> GatewayConfigResult { + let config = sqlx::query_as::<_, ConfigItem>( + r#" + SELECT * FROM config_settings + WHERE service_scope = $1 AND config_key = $2 AND is_active = TRUE + "#, + ) + .bind(service_scope) + .bind(config_key) + .fetch_optional(&*self.db_pool) + .await? + .ok_or_else(|| GatewayConfigError::NotFound { + service_scope: service_scope.to_string(), + key: config_key.to_string(), + })?; + + Ok(config) + } + + /// Retrieves configuration from Redis cache + async fn get_from_cache( + &self, + service_scope: &str, + config_key: &str, + ) -> GatewayConfigResult> { + let redis_key = format!("config:{}:{}", service_scope, config_key); + let mut redis = self.redis.write().await; + + let cached: Option = redis::cmd("GET") + .arg(&redis_key) + .query_async(&mut *redis) + .await + .map_err(GatewayConfigError::Redis)?; + + if let Some(cached_json) = cached { + let config: ConfigItem = serde_json::from_str(&cached_json)?; + Ok(Some(config)) + } else { + Ok(None) + } + } + + /// Stores configuration in Redis cache + async fn set_in_cache(&self, config: &ConfigItem) -> GatewayConfigResult<()> { + let redis_key = format!("config:{}:{}", config.service_scope, config.config_key); + let config_json = serde_json::to_string(config)?; + let mut redis = self.redis.write().await; + + // Set with 5-minute TTL + redis::cmd("SETEX") + .arg(&redis_key) + .arg(300) // 5 minutes + .arg(&config_json) + .query_async::<()>(&mut *redis) + .await + .map_err(GatewayConfigError::Redis)?; + + Ok(()) + } + + /// Invalidates Redis cache for a configuration + async fn invalidate_cache(&self, service_scope: &str, config_key: &str) -> GatewayConfigResult<()> { + let redis_key = format!("config:{}:{}", service_scope, config_key); + let mut redis = self.redis.write().await; + + redis::cmd("DEL") + .arg(&redis_key) + .query_async::<()>(&mut *redis) + .await + .map_err(GatewayConfigError::Redis)?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + + // Note: These tests require a running PostgreSQL and Redis instance + // They are integration tests and should be run with --ignored flag +} diff --git a/services/api/src/config/mod.rs b/services/api/src/config/mod.rs new file mode 100644 index 000000000..4b5136c3b --- /dev/null +++ b/services/api/src/config/mod.rs @@ -0,0 +1,13 @@ +//! Configuration management module with PostgreSQL hot-reload and validation + +pub mod authz; +pub mod endpoints; +pub mod manager; +pub mod validator; + +pub use endpoints::GatewayConfigService; +pub use manager::{ConfigItem, ConfigurationManager}; +pub use validator::{ConfigValidator, ValidationRules}; + +// Re-export RBAC types +pub use authz::{AuthzMetrics, AuthzService, PermissionResult}; diff --git a/services/api/src/config/validator.rs b/services/api/src/config/validator.rs new file mode 100644 index 000000000..08d4a57ee --- /dev/null +++ b/services/api/src/config/validator.rs @@ -0,0 +1,347 @@ +//! Configuration validation with type checking, range validation, and regex matching + +use crate::error::{GatewayConfigError, GatewayConfigResult}; +use regex::Regex; +use serde_json::Value; +use std::collections::HashMap; + +/// Configuration validator supporting type, range, regex, and enum validation +pub struct ConfigValidator { + /// Compiled regex patterns cache + regex_cache: HashMap, +} + +/// Validation rules from database +#[derive(Debug, Clone, serde::Deserialize)] +pub struct ValidationRules { + /// Minimum value for numbers + pub min: Option, + /// Maximum value for numbers + pub max: Option, + /// Minimum length for strings/arrays + pub min_len: Option, + /// Maximum length for strings/arrays + pub max_len: Option, + /// Regex pattern for strings + pub regex: Option, + /// Allowed enum values + #[serde(rename = "enum")] + pub enum_values: Option>, +} + +impl ConfigValidator { + /// Creates a new ConfigValidator + pub fn new() -> Self { + Self { + regex_cache: HashMap::new(), + } + } + + /// Validates a configuration value against its data type and validation rules + /// + /// # Arguments + /// * `value` - The configuration value to validate + /// + /// * `data_type` - Expected data type (string, integer, float, boolean, json, array) + /// * `rules` - Optional validation rules + /// + /// # Returns + /// + /// Ok(()) if validation passes, Err with descriptive message otherwise + pub fn validate( + &mut self, + value: &Value, + data_type: &str, + rules: Option<&Value>, + ) -> GatewayConfigResult<()> { + // Parse validation rules if provided + let validation_rules: Option = match rules { + Some(r) => serde_json::from_value(r.clone()).ok(), + None => None, + }; + + // Type validation + self.validate_type(value, data_type)?; + + // Additional validations based on rules + if let Some(rules) = validation_rules { + match data_type { + "integer" | "float" => self.validate_numeric(value, &rules)?, + "string" => self.validate_string(value, &rules)?, + "array" => self.validate_array(value, &rules)?, + _ => {}, + } + } + + Ok(()) + } + + /// Validates that the value matches the expected type + fn validate_type(&self, value: &Value, data_type: &str) -> GatewayConfigResult<()> { + let matches = match data_type { + "string" => value.is_string(), + "integer" => value.is_i64() || value.is_u64(), + "float" => value.is_f64() || value.is_i64() || value.is_u64(), + "boolean" => value.is_boolean(), + "json" => value.is_object(), + "array" => value.is_array(), + _ => { + return Err(GatewayConfigError::Validation(format!( + "Unknown data type: {}", + data_type + ))) + }, + }; + + if !matches { + return Err(GatewayConfigError::Validation(format!( + "Type mismatch: expected {}, got {}", + data_type, + value_type_name(value) + ))); + } + + Ok(()) + } + + /// Validates numeric values against min/max rules + fn validate_numeric(&self, value: &Value, rules: &ValidationRules) -> GatewayConfigResult<()> { + let num = value + .as_f64() + .ok_or_else(|| GatewayConfigError::Validation("Not a number".to_string()))?; + + if let Some(min) = rules.min { + if num < min { + return Err(GatewayConfigError::Validation(format!( + "Value {} is below minimum {}", + num, min + ))); + } + } + + if let Some(max) = rules.max { + if num > max { + return Err(GatewayConfigError::Validation(format!( + "Value {} exceeds maximum {}", + num, max + ))); + } + } + + Ok(()) + } + + /// Validates string values against length and regex rules + fn validate_string(&mut self, value: &Value, rules: &ValidationRules) -> GatewayConfigResult<()> { + let s = value + .as_str() + .ok_or_else(|| GatewayConfigError::Validation("Not a string".to_string()))?; + + // Length validation + if let Some(min_len) = rules.min_len { + if s.len() < min_len { + return Err(GatewayConfigError::Validation(format!( + "String length {} is below minimum {}", + s.len(), + min_len + ))); + } + } + + if let Some(max_len) = rules.max_len { + if s.len() > max_len { + return Err(GatewayConfigError::Validation(format!( + "String length {} exceeds maximum {}", + s.len(), + max_len + ))); + } + } + + // Regex validation + if let Some(pattern) = &rules.regex { + if !self.regex_cache.contains_key(pattern) { + let compiled = Regex::new(pattern).map_err(|e| { + GatewayConfigError::Validation(format!("Invalid regex pattern '{}': {}", pattern, e)) + })?; + self.regex_cache.insert(pattern.clone(), compiled); + } + let regex = self + .regex_cache + .get(pattern) + .ok_or_else(|| GatewayConfigError::Validation("Regex cache miss".to_owned()))?; + + if !regex.is_match(s) { + return Err(GatewayConfigError::Validation(format!( + "String '{}' does not match pattern '{}'", + s, pattern + ))); + } + } + + // Enum validation + if let Some(enum_values) = &rules.enum_values { + if !enum_values.contains(&s.to_string()) { + return Err(GatewayConfigError::Validation(format!( + "Value '{}' is not in allowed values: {:?}", + s, enum_values + ))); + } + } + + Ok(()) + } + + /// Validates array values against length rules + fn validate_array(&self, value: &Value, rules: &ValidationRules) -> GatewayConfigResult<()> { + let arr = value + .as_array() + .ok_or_else(|| GatewayConfigError::Validation("Not an array".to_string()))?; + + if let Some(min_len) = rules.min_len { + if arr.len() < min_len { + return Err(GatewayConfigError::Validation(format!( + "Array length {} is below minimum {}", + arr.len(), + min_len + ))); + } + } + + if let Some(max_len) = rules.max_len { + if arr.len() > max_len { + return Err(GatewayConfigError::Validation(format!( + "Array length {} exceeds maximum {}", + arr.len(), + max_len + ))); + } + } + + Ok(()) + } +} + +impl Default for ConfigValidator { + fn default() -> Self { + Self::new() + } +} + +/// Returns a human-readable name for a `JSON` value's type +fn value_type_name(value: &Value) -> &'static str { + match value { + Value::Null => "null", + Value::Bool(_) => "boolean", + Value::Number(_) => "number", + Value::String(_) => "string", + Value::Array(_) => "array", + Value::Object(_) => "object", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_validate_integer_type() { + let mut validator = ConfigValidator::new(); + assert!(validator.validate(&json!(42), "integer", None).is_ok()); + assert!(validator + .validate(&json!("not a number"), "integer", None) + .is_err()); + } + + #[test] + fn test_validate_float_type() { + let mut validator = ConfigValidator::new(); + assert!(validator.validate(&json!(3.14), "float", None).is_ok()); + assert!(validator.validate(&json!(42), "float", None).is_ok()); // Integers valid as floats + } + + #[test] + fn test_validate_string_type() { + let mut validator = ConfigValidator::new(); + assert!(validator.validate(&json!("hello"), "string", None).is_ok()); + assert!(validator.validate(&json!(123), "string", None).is_err()); + } + + #[test] + fn test_validate_numeric_range() { + let mut validator = ConfigValidator::new(); + let rules = json!({"min": 0.0, "max": 100.0}); + + assert!(validator + .validate(&json!(50.0), "float", Some(&rules)) + .is_ok()); + assert!(validator + .validate(&json!(-1.0), "float", Some(&rules)) + .is_err()); + assert!(validator + .validate(&json!(101.0), "float", Some(&rules)) + .is_err()); + } + + #[test] + fn test_validate_string_length() { + let mut validator = ConfigValidator::new(); + let rules = json!({"min_len": 3, "max_len": 10}); + + assert!(validator + .validate(&json!("hello"), "string", Some(&rules)) + .is_ok()); + assert!(validator + .validate(&json!("hi"), "string", Some(&rules)) + .is_err()); + assert!(validator + .validate(&json!("this is too long"), "string", Some(&rules)) + .is_err()); + } + + #[test] + fn test_validate_regex() { + let mut validator = ConfigValidator::new(); + let rules = json!({"regex": "^[A-Z]{3}$"}); + + assert!(validator + .validate(&json!("USD"), "string", Some(&rules)) + .is_ok()); + assert!(validator + .validate(&json!("usd"), "string", Some(&rules)) + .is_err()); + assert!(validator + .validate(&json!("US"), "string", Some(&rules)) + .is_err()); + } + + #[test] + fn test_validate_enum() { + let mut validator = ConfigValidator::new(); + let rules = json!({"enum": ["FIXED", "VOLUME_BASED", "SPREAD_BASED"]}); + + assert!(validator + .validate(&json!("FIXED"), "string", Some(&rules)) + .is_ok()); + assert!(validator + .validate(&json!("INVALID"), "string", Some(&rules)) + .is_err()); + } + + #[test] + fn test_validate_array_length() { + let mut validator = ConfigValidator::new(); + let rules = json!({"min_len": 2, "max_len": 5}); + + assert!(validator + .validate(&json!([1, 2, 3]), "array", Some(&rules)) + .is_ok()); + assert!(validator + .validate(&json!([1]), "array", Some(&rules)) + .is_err()); + assert!(validator + .validate(&json!([1, 2, 3, 4, 5, 6]), "array", Some(&rules)) + .is_err()); + } +} diff --git a/services/api/src/error.rs b/services/api/src/error.rs new file mode 100644 index 000000000..49fcaf524 --- /dev/null +++ b/services/api/src/error.rs @@ -0,0 +1,31 @@ +//! Error types for API Gateway configuration management + +use thiserror::Error; + +/// Configuration management errors +#[derive(Debug, Error)] +pub enum GatewayConfigError { + #[error("Database error: {0}")] + Database(#[from] sqlx::Error), + + #[error("Redis error: {0}")] + Redis(#[from] redis::RedisError), + + #[error("Validation error: {0}")] + Validation(String), + + #[error("Configuration not found: {service_scope}/{key}")] + NotFound { service_scope: String, key: String }, + + #[error("Serialization error: {0}")] + Serialization(#[from] serde_json::Error), + + #[error("Internal error: {0}")] + Internal(String), + + #[error("Unauthorized: {0}")] + Unauthorized(String), +} + +/// Result type for configuration operations +pub type GatewayConfigResult = Result; diff --git a/services/api/src/grpc/backtesting_proxy.rs b/services/api/src/grpc/backtesting_proxy.rs new file mode 100644 index 000000000..c837de5a5 --- /dev/null +++ b/services/api/src/grpc/backtesting_proxy.rs @@ -0,0 +1,582 @@ +//! Backtesting Service Proxy - Zero-copy gRPC forwarding +//! +//! This module implements a high-performance proxy for the backtesting service. +//! Design goals: +//! - <10μs routing overhead +//! - Zero-copy forwarding where possible +//! - Connection pooling via tonic::transport::Channel +//! - Circuit breaker for backend failures +//! - Health checking of backend service + +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::RwLock; +use tonic::transport::{Certificate, ClientTlsConfig, Identity}; +use tonic::{Request, Response, Status}; +use tonic_health::pb::health_client::HealthClient; +use tracing::{debug, error, info, warn, instrument}; + +// Import generated protobuf types from build.rs +use crate::foxhunt::tli::{ + backtesting_service_client::BacktestingServiceClient, + backtesting_service_server::BacktestingService, + BacktestProgressEvent, + GetBacktestResultsRequest, + GetBacktestResultsResponse, + GetBacktestStatusRequest, + GetBacktestStatusResponse, + ListBacktestsRequest, + ListBacktestsResponse, + // Request/Response types + StartBacktestRequest, + StartBacktestResponse, + StopBacktestRequest, + StopBacktestResponse, + SubscribeBacktestProgressRequest, +}; + +/// Health check state for circuit breaker +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HealthState { + Healthy, + Degraded, + Unhealthy, +} + +/// Health checker for backend service with circuit breaker +pub struct HealthChecker { + state: RwLock, + consecutive_failures: RwLock, + last_health_check: RwLock, + failure_threshold: u32, + health_check_interval: Duration, +} + +impl HealthChecker { + pub fn new(failure_threshold: u32, health_check_interval: Duration) -> Self { + Self { + state: RwLock::new(HealthState::Healthy), + consecutive_failures: RwLock::new(0), + last_health_check: RwLock::new(Instant::now()), + failure_threshold, + health_check_interval, + } + } + + /// Record a successful request + pub async fn record_success(&self) { + let mut failures = self.consecutive_failures.write().await; + *failures = 0; + + let mut state = self.state.write().await; + if *state != HealthState::Healthy { + info!("Backtesting service backend recovered to healthy state"); + *state = HealthState::Healthy; + } + } + + /// Record a failed request + pub async fn record_failure(&self) { + let mut failures = self.consecutive_failures.write().await; + *failures += 1; + + let mut state = self.state.write().await; + + if *failures >= self.failure_threshold && *state != HealthState::Unhealthy { + error!( + "Backtesting service backend marked as unhealthy after {} consecutive failures", + failures + ); + *state = HealthState::Unhealthy; + } else if *failures >= self.failure_threshold / 2 && *state == HealthState::Healthy { + warn!( + "Backtesting service backend degraded after {} failures", + failures + ); + *state = HealthState::Degraded; + } + } + + /// Check if backend is healthy enough to accept requests + pub async fn is_healthy(&self) -> bool { + let state = self.state.read().await; + *state != HealthState::Unhealthy + } + + /// Get current health state + pub async fn get_state(&self) -> HealthState { + *self.state.read().await + } + + /// Perform proactive gRPC health check using tonic-health + pub async fn perform_health_check(&self, channel: &tonic::transport::Channel) { + // Check if enough time has passed since last check + let now = Instant::now(); + let should_check = { + let last_check = self.last_health_check.read().await; + now.duration_since(*last_check) >= self.health_check_interval + }; + + if !should_check { + return; + } + + // Update last check time + { + let mut last_check = self.last_health_check.write().await; + *last_check = now; + } + + // Perform gRPC health check + let mut health_client = HealthClient::new(channel.clone()); + let request = tonic_health::pb::HealthCheckRequest { + service: "foxhunt.tli.BacktestingService".to_string(), + }; + + match health_client.check(request).await { + Ok(response) => { + let status = response.into_inner().status; + if status == tonic_health::pb::health_check_response::ServingStatus::Serving as i32 + { + self.record_success().await; + debug!("Backtesting service health check: SERVING"); + } else { + self.record_failure().await; + warn!( + "Backtesting service health check: NOT_SERVING (status: {})", + status + ); + } + }, + Err(e) => { + self.record_failure().await; + error!("Backtesting service health check failed: {}", e); + }, + } + } +} + +/// Backtesting service proxy with zero-copy forwarding +pub struct BacktestingServiceProxy { + /// Client connection to backend service + /// tonic::transport::Channel is cheap to clone and reuses connections + client: BacktestingServiceClient, + + /// Health checker with circuit breaker + health_checker: Arc, + + /// Backend service URL for logging + backend_url: String, + + /// Underlying channel for health checks + channel: tonic::transport::Channel, +} + +impl BacktestingServiceProxy { + /// Create a new backtesting service proxy with TLS/mTLS support + /// + /// # Arguments + /// * `backend_url` - URL of the backend backtesting service (e.g., "https://localhost:50053") + /// * `ca_cert_path` - Path to CA certificate for server verification + /// * `client_cert_path` - Path to client certificate for mTLS authentication + /// * `client_key_path` - Path to client private key for mTLS authentication + /// + /// # Returns + /// * `Result` - Proxy instance or connection error + pub async fn new( + backend_url: &str, + ca_cert_path: Option<&str>, + client_cert_path: Option<&str>, + client_key_path: Option<&str>, + ) -> Result> { + info!( + "Connecting to backtesting service backend at {}", + backend_url + ); + + // Establish connection to backend service + // tonic::transport::Channel automatically handles: + // - Connection pooling + // - Keep-alive + // - Load balancing (if multiple endpoints provided) + let mut endpoint = tonic::transport::Endpoint::from_shared(backend_url.to_string())? + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(30)) + .tcp_keepalive(Some(Duration::from_secs(60))) + .http2_keep_alive_interval(Duration::from_secs(30)) + .keep_alive_while_idle(true); + + // Configure TLS if certificate paths provided (HTTPS URLs) + if backend_url.starts_with("https://") { + match (ca_cert_path, client_cert_path, client_key_path) { + (Some(ca_path), Some(cert_path), Some(key_path)) => { + info!("Configuring TLS with mTLS (client certificates)"); + info!(" CA cert: {}", ca_path); + info!(" Client cert: {}", cert_path); + info!(" Client key: {}", key_path); + + // Load certificates from files + info!("Reading CA certificate..."); + let ca_pem = tokio::fs::read_to_string(ca_path).await.map_err(|e| { + error!("Failed to read CA certificate at {}: {}", ca_path, e); + format!("Failed to read CA certificate at {}: {}", ca_path, e) + })?; + info!("CA certificate loaded ({} bytes)", ca_pem.len()); + + info!("Reading client certificate..."); + let client_cert_pem = + tokio::fs::read_to_string(cert_path).await.map_err(|e| { + error!("Failed to read client certificate at {}: {}", cert_path, e); + format!("Failed to read client certificate at {}: {}", cert_path, e) + })?; + info!( + "Client certificate loaded ({} bytes)", + client_cert_pem.len() + ); + + info!("Reading client key..."); + let client_key_pem = + tokio::fs::read_to_string(key_path).await.map_err(|e| { + error!("Failed to read client key at {}: {}", key_path, e); + format!("Failed to read client key at {}: {}", key_path, e) + })?; + info!("Client key loaded ({} bytes)", client_key_pem.len()); + + // Create TLS configuration with mTLS (client authentication) + // Extract hostname from backend_url for SNI (Server Name Indication) + let hostname = if let Some(host) = backend_url.strip_prefix("https://") { + host.split(':').next().unwrap_or("backtesting_service") + } else { + "backtesting_service" + }; + + let tls_config = ClientTlsConfig::new() + .ca_certificate(Certificate::from_pem(&ca_pem)) + .identity(Identity::from_pem(&client_cert_pem, &client_key_pem)) + .domain_name(hostname); // Use actual hostname for SNI + + info!("TLS SNI hostname: {}", hostname); + + endpoint = endpoint.tls_config(tls_config).map_err(|e| { + error!("Failed to apply TLS configuration: {:?}", e); + error!(" TLS config error: {}", e); + e + })?; + info!("TLS configuration with mTLS applied successfully"); + }, + (Some(ca_path), None, None) => { + info!("Configuring TLS with server verification only (no client cert)"); + let ca_pem = tokio::fs::read_to_string(ca_path).await.map_err(|e| { + format!("Failed to read CA certificate at {}: {}", ca_path, e) + })?; + + let tls_config = ClientTlsConfig::new() + .ca_certificate(Certificate::from_pem(&ca_pem)) + .domain_name("foxhunt-services"); // Match server cert CN + + endpoint = endpoint.tls_config(tls_config)?; + info!("TLS configuration with server verification applied successfully"); + }, + _ => { + warn!( + "HTTPS URL provided but certificate paths incomplete - connection may fail" + ); + warn!( + " CA: {:?}, Client cert: {:?}, Client key: {:?}", + ca_cert_path, client_cert_path, client_key_path + ); + }, + } + } else { + info!("Using HTTP (no TLS) for backtesting service connection"); + } + + let channel = endpoint.connect().await.map_err(|e| { + error!("Failed to connect to backtesting service: {:?}", e); + error!(" Error details: {}", e); + e + })?; + + info!("Successfully established channel connection to backtesting service"); + + let client = BacktestingServiceClient::new(channel.clone()); + + // Initialize health checker with circuit breaker + let health_checker = Arc::new(HealthChecker::new( + 5_u32, // failure_threshold: 5 consecutive failures + Duration::from_secs(10), // health_check_interval: 10 seconds + )); + + info!("Successfully connected to backtesting service backend"); + + Ok(Self { + client, + health_checker, + backend_url: backend_url.to_string(), + channel, + }) + } + + /// Perform background health check (should be called periodically) + pub async fn background_health_check(&self) { + self.health_checker + .perform_health_check(&self.channel) + .await; + } + + /// Check if backend is currently healthy + pub async fn is_backend_healthy(&self) -> bool { + self.health_checker.is_healthy().await + } + + /// Forward request with health checking and latency tracking + async fn forward_with_health_check( + &self, + operation: &str, + forward_fn: F, + ) -> Result + where + F: FnOnce() -> Fut, + Fut: std::future::Future>, + { + // Check health before forwarding + if !self.health_checker.is_healthy().await { + return Err(Status::unavailable(format!( + "Backtesting service backend is unhealthy ({})", + self.backend_url + ))); + } + + // Track latency + let start = Instant::now(); + + // Forward request + let result = forward_fn().await; + + let elapsed = start.elapsed(); + + // Record health outcome + match &result { + Ok(_) => { + self.health_checker.record_success().await; + debug!( + operation = operation, + latency_us = elapsed.as_micros(), + "Forwarded request to backtesting service" + ); + }, + Err(status) => { + self.health_checker.record_failure().await; + error!( + operation = operation, + error = %status, + latency_us = elapsed.as_micros(), + "Failed to forward request to backtesting service" + ); + }, + } + + result + } +} + +#[tonic::async_trait] +impl BacktestingService for BacktestingServiceProxy { + // Define streaming response type for subscribe_backtest_progress + type SubscribeBacktestProgressStream = tonic::codec::Streaming; + + /// Forward start_backtest request to backend + #[instrument(skip_all)] + async fn start_backtest( + &self, + request: Request, + ) -> Result, Status> { + self.forward_with_health_check("start_backtest", || async { + // Clone the client (cheap operation, reuses connection) + let mut client = self.client.clone(); + + // Extract inner request and forward + let inner_request = request.into_inner(); + + // Forward to backend - zero additional allocations + client.start_backtest(inner_request).await + }) + .await + } + + /// Forward get_backtest_status request to backend + #[instrument(skip_all)] + async fn get_backtest_status( + &self, + request: Request, + ) -> Result, Status> { + self.forward_with_health_check("get_backtest_status", || async { + let mut client = self.client.clone(); + let inner_request = request.into_inner(); + client.get_backtest_status(inner_request).await + }) + .await + } + + /// Forward get_backtest_results request to backend + #[instrument(skip_all)] + async fn get_backtest_results( + &self, + request: Request, + ) -> Result, Status> { + self.forward_with_health_check("get_backtest_results", || async { + let mut client = self.client.clone(); + let inner_request = request.into_inner(); + client.get_backtest_results(inner_request).await + }) + .await + } + + /// Forward list_backtests request to backend + #[instrument(skip_all)] + async fn list_backtests( + &self, + request: Request, + ) -> Result, Status> { + self.forward_with_health_check("list_backtests", || async { + let mut client = self.client.clone(); + let inner_request = request.into_inner(); + client.list_backtests(inner_request).await + }) + .await + } + + /// Forward subscribe_backtest_progress streaming request to backend + #[instrument(skip_all)] + async fn subscribe_backtest_progress( + &self, + request: Request, + ) -> Result, Status> { + self.forward_with_health_check("subscribe_backtest_progress", || async { + let mut client = self.client.clone(); + let inner_request = request.into_inner(); + + // Forward streaming request - the response is already a stream + let response = client.subscribe_backtest_progress(inner_request).await?; + + // Extract the stream and pass it through + Ok(Response::new(response.into_inner())) + }) + .await + } + + /// Forward stop_backtest request to backend + #[instrument(skip_all)] + async fn stop_backtest( + &self, + request: Request, + ) -> Result, Status> { + self.forward_with_health_check("stop_backtest", || async { + let mut client = self.client.clone(); + let inner_request = request.into_inner(); + client.stop_backtest(inner_request).await + }) + .await + } +} + +// Implement BacktestingService for Arc to enable Arc wrapping +#[tonic::async_trait] +impl BacktestingService for Arc { + type SubscribeBacktestProgressStream = tonic::codec::Streaming; + + #[instrument(skip_all)] + async fn start_backtest( + &self, + request: Request, + ) -> Result, Status> { + self.as_ref().start_backtest(request).await + } + + #[instrument(skip_all)] + async fn get_backtest_status( + &self, + request: Request, + ) -> Result, Status> { + self.as_ref().get_backtest_status(request).await + } + + #[instrument(skip_all)] + async fn get_backtest_results( + &self, + request: Request, + ) -> Result, Status> { + self.as_ref().get_backtest_results(request).await + } + + #[instrument(skip_all)] + async fn list_backtests( + &self, + request: Request, + ) -> Result, Status> { + self.as_ref().list_backtests(request).await + } + + #[instrument(skip_all)] + async fn subscribe_backtest_progress( + &self, + request: Request, + ) -> Result, Status> { + self.as_ref().subscribe_backtest_progress(request).await + } + + #[instrument(skip_all)] + async fn stop_backtest( + &self, + request: Request, + ) -> Result, Status> { + self.as_ref().stop_backtest(request).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_health_checker_success() { + let checker = HealthChecker::new(3, Duration::from_secs(10)); + + assert_eq!(checker.get_state().await, HealthState::Healthy); + assert!(checker.is_healthy().await); + + checker.record_success().await; + assert_eq!(checker.get_state().await, HealthState::Healthy); + } + + #[tokio::test] + async fn test_health_checker_failure() { + let checker = HealthChecker::new(3, Duration::from_secs(10)); + + checker.record_failure().await; + assert_eq!(checker.get_state().await, HealthState::Degraded); + + checker.record_failure().await; + assert_eq!(checker.get_state().await, HealthState::Degraded); + + checker.record_failure().await; + assert_eq!(checker.get_state().await, HealthState::Unhealthy); + assert!(!checker.is_healthy().await); + } + + #[tokio::test] + async fn test_health_checker_recovery() { + let checker = HealthChecker::new(3, Duration::from_secs(10)); + + // Mark as unhealthy + for _ in 0..3 { + checker.record_failure().await; + } + assert_eq!(checker.get_state().await, HealthState::Unhealthy); + + // Recovery + checker.record_success().await; + assert_eq!(checker.get_state().await, HealthState::Healthy); + assert!(checker.is_healthy().await); + } +} diff --git a/services/api/src/grpc/backtesting_proxy_bench.rs b/services/api/src/grpc/backtesting_proxy_bench.rs new file mode 100644 index 000000000..43122c117 --- /dev/null +++ b/services/api/src/grpc/backtesting_proxy_bench.rs @@ -0,0 +1,41 @@ +//! Backtesting proxy latency benchmark +//! +//! Measures routing overhead to confirm <10μs target + +#[cfg(test)] +mod benchmarks { + use super::super::BacktestingServiceProxy; + use std::time::Instant; + + #[tokio::test] + #[ignore = "Only run when backend is available"] + async fn benchmark_routing_latency() { + // This test requires a running backtesting service at localhost:50052 + let proxy = BacktestingServiceProxy::new("http://localhost:50052", None, None, None) + .await + .expect("Failed to create proxy"); + + // Warmup + for _ in 0..100 { + let _health = proxy.health_checker.is_healthy().await; + } + + // Measure + let iterations = 10000; + let start = Instant::now(); + + for _ in 0..iterations { + let _health = proxy.health_checker.is_healthy().await; + } + + let elapsed = start.elapsed(); + let avg_us = elapsed.as_micros() as f64 / iterations as f64; + + println!("Average routing overhead: {:.2}μs", avg_us); + println!("Target: <10μs"); + + // This is just health checking, actual routing will be slightly higher + // but should still be well under 10μs + assert!(avg_us < 1.0, "Health check overhead too high: {:.2}μs", avg_us); + } +} diff --git a/services/api/src/grpc/broker_gateway_proxy.rs b/services/api/src/grpc/broker_gateway_proxy.rs new file mode 100644 index 000000000..258fabe6e --- /dev/null +++ b/services/api/src/grpc/broker_gateway_proxy.rs @@ -0,0 +1,197 @@ +//! Broker Gateway Service Proxy - Zero-copy gRPC forwarding for broker_gateway.BrokerGatewayService +//! +//! Forwards to broker-gateway:50056 (separate service). + +use futures::Stream; +use std::pin::Pin; +use std::time::Duration; +use tonic::{Request, Response, Status}; +use tracing::{error, instrument}; + +use crate::broker_gateway::broker_gateway_service_client::BrokerGatewayServiceClient; +use crate::broker_gateway::broker_gateway_service_server::BrokerGatewayService; +use crate::broker_gateway::{ + CancelOrderRequest, CancelOrderResponse, ExecutionEvent, GetAccountStateRequest, + GetAccountStateResponse, GetPositionsRequest, GetPositionsResponse, + GetSessionStatusRequest, GetSessionStatusResponse, HealthCheckRequest, HealthCheckResponse, + RouteOrderRequest, RouteOrderResponse, StreamAccountStateRequest, StreamExecutionsRequest, + StreamSessionStatusRequest, +}; + +#[derive(Debug, Clone)] +pub struct BrokerGatewayProxy { + client: BrokerGatewayServiceClient, +} + +impl BrokerGatewayProxy { + pub fn new(client: BrokerGatewayServiceClient) -> Self { + Self { client } + } +} + +#[tonic::async_trait] +impl BrokerGatewayService for BrokerGatewayProxy { + type StreamExecutionsStream = + Pin> + Send>>; + + #[instrument(skip(self, request), err)] + async fn route_order( + &self, + request: Request, + ) -> Result, Status> { + self.client.clone().route_order(request).await.map_err(|e| { + error!("Backend RouteOrder failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn cancel_order( + &self, + request: Request, + ) -> Result, Status> { + self.client.clone().cancel_order(request).await.map_err(|e| { + error!("Backend CancelOrder failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_account_state( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_account_state(request) + .await + .map_err(|e| { + error!("Backend GetAccountState failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_positions( + &self, + request: Request, + ) -> Result, Status> { + self.client.clone().get_positions(request).await.map_err(|e| { + error!("Backend GetPositions failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_session_status( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_session_status(request) + .await + .map_err(|e| { + error!("Backend GetSessionStatus failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn stream_executions( + &self, + request: Request, + ) -> Result, Status> { + let stream = self + .client + .clone() + .stream_executions(request) + .await + .map_err(|e| { + error!("Backend StreamExecutions failed: {}", e); + e + })?; + Ok(Response::new(Box::pin(stream.into_inner()))) + } + + #[instrument(skip(self, request), err)] + async fn health_check( + &self, + request: Request, + ) -> Result, Status> { + self.client.clone().health_check(request).await.map_err(|e| { + error!("Backend HealthCheck failed: {}", e); + e + }) + } + + type StreamAccountStateStream = + Pin> + Send>>; + + #[instrument(skip(self, request), err)] + async fn stream_account_state( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let interval_secs = if req.interval_seconds == 0 { + 3 + } else { + req.interval_seconds.clamp(1, 60) + }; + let mut client = self.client.clone(); + + let stream = async_stream::stream! { + let mut interval = tokio::time::interval(Duration::from_secs(u64::from(interval_secs))); + loop { + interval.tick().await; + match client.get_account_state(Request::new(GetAccountStateRequest { + account_id: String::new(), + })).await { + Ok(resp) => yield Ok(resp.into_inner()), + Err(e) => { + error!("Backend GetAccountState failed: {}", e); + yield Err(e); + break; + } + } + } + }; + + Ok(Response::new(Box::pin(stream))) + } + + type StreamSessionStatusStream = + Pin> + Send>>; + + #[instrument(skip(self, request), err)] + async fn stream_session_status( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let interval_secs = if req.interval_seconds == 0 { + 5 + } else { + req.interval_seconds.clamp(1, 60) + }; + let mut client = self.client.clone(); + + let stream = async_stream::stream! { + let mut interval = tokio::time::interval(Duration::from_secs(u64::from(interval_secs))); + loop { + interval.tick().await; + match client.get_session_status(Request::new(GetSessionStatusRequest::default())).await { + Ok(resp) => yield Ok(resp.into_inner()), + Err(e) => { + error!("Backend GetSessionStatus failed: {}", e); + yield Err(e); + break; + } + } + } + }; + + Ok(Response::new(Box::pin(stream))) + } +} diff --git a/services/api/src/grpc/config_proxy.rs b/services/api/src/grpc/config_proxy.rs new file mode 100644 index 000000000..3f6069504 --- /dev/null +++ b/services/api/src/grpc/config_proxy.rs @@ -0,0 +1,220 @@ +//! Config Service Proxy - Zero-copy gRPC forwarding for config.ConfigService +//! +//! Forwards to trading-service backend (config runs in the same process). + +use futures::Stream; +use std::pin::Pin; +use tonic::{Request, Response, Status}; +use tracing::{error, instrument}; + +use crate::config_backend::config_service_client::ConfigServiceClient; +use crate::config_backend::config_service_server::ConfigService; +use crate::config_backend::{ + ConfigChangeEvent, DeleteConfigurationRequest, DeleteConfigurationResponse, + ExportConfigurationRequest, ExportConfigurationResponse, GetConfigSchemaRequest, + GetConfigSchemaResponse, GetConfigurationHistoryRequest, GetConfigurationHistoryResponse, + GetConfigurationRequest, GetConfigurationResponse, ImportConfigurationRequest, + ImportConfigurationResponse, ListCategoriesRequest, ListCategoriesResponse, + RollbackConfigurationRequest, RollbackConfigurationResponse, StreamConfigChangesRequest, + UpdateConfigSchemaRequest, UpdateConfigSchemaResponse, UpdateConfigurationRequest, + UpdateConfigurationResponse, ValidateConfigurationRequest, ValidateConfigurationResponse, +}; + +#[derive(Debug, Clone)] +pub struct ConfigServiceProxy { + client: ConfigServiceClient, +} + +impl ConfigServiceProxy { + pub fn new(client: ConfigServiceClient) -> Self { + Self { client } + } +} + +#[tonic::async_trait] +impl ConfigService for ConfigServiceProxy { + type StreamConfigChangesStream = + Pin> + Send>>; + + #[instrument(skip(self, request), err)] + async fn get_configuration( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_configuration(request) + .await + .map_err(|e| { + error!("Backend GetConfiguration failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn update_configuration( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .update_configuration(request) + .await + .map_err(|e| { + error!("Backend UpdateConfiguration failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn delete_configuration( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .delete_configuration(request) + .await + .map_err(|e| { + error!("Backend DeleteConfiguration failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn list_categories( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .list_categories(request) + .await + .map_err(|e| { + error!("Backend ListCategories failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn stream_config_changes( + &self, + request: Request, + ) -> Result, Status> { + let stream = self + .client + .clone() + .stream_config_changes(request) + .await + .map_err(|e| { + error!("Backend StreamConfigChanges failed: {}", e); + e + })?; + Ok(Response::new(Box::pin(stream.into_inner()))) + } + + #[instrument(skip(self, request), err)] + async fn validate_configuration( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .validate_configuration(request) + .await + .map_err(|e| { + error!("Backend ValidateConfiguration failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_configuration_history( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_configuration_history(request) + .await + .map_err(|e| { + error!("Backend GetConfigurationHistory failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn rollback_configuration( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .rollback_configuration(request) + .await + .map_err(|e| { + error!("Backend RollbackConfiguration failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn export_configuration( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .export_configuration(request) + .await + .map_err(|e| { + error!("Backend ExportConfiguration failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn import_configuration( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .import_configuration(request) + .await + .map_err(|e| { + error!("Backend ImportConfiguration failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_config_schema( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_config_schema(request) + .await + .map_err(|e| { + error!("Backend GetConfigSchema failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn update_config_schema( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .update_config_schema(request) + .await + .map_err(|e| { + error!("Backend UpdateConfigSchema failed: {}", e); + e + }) + } +} diff --git a/services/api/src/grpc/data_acquisition_proxy.rs b/services/api/src/grpc/data_acquisition_proxy.rs new file mode 100644 index 000000000..f892b8a4a --- /dev/null +++ b/services/api/src/grpc/data_acquisition_proxy.rs @@ -0,0 +1,138 @@ +//! Data Acquisition Service Proxy - Zero-copy gRPC forwarding +//! +//! Forwards to data-acquisition-service:50057 (separate service). +//! All RPCs are unary (no streaming) — simplest proxy. + +use futures::Stream; +use std::pin::Pin; +use std::time::Duration; +use tonic::{Request, Response, Status}; +use tracing::{error, instrument}; + +use crate::data_acquisition::data_acquisition_service_client::DataAcquisitionServiceClient; +use crate::data_acquisition::data_acquisition_service_server::DataAcquisitionService; +use crate::data_acquisition::{ + CancelDownloadRequest, CancelDownloadResponse, GetDownloadStatusRequest, + GetDownloadStatusResponse, HealthCheckRequest, HealthCheckResponse, + ListDownloadJobsRequest, ListDownloadJobsResponse, ScheduleDownloadRequest, + ScheduleDownloadResponse, StreamDownloadStatusRequest, +}; + +#[derive(Debug, Clone)] +pub struct DataAcquisitionProxy { + client: DataAcquisitionServiceClient, +} + +impl DataAcquisitionProxy { + pub fn new(client: DataAcquisitionServiceClient) -> Self { + Self { client } + } +} + +#[tonic::async_trait] +impl DataAcquisitionService for DataAcquisitionProxy { + #[instrument(skip(self, request), err)] + async fn schedule_download( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .schedule_download(request) + .await + .map_err(|e| { + error!("Backend ScheduleDownload failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_download_status( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_download_status(request) + .await + .map_err(|e| { + error!("Backend GetDownloadStatus failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn cancel_download( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .cancel_download(request) + .await + .map_err(|e| { + error!("Backend CancelDownload failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn list_download_jobs( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .list_download_jobs(request) + .await + .map_err(|e| { + error!("Backend ListDownloadJobs failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn health_check( + &self, + request: Request, + ) -> Result, Status> { + self.client.clone().health_check(request).await.map_err(|e| { + error!("Backend HealthCheck failed: {}", e); + e + }) + } + + type StreamDownloadStatusStream = + Pin> + Send>>; + + #[instrument(skip(self, request), err)] + async fn stream_download_status( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let interval_secs = if req.interval_seconds == 0 { + 5 + } else { + req.interval_seconds.clamp(1, 60) + }; + let mut client = self.client.clone(); + + let stream = async_stream::stream! { + let mut interval = tokio::time::interval(Duration::from_secs(u64::from(interval_secs))); + loop { + interval.tick().await; + match client.list_download_jobs(Request::new(ListDownloadJobsRequest::default())).await { + Ok(resp) => yield Ok(resp.into_inner()), + Err(e) => { + error!("Backend ListDownloadJobs failed: {}", e); + yield Err(e); + break; + } + } + } + }; + + Ok(Response::new(Box::pin(stream))) + } +} diff --git a/services/api/src/grpc/ml_proxy.rs b/services/api/src/grpc/ml_proxy.rs new file mode 100644 index 000000000..224eef24c --- /dev/null +++ b/services/api/src/grpc/ml_proxy.rs @@ -0,0 +1,230 @@ +//! ML Inference Service Proxy - Zero-copy gRPC forwarding for ml.MLService +//! +//! Forwards to trading-service backend (ML inference lives in same process). + +use futures::Stream; +use std::pin::Pin; +use std::time::Duration; +use tonic::{Request, Response, Status}; +use tracing::{error, instrument}; + +use crate::ml_inference::ml_service_client::MlServiceClient; +use crate::ml_inference::ml_service_server::MlService; +use crate::ml_inference::{ + GetAvailableModelsRequest, GetAvailableModelsResponse, GetEnsembleVoteRequest, + GetEnsembleVoteResponse, GetFeatureImportanceRequest, GetFeatureImportanceResponse, + GetModelPerformanceRequest, GetModelPerformanceResponse, GetModelStatusRequest, + GetModelStatusResponse, GetPredictionRequest, GetPredictionResponse, ModelMetricsEvent, + PredictionEvent, RetrainModelRequest, RetrainModelResponse, SignalStrengthEvent, + StreamModelMetricsRequest, StreamModelStatusRequest, StreamPredictionsRequest, + StreamSignalStrengthRequest, +}; + +#[derive(Debug, Clone)] +pub struct MlServiceProxy { + client: MlServiceClient, +} + +impl MlServiceProxy { + pub fn new(client: MlServiceClient) -> Self { + Self { client } + } +} + +#[tonic::async_trait] +impl MlService for MlServiceProxy { + type StreamPredictionsStream = + Pin> + Send>>; + type StreamModelMetricsStream = + Pin> + Send>>; + type StreamSignalStrengthStream = + Pin> + Send>>; + type StreamModelStatusStream = + Pin> + Send>>; + + #[instrument(skip(self, request), err)] + async fn get_prediction( + &self, + request: Request, + ) -> Result, Status> { + self.client.clone().get_prediction(request).await.map_err(|e| { + error!("Backend GetPrediction failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn stream_predictions( + &self, + request: Request, + ) -> Result, Status> { + let stream = self + .client + .clone() + .stream_predictions(request) + .await + .map_err(|e| { + error!("Backend StreamPredictions failed: {}", e); + e + })?; + Ok(Response::new(Box::pin(stream.into_inner()))) + } + + #[instrument(skip(self, request), err)] + async fn get_ensemble_vote( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_ensemble_vote(request) + .await + .map_err(|e| { + error!("Backend GetEnsembleVote failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_model_status( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_model_status(request) + .await + .map_err(|e| { + error!("Backend GetModelStatus failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_available_models( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_available_models(request) + .await + .map_err(|e| { + error!("Backend GetAvailableModels failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn retrain_model( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .retrain_model(request) + .await + .map_err(|e| { + error!("Backend RetrainModel failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_model_performance( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_model_performance(request) + .await + .map_err(|e| { + error!("Backend GetModelPerformance failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn stream_model_metrics( + &self, + request: Request, + ) -> Result, Status> { + let stream = self + .client + .clone() + .stream_model_metrics(request) + .await + .map_err(|e| { + error!("Backend StreamModelMetrics failed: {}", e); + e + })?; + Ok(Response::new(Box::pin(stream.into_inner()))) + } + + #[instrument(skip(self, request), err)] + async fn get_feature_importance( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_feature_importance(request) + .await + .map_err(|e| { + error!("Backend GetFeatureImportance failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn stream_signal_strength( + &self, + request: Request, + ) -> Result, Status> { + let stream = self + .client + .clone() + .stream_signal_strength(request) + .await + .map_err(|e| { + error!("Backend StreamSignalStrength failed: {}", e); + e + })?; + Ok(Response::new(Box::pin(stream.into_inner()))) + } + + #[instrument(skip(self, request), err)] + async fn stream_model_status( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let interval_secs = if req.interval_seconds == 0 { + 5 + } else { + req.interval_seconds.clamp(1, 60) + }; + let model_name = req.model_name; + let mut client = self.client.clone(); + + let stream = async_stream::stream! { + let mut interval = tokio::time::interval(Duration::from_secs(u64::from(interval_secs))); + loop { + interval.tick().await; + match client.get_model_status(Request::new(GetModelStatusRequest { + model_name: if model_name.is_empty() { None } else { Some(model_name.clone()) }, + })).await { + Ok(resp) => yield Ok(resp.into_inner()), + Err(e) => { + error!("Backend GetModelStatus failed: {}", e); + yield Err(e); + break; + } + } + } + }; + + Ok(Response::new(Box::pin(stream))) + } +} diff --git a/services/api/src/grpc/ml_trading_proxy.rs b/services/api/src/grpc/ml_trading_proxy.rs new file mode 100644 index 000000000..1042ee639 --- /dev/null +++ b/services/api/src/grpc/ml_trading_proxy.rs @@ -0,0 +1,483 @@ +//! ML Trading Proxy - Zero-copy gRPC forwarding for ML-based trading operations +//! +//! This module implements a high-performance proxy for ML-specific trading operations: +//! - SubmitMLOrder: Submit orders based on ensemble ML predictions +//! - GetMLPredictions: Query historical ML prediction performance +//! - GetMLPerformance: Get ML model performance metrics +//! +//! Architecture: +//! - Zero-copy message forwarding (routing overhead <10μs) +//! - Connection pooling via tonic::transport::Channel +//! - Circuit breaker integration for backend failures +//! - Health checking integration +//! +//! Security: +//! - Permission checks: "trading.submit" for SubmitMLOrder +//! - Permission checks: "trading.view" for read operations +//! - Rate limiting: 100 requests/minute for GetMLPredictions, 20 requests/minute for GetMLPerformance +//! - Audit logging for all operations + +use chrono::Utc; +use serde_json::json; +use std::sync::Arc; +use tonic::{Request, Response, Status}; +use tracing::{error, info, instrument, warn}; + +// Import authentication components +use crate::auth::interceptor::JwtClaims; + +// Import rate limiting components +use governor::{ + clock::DefaultClock, state::keyed::DefaultKeyedStateStore, Quota, + RateLimiter as GovernorRateLimiter, +}; +use std::num::NonZeroU32; + +// Import the Trading Service backend proto (where ML methods are defined) +use crate::trading_backend::trading_service_client::TradingServiceClient; +use crate::trading_backend::{ + MlOrderRequest, MlOrderResponse, MlPerformanceRequest, MlPerformanceResponse, + MlPredictionsRequest, MlPredictionsResponse, +}; + +/// ML Trading Proxy +/// +/// Provides zero-copy forwarding of ML trading requests to the backend Trading Service. +/// +/// The Trading Service handles: +/// - Ensemble ML prediction aggregation (DQN, MAMBA-2, PPO, TFT) +/// - Order execution based on ML signals +/// - ML prediction tracking and performance analysis +/// +/// # Performance +/// - Uses connection pooling and circuit breakers for high availability +/// - Target routing overhead: <10μs per request +#[derive(Clone)] +pub struct MlTradingProxy { + /// Backend Trading Service client with connection pooling + client: TradingServiceClient, + /// Rate limiter: 100 requests/minute per user for GetMLPredictions + rate_limiter_predictions: + Arc, DefaultClock>>, + /// Rate limiter: 20 requests/minute per user for GetMLPerformance (expensive queries) + rate_limiter_performance: + Arc, DefaultClock>>, +} + +impl MlTradingProxy { + /// Create a new ML Trading proxy + /// + /// # Arguments + /// * `client` - Pre-configured Trading Service client with circuit breaker + /// + /// # Performance + /// - Uses Arc-based channel cloning for zero-copy client reuse + /// - Connection pooling managed by tonic::transport::Channel + pub fn new(client: TradingServiceClient) -> Self { + // Create rate limiter for predictions: 100 requests per minute per user + // SAFETY: NonZeroU32::new on positive constants always returns Some + #[allow(clippy::unwrap_used)] + let quota_predictions = Quota::per_minute(NonZeroU32::new(100).unwrap()); + let rate_limiter_predictions = Arc::new(GovernorRateLimiter::keyed(quota_predictions)); + + // Create rate limiter for performance queries: 20 requests per minute per user (expensive) + // SAFETY: NonZeroU32::new on positive constants always returns Some + #[allow(clippy::unwrap_used)] + let quota_performance = Quota::per_minute(NonZeroU32::new(20).unwrap()); + let rate_limiter_performance = Arc::new(GovernorRateLimiter::keyed(quota_performance)); + + Self { + client, + rate_limiter_predictions, + rate_limiter_performance, + } + } + + /// Submit ML-generated trading order with ensemble predictions + /// + /// # Security + /// - Requires "trading.submit" permission (validated by auth interceptor) + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + /// + /// # Flow + /// 1. Receives MLOrderRequest with features and model selection + /// 2. Forwards to Trading Service + /// 3. Trading Service: + /// - Runs ensemble prediction (or specific model) + /// - Executes trading logic (BUY/SELL/HOLD) + /// - Records prediction in ensemble_predictions table + /// - Submits order if action is BUY/SELL + /// 4. Returns order_id, prediction_id, action, confidence + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + pub async fn submit_ml_order( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying SubmitMLOrder request"); + + // Clone client (cheap Arc increment) for concurrent request handling + let mut client = self.client.clone(); + + // Forward request with zero-copy + let response = client.submit_ml_order(request).await.map_err(|e| { + error!("Backend SubmitMLOrder failed: {}", e); + e + })?; + + info!("SubmitMLOrder request forwarded successfully"); + Ok(response) + } + + /// Get ML prediction history with outcomes + /// + /// # Security + /// - Requires "trading.view" permission (validated by caller) + /// - Rate limit: 100 requests/minute per user + /// + /// # Validation + /// - symbol: required, must be valid format (alphanumeric + dots) + /// - model_filter: optional, must be in [DQN, MAMBA2, PPO, TFT, TLOB, Liquid] + /// - limit: optional, default 10, max 100 + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + /// + /// # Returns + /// List of ML predictions with: + /// - Ensemble voting results (action, signal, confidence) + /// - Individual model predictions (DQN, MAMBA-2, PPO, TFT) + /// - Actual P&L if order was executed and filled + /// - Order ID linkage + #[instrument(skip(self, request, claims), fields(request_id = %uuid::Uuid::new_v4(), user = %claims.sub), err)] + pub async fn get_ml_predictions( + &self, + request: Request, + claims: &JwtClaims, + ) -> Result, Status> { + info!( + "Processing GetMLPredictions request for user: {}", + claims.sub + ); + + // Step 1: Check rate limit (100 requests/minute per user) + if self + .rate_limiter_predictions + .check_key(&claims.sub) + .is_err() + { + warn!( + "Rate limit exceeded for user {} on GetMLPredictions", + claims.sub + ); + return Err(Status::resource_exhausted( + "Rate limit exceeded: maximum 100 requests per minute for ML predictions queries", + )); + } + + // Step 2: Validate permission (requires "trading.view" scope) + if !claims.permissions.contains(&"trading.view".to_string()) { + warn!( + "Permission denied: user {} lacks 'trading.view' scope for GetMLPredictions", + claims.sub + ); + return Err(Status::permission_denied( + "Insufficient permissions: 'trading.view' scope required", + )); + } + + // Step 3: Extract and validate request parameters + let req_inner = request.into_inner(); + let symbol = req_inner.symbol.trim(); + let model_filter = req_inner.model_name.as_deref(); + let limit = if req_inner.limit == 0 { + 10 + } else { + req_inner.limit + }; + + // Validate symbol (required, must be alphanumeric + dots) + if symbol.is_empty() { + return Err(Status::invalid_argument( + "Symbol is required and cannot be empty", + )); + } + if !symbol.chars().all(|c| c.is_alphanumeric() || c == '.') { + return Err(Status::invalid_argument(format!( + "Invalid symbol format: '{}' (must be alphanumeric with optional dots)", + symbol + ))); + } + + // Validate model_filter (optional, must be valid model name) + if let Some(model) = model_filter { + let valid_models = ["DQN", "MAMBA2", "PPO", "TFT", "TLOB", "Liquid"]; + if !valid_models.contains(&model) { + return Err(Status::invalid_argument(format!( + "Invalid model_filter: '{}' (must be one of: {})", + model, + valid_models.join(", ") + ))); + } + } + + // Validate limit (default 10, max 100) + if limit < 1 { + return Err(Status::invalid_argument("Limit must be at least 1")); + } + if limit > 100 { + return Err(Status::invalid_argument( + "Limit cannot exceed 100 (maximum predictions per query)", + )); + } + + info!( + "Validated request: symbol={}, model_filter={:?}, limit={}", + symbol, model_filter, limit + ); + + // Step 4: Forward request to Trading Service + let mut client = self.client.clone(); + let backend_request = Request::new(MlPredictionsRequest { + symbol: symbol.to_string(), + model_name: model_filter.map(|s| s.to_string()), + limit, + start_time: None, + end_time: None, + }); + + let response = client + .get_ml_predictions(backend_request) + .await + .map_err(|e| { + error!("Backend GetMLPredictions failed: {}", e); + + // Map backend errors to appropriate status codes + match e.code() { + tonic::Code::Unavailable => Status::unavailable( + "Trading Service temporarily unavailable - please retry", + ), + tonic::Code::NotFound => { + Status::not_found(format!("No predictions found for symbol: {}", symbol)) + }, + tonic::Code::Internal => { + Status::internal("Database error occurred while retrieving predictions") + }, + _ => e, + } + })?; + + let results_count = response.get_ref().predictions.len(); + info!( + "GetMLPredictions successful: symbol={}, results_count={}", + symbol, results_count + ); + + // Step 5: Audit log the query (non-blocking) + let audit_log = json!({ + "action": "get_ml_predictions", + "user": claims.sub, + "symbol": symbol, + "model_filter": model_filter, + "limit": limit, + "results_count": results_count, + "timestamp": Utc::now().to_rfc3339(), + }); + info!("Audit: {}", audit_log); + + // Step 6: Return predictions + Ok(response) + } + + /// Get ML model performance metrics + /// + /// # Security + /// - Requires "trading.view" permission (validated by caller) + /// - Rate limit: 20 requests/minute per user (performance queries are expensive) + /// + /// # Validation + /// - model_name: optional, must be in [DQN, MAMBA_2, PPO, TFT] + /// - time_range: start_time must be before end_time + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + /// - Response caching: 60 seconds (expensive queries) + /// + /// # Returns + /// Performance metrics per model: + /// - Total predictions made + /// - Accuracy (correct/total) + /// - Sharpe ratio (risk-adjusted returns) + /// - Average P&L per prediction + /// + /// # Filters + /// - By model name (optional): DQN, MAMBA_2, PPO, TFT + /// - By time range (optional): start_time, end_time + /// + /// # Audit Logging + /// All performance queries are logged for security and compliance monitoring. + /// Performance queries are sensitive as they reveal ML model effectiveness. + #[instrument(skip(self, request, claims), fields( + request_id = %uuid::Uuid::new_v4(), + user = %claims.sub, + model_filter = ?request.get_ref().model_name + ), err)] + pub async fn get_ml_performance( + &self, + request: Request, + claims: &JwtClaims, + ) -> Result, Status> { + info!( + "Processing GetMLPerformance request for user: {}", + claims.sub + ); + + // Step 1: Check rate limit (20 requests/minute - performance queries are expensive) + if self + .rate_limiter_performance + .check_key(&claims.sub) + .is_err() + { + warn!( + "Rate limit exceeded for user {} on GetMLPerformance", + claims.sub + ); + return Err(Status::resource_exhausted( + "Rate limit exceeded: maximum 20 requests per minute for ML performance queries (expensive operation)" + )); + } + + // Step 2: Validate permission (requires "trading.view" scope) + if !claims.permissions.contains(&"trading.view".to_string()) { + warn!( + "Permission denied: user {} lacks 'trading.view' scope for GetMLPerformance", + claims.sub + ); + return Err(Status::permission_denied( + "Insufficient permissions: 'trading.view' scope required", + )); + } + + // Step 3: Extract and validate request parameters + let req_inner = request.into_inner(); + let model_filter = req_inner.model_name.as_deref(); + let start_time = req_inner.start_time; + let end_time = req_inner.end_time; + + // Validate model_name (optional, must be valid model ID) + if let Some(model) = model_filter { + let valid_models = ["DQN", "MAMBA_2", "PPO", "TFT"]; + if !valid_models.contains(&model) { + warn!( + "Invalid model_name provided: {} (user: {})", + model, claims.sub + ); + return Err(Status::invalid_argument(format!( + "Invalid model name: '{}' (must be one of: {})", + model, + valid_models.join(", ") + ))); + } + } + + // Validate time range (start_time must be before end_time) + if let (Some(start), Some(end)) = (start_time, end_time) { + if start > end { + warn!( + "Invalid time range: start={}, end={} (user: {})", + start, end, claims.sub + ); + return Err(Status::invalid_argument(format!( + "Invalid time range: start_time ({}) must be before end_time ({})", + start, end + ))); + } + } + + info!( + "Validated request: model_filter={:?}, time_range=({:?}, {:?})", + model_filter, start_time, end_time + ); + + // Step 4: Forward request to Trading Service + let mut client = self.client.clone(); + let backend_request = Request::new(MlPerformanceRequest { + model_name: model_filter.map(|s| s.to_string()), + start_time, + end_time, + }); + + let response = client + .get_ml_performance(backend_request) + .await + .map_err(|e| { + error!("Backend GetMLPerformance failed: {}", e); + + // Map backend errors to appropriate status codes + match e.code() { + tonic::Code::Unavailable => Status::unavailable( + "Trading Service temporarily unavailable - please retry", + ), + tonic::Code::NotFound => { + Status::not_found("No performance data available for the specified filters") + }, + tonic::Code::Internal => Status::internal( + "Database error occurred while retrieving performance metrics", + ), + _ => e, + } + })?; + + let models_count = response.get_ref().models.len(); + info!("GetMLPerformance successful: models_count={}", models_count); + + // Step 5: Audit log the query (performance queries are sensitive) + // Log aggregated metrics for security monitoring + let audit_log = json!({ + "action": "get_ml_performance", + "user": claims.sub, + "model_filter": model_filter, + "time_range": { + "start": start_time, + "end": end_time + }, + "results": { + "models_count": models_count, + "model_names": response.get_ref().models.iter().map(|m| &m.model_name).collect::>() + }, + "timestamp": Utc::now().to_rfc3339(), + }); + info!("Audit: {}", audit_log); + + // Note: Response caching (60 seconds) would be implemented at a higher layer + // (e.g., nginx/envoy proxy) to avoid adding Redis dependency to this proxy layer. + // Cache key format: ml_performance:{model_filter}:{timestamp_minute} + // This keeps the proxy layer lightweight and focused on routing/validation. + + // Step 6: Return performance metrics + Ok(response) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ml_trading_proxy_creation() { + // This test validates the proxy struct can be created + // Full integration tests require running backend Trading Service + // Integration tests are in services/api/tests/service_proxy_tests.rs + } + + #[test] + fn test_ml_trading_proxy_is_send_sync() { + // Validate that MlTradingProxy can be shared across threads + fn assert_send_sync() {} + assert_send_sync::(); + } +} diff --git a/services/api/src/grpc/ml_training_proxy.rs b/services/api/src/grpc/ml_training_proxy.rs new file mode 100644 index 000000000..b3f33c4cf --- /dev/null +++ b/services/api/src/grpc/ml_training_proxy.rs @@ -0,0 +1,632 @@ +//! ML Training Service Proxy - Zero-copy gRPC forwarding for ML training operations +//! +//! This module implements a high-performance proxy for the ML Training Service with: +//! - Zero-copy message forwarding (routing overhead <10μs) +//! - Connection pooling via tonic::transport::Channel +//! - Circuit breaker integration for backend failures +//! - Efficient streaming support for training metrics +//! - Health checking integration + +use futures::Stream; +use std::pin::Pin; +use tonic::{Request, Response, Status}; +use tracing::{error, info, instrument, warn}; + +// Import the generated ML training service protobuf definitions from lib.rs +use crate::ml_training::ml_training_service_client::MlTrainingServiceClient; +use crate::ml_training::ml_training_service_server::{MlTrainingService, MlTrainingServiceServer}; +use crate::ml_training::{ + // Batch tuning types + BatchStartTuningJobsRequest, + BatchStartTuningJobsResponse, + GetBatchTuningStatusRequest, + GetBatchTuningStatusResponse, + GetTrainingJobDetailsRequest, + GetTrainingJobDetailsResponse, + GetTuningJobStatusRequest, + GetTuningJobStatusResponse, + HealthCheckRequest, + HealthCheckResponse, + ListAvailableModelsRequest, + ListAvailableModelsResponse, + ListTrainingJobsRequest, + ListTrainingJobsResponse, + ProgressUpdate, + StartTrainingRequest, + StartTrainingResponse, + // Tuning job types + StartTuningJobRequest, + StartTuningJobResponse, + StopBatchTuningJobRequest, + StopBatchTuningJobResponse, + StopTrainingRequest, + StopTrainingResponse, + StopTuningJobRequest, + StopTuningJobResponse, + StreamProgressRequest, + SubscribeToTrainingStatusRequest, + TrainModelRequest, + TrainModelResponse, + TrainingStatusUpdate, + // Job completion and model promotion types + JobCompletionReport, + JobCompletionAck, + ListPendingPromotionsRequest, + ListPendingPromotionsResponse, + ApprovePromotionRequest, + ApprovePromotionResponse, + RejectPromotionRequest, + RejectPromotionResponse, + ApproveModelRequest, + ApproveModelResponse, + RejectModelRequest, + RejectModelResponse, +}; + +/// ML Training Service Proxy +/// +/// Provides zero-copy forwarding of gRPC requests to the backend ML Training Service. +/// +/// Uses connection pooling and circuit breakers for high availability and performance. +#[derive(Debug, Clone)] +pub struct MlTrainingProxy { + /// Backend ML Training Service client with connection pooling + client: MlTrainingServiceClient, +} + +impl MlTrainingProxy { + /// Create a new ML Training Service proxy + /// + /// # Arguments + /// * `client` - Pre-configured ML Training Service client with circuit breaker + /// + /// # Performance + /// - Uses Arc-based channel cloning for zero-copy client reuse + /// + /// - Connection pooling managed by tonic::transport::Channel + pub fn new(client: MlTrainingServiceClient) -> Self { + Self { client } + } + + /// Convert proxy into a tonic server instance + pub fn into_server(self) -> MlTrainingServiceServer { + MlTrainingServiceServer::new(self) + } +} + +#[tonic::async_trait] +impl MlTrainingService for MlTrainingProxy { + /// Server streaming type for training status updates + type SubscribeToTrainingStatusStream = + Pin> + Send>>; + + /// Server streaming type for tuning progress updates + type StreamTuningProgressStream = + Pin> + Send>>; + + /// Start a new model training job + /// + /// # Performance + /// - Zero-copy message forwarding + /// + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn start_training( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying StartTraining request"); + + // Clone client (cheap Arc increment) for concurrent request handling + let mut client = self.client.clone(); + + // Forward request with zero-copy + let response = client.start_training(request).await.map_err(|e| { + error!("Backend StartTraining failed: {}", e); + e + })?; + + info!("StartTraining request forwarded successfully"); + Ok(response) + } + + /// Subscribe to real-time training status updates (server streaming) + /// + /// # Performance + /// - Zero-copy stream forwarding + /// + /// - No intermediate buffering + /// - Direct stream passthrough from backend + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn subscribe_to_training_status( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying SubscribeToTrainingStatus streaming request"); + + let mut client = self.client.clone(); + + // Get backend stream response + let stream_response = client + .subscribe_to_training_status(request) + .await + .map_err(|e| { + error!("Backend SubscribeToTrainingStatus failed: {}", e); + e + })?; + + // Extract inner stream and forward directly (zero-copy) + let stream = stream_response.into_inner(); + let boxed_stream = Box::pin(stream) as Self::SubscribeToTrainingStatusStream; + + info!("SubscribeToTrainingStatus streaming request forwarded successfully"); + Ok(Response::new(boxed_stream)) + } + + /// Stop a running training job + /// + /// # Performance + /// - Zero-copy message forwarding + /// + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn stop_training( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying StopTraining request"); + + let mut client = self.client.clone(); + let response = client.stop_training(request).await.map_err(|e| { + error!("Backend StopTraining failed: {}", e); + e + })?; + + info!("StopTraining request forwarded successfully"); + Ok(response) + } + + /// List available ML models + /// + /// # Performance + /// - Zero-copy message forwarding + /// + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn list_available_models( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying ListAvailableModels request"); + + let mut client = self.client.clone(); + let response = client.list_available_models(request).await.map_err(|e| { + error!("Backend ListAvailableModels failed: {}", e); + e + })?; + + info!("ListAvailableModels request forwarded successfully"); + Ok(response) + } + + /// List training job history + /// + /// # Performance + /// - Zero-copy message forwarding + /// + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn list_training_jobs( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying ListTrainingJobs request"); + + let mut client = self.client.clone(); + let response = client.list_training_jobs(request).await.map_err(|e| { + error!("Backend ListTrainingJobs failed: {}", e); + e + })?; + + info!("ListTrainingJobs request forwarded successfully"); + Ok(response) + } + + /// Get detailed information about a training job + /// + /// # Performance + /// - Zero-copy message forwarding + /// + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn get_training_job_details( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying GetTrainingJobDetails request"); + + let mut client = self.client.clone(); + let response = client + .get_training_job_details(request) + .await + .map_err(|e| { + error!("Backend GetTrainingJobDetails failed: {}", e); + e + })?; + + info!("GetTrainingJobDetails request forwarded successfully"); + Ok(response) + } + + /// Health check for ML Training Service backend + /// + /// # Performance + /// - Zero-copy message forwarding + /// + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn health_check( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying HealthCheck request"); + + let mut client = self.client.clone(); + let response = client.health_check(request).await.map_err(|e| { + warn!("Backend HealthCheck failed: {}", e); + e + })?; + + info!("HealthCheck request forwarded successfully"); + Ok(response) + } + + /// Start a new hyperparameter tuning job + /// + /// # Performance + /// - Zero-copy message forwarding + /// + /// - Routing overhead target: <10μs + /// + /// # Security + /// - Requires "ml.tune" permission in JWT metadata + /// - JWT validation handled by interceptor layer + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn start_tuning_job( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying StartTuningJob request"); + + // Clone client (cheap Arc increment) for concurrent request handling + let mut client = self.client.clone(); + + // Forward request with zero-copy + // Note: JWT validation and "ml.tune" permission check handled by interceptor + let response = client.start_tuning_job(request).await.map_err(|e| { + error!("Backend StartTuningJob failed: {}", e); + e + })?; + + info!("StartTuningJob request forwarded successfully"); + Ok(response) + } + + /// Get status and best parameters from a tuning job + /// + /// # Performance + /// - Zero-copy message forwarding + /// + /// - Routing overhead target: <10μs + /// + /// # Security + /// - Job ownership validation handled by backend service + /// - User can only query their own tuning jobs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn get_tuning_job_status( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying GetTuningJobStatus request"); + + let mut client = self.client.clone(); + + // Forward request - backend validates job ownership via user_id from JWT + let response = client.get_tuning_job_status(request).await.map_err(|e| { + error!("Backend GetTuningJobStatus failed: {}", e); + e + })?; + + info!("GetTuningJobStatus request forwarded successfully"); + Ok(response) + } + + /// Stop a running hyperparameter tuning job + /// + /// # Performance + /// - Zero-copy message forwarding + /// + /// - Routing overhead target: <10μs + /// + /// # Security + /// - Job ownership validation handled by backend service + /// - User can only stop their own tuning jobs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn stop_tuning_job( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying StopTuningJob request"); + + let mut client = self.client.clone(); + + // Forward stop request - backend validates job ownership via user_id from JWT + let response = client.stop_tuning_job(request).await.map_err(|e| { + error!("Backend StopTuningJob failed: {}", e); + e + })?; + + info!("StopTuningJob request forwarded successfully"); + Ok(response) + } + + /// INTERNAL: Train a single model instance with specific hyperparameters + /// + /// # Performance + /// - Zero-copy message forwarding + /// + /// - Routing overhead target: <10μs + /// + /// # Note + /// - Called by Optuna subprocess during tuning trials + /// - Not typically called directly by clients + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn train_model( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying TrainModel request (INTERNAL)"); + + let mut client = self.client.clone(); + + // Forward internal training request + let response = client.train_model(request).await.map_err(|e| { + error!("Backend TrainModel failed: {}", e); + e + })?; + + info!("TrainModel request forwarded successfully"); + Ok(response) + } + + /// Stream real-time tuning progress updates (server streaming) + /// + /// # Performance + /// - Zero-copy stream forwarding + /// + /// - No intermediate buffering + /// - Direct stream passthrough from backend + /// + /// # Security + /// - Tuning job ownership validation handled by backend service + /// - User can only subscribe to their own tuning jobs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn stream_tuning_progress( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying StreamTuningProgress streaming request"); + + let mut client = self.client.clone(); + + // Get backend stream response + let stream_response = client.stream_tuning_progress(request).await.map_err(|e| { + error!("Backend StreamTuningProgress failed: {}", e); + e + })?; + + // Extract inner stream and forward directly (zero-copy) + let stream = stream_response.into_inner(); + let boxed_stream = Box::pin(stream) as Self::StreamTuningProgressStream; + + info!("StreamTuningProgress streaming request forwarded successfully"); + Ok(Response::new(boxed_stream)) + } + + /// Start batch tuning for multiple models with automatic dependency resolution + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + /// + /// # Security + /// - Requires "ml.tune" permission in JWT metadata + /// - JWT validation handled by interceptor layer + /// + /// # Features + /// - Sequential model tuning with dependency resolution + /// - Automatic YAML export of best hyperparameters + /// - Supports all model types (DQN, PPO, MAMBA_2, TFT, etc.) + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn batch_start_tuning_jobs( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying BatchStartTuningJobs request"); + + // Clone client (cheap Arc increment) for concurrent request handling + let mut client = self.client.clone(); + + // Forward batch tuning request with zero-copy + // Note: JWT validation and "ml.tune" permission check handled by interceptor + let response = client.batch_start_tuning_jobs(request).await.map_err(|e| { + error!("Backend BatchStartTuningJobs failed: {}", e); + e + })?; + + info!("BatchStartTuningJobs request forwarded successfully"); + Ok(response) + } + + /// Get batch tuning job status with per-model results + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + /// + /// # Security + /// - Batch job ownership validation handled by backend service + /// - User can only query their own batch jobs + /// + /// # Returns + /// - Batch status (PENDING, RUNNING, COMPLETED, FAILED, STOPPED) + /// - Per-model tuning results + /// - Current execution progress + /// - Estimated completion time + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn get_batch_tuning_status( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying GetBatchTuningStatus request"); + + let mut client = self.client.clone(); + + // Forward request - backend validates batch job ownership via user_id from JWT + let response = client.get_batch_tuning_status(request).await.map_err(|e| { + error!("Backend GetBatchTuningStatus failed: {}", e); + e + })?; + + info!("GetBatchTuningStatus request forwarded successfully"); + Ok(response) + } + + /// Stop a running batch tuning job + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + /// + /// # Security + /// - Batch job ownership validation handled by backend service + /// - User can only stop their own batch jobs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn stop_batch_tuning_job( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying StopBatchTuningJob request"); + + let mut client = self.client.clone(); + + // Forward stop request - backend validates batch job ownership via user_id from JWT + let response = client.stop_batch_tuning_job(request).await.map_err(|e| { + error!("Backend StopBatchTuningJob failed: {}", e); + e + })?; + + info!("StopBatchTuningJob request forwarded successfully"); + Ok(response) + } + + /// Report job completion (called by training-uploader sidecar) + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn report_job_completion( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying ReportJobCompletion request"); + let mut client = self.client.clone(); + let response = client.report_job_completion(request).await.map_err(|e| { + error!("Backend ReportJobCompletion failed: {}", e); + e + })?; + Ok(response) + } + + /// List models pending promotion approval + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn list_pending_promotions( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying ListPendingPromotions request"); + let mut client = self.client.clone(); + let response = client.list_pending_promotions(request).await.map_err(|e| { + error!("Backend ListPendingPromotions failed: {}", e); + e + })?; + Ok(response) + } + + /// Approve a pending model promotion + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn approve_promotion( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying ApprovePromotion request"); + let mut client = self.client.clone(); + let response = client.approve_promotion(request).await.map_err(|e| { + error!("Backend ApprovePromotion failed: {}", e); + e + })?; + Ok(response) + } + + /// Reject a pending model promotion + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn reject_promotion( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying RejectPromotion request"); + let mut client = self.client.clone(); + let response = client.reject_promotion(request).await.map_err(|e| { + error!("Backend RejectPromotion failed: {}", e); + e + })?; + Ok(response) + } + + /// Approve a model for promotion + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn approve_model( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying ApproveModel request"); + let mut client = self.client.clone(); + let response = client.approve_model(request).await.map_err(|e| { + error!("Backend ApproveModel failed: {}", e); + e + })?; + Ok(response) + } + + /// Reject a model promotion + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn reject_model( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying RejectModel request"); + let mut client = self.client.clone(); + let response = client.reject_model(request).await.map_err(|e| { + error!("Backend RejectModel failed: {}", e); + e + })?; + Ok(response) + } +} + +#[cfg(test)] +mod tests { + + #[test] + fn test_proxy_creation() { + // This test validates the proxy struct can be created + // Full integration tests require running backend service + } +} diff --git a/services/api/src/grpc/mod.rs b/services/api/src/grpc/mod.rs new file mode 100644 index 000000000..9d21a4d8a --- /dev/null +++ b/services/api/src/grpc/mod.rs @@ -0,0 +1,40 @@ +//! gRPC modules for API Gateway +//! +//! This module contains gRPC handlers and proxies for backend services: +//! - Trading Service (zero-copy proxy, <10us routing overhead) +//! - Backtesting Service (zero-copy proxy) +//! - ML Training Service (zero-copy proxy) +//! - Trading Agent Service (zero-copy proxy) +//! - Monitoring Service (direct Prometheus handler) + +pub mod backtesting_proxy; +pub mod broker_gateway_proxy; +pub mod config_proxy; +pub mod data_acquisition_proxy; +pub mod ml_proxy; +pub mod ml_trading_proxy; +pub mod ml_training_proxy; +pub mod monitoring_handler; +pub mod pods_handler; +pub mod risk_proxy; +pub mod server; +pub mod trading_agent_proxy; +pub mod trading_direct_proxy; +pub mod trading_proxy; + +pub use backtesting_proxy::BacktestingServiceProxy; +pub use broker_gateway_proxy::BrokerGatewayProxy; +pub use config_proxy::ConfigServiceProxy; +pub use data_acquisition_proxy::DataAcquisitionProxy; +pub use ml_proxy::MlServiceProxy; +pub use ml_trading_proxy::MlTradingProxy; +pub use ml_training_proxy::MlTrainingProxy; +pub use monitoring_handler::MonitoringServiceHandler; +pub use risk_proxy::RiskServiceProxy; +pub use server::{ + setup_ml_training_client, setup_ml_training_proxy, setup_trading_agent_client, + setup_trading_agent_proxy, MlTrainingBackendConfig, TradingAgentBackendConfig, +}; +pub use trading_agent_proxy::TradingAgentProxy; +pub use trading_direct_proxy::TradingDirectProxy; +pub use trading_proxy::{BackendHealthState, HealthChecker, ServiceHealthEntry, TradingServiceProxy}; diff --git a/services/api/src/grpc/monitoring_handler.rs b/services/api/src/grpc/monitoring_handler.rs new file mode 100644 index 000000000..b273588db --- /dev/null +++ b/services/api/src/grpc/monitoring_handler.rs @@ -0,0 +1,1386 @@ +//! Monitoring Service Handler - Direct Prometheus scraping for training metrics +//! +//! Serves `GetLiveTrainingMetrics`, `StreamTrainingMetrics`, and `GetEpochHistory` +//! by querying Prometheus directly, eliminating the monitoring-service backend. +//! +//! The merged monitoring.proto defines 16 RPCs (system health + training). +//! This handler implements the 3 training RPCs with real Prometheus scraping; +//! the 13 system health RPCs return UNIMPLEMENTED until real health checks are wired. + +use std::collections::{HashMap, VecDeque}; +use std::pin::Pin; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use futures::Stream; +use serde::Deserialize; +use tokio::sync::RwLock; +use tonic::{Request, Response, Status}; +use tracing::{error, info, instrument}; + +use crate::monitoring::monitoring_service_server::{MonitoringService, MonitoringServiceServer}; +use crate::monitoring::{ + AcknowledgeAlertRequest, AcknowledgeAlertResponse, AlertEvent, + ClusterPodsResponse, EpochFinancialSnapshot, GetActiveAlertsRequest, GetActiveAlertsResponse, + GetEpochHistoryRequest, GetEpochHistoryResponse, GetHealthCheckRequest, + GetHealthCheckResponse, GetLatencyMetricsRequest, GetLatencyMetricsResponse, + GetLiveTrainingMetricsRequest, GetLiveTrainingMetricsResponse, GetMetricsRequest, + GetMetricsResponse, GetSystemStatusRequest, GetSystemStatusResponse, + GetThroughputMetricsRequest, GetThroughputMetricsResponse, GpuSnapshot, HealthCheck, + HealthStatus, Metric, MetricType, MetricsEvent, ServiceHealth, ServiceState, ServiceStatus, + StreamTrainingMetricsRequest, SubscribeClusterPodsRequest, SystemHealth, SystemMetrics, + SystemStatus, SystemStatusChangeType, SystemStatusEvent, TrainingSession, +}; + +// ============================================================================ +// Prometheus client (ported from monitoring_service/src/prometheus_client.rs) +// ============================================================================ + +/// A single Prometheus instant-query result entry +#[derive(Debug, Deserialize)] +struct PromResult { + metric: HashMap, + value: (f64, String), // (timestamp, value_string) +} + +#[derive(Debug, Deserialize)] +struct PromData { + result: Vec, +} + +#[derive(Debug, Deserialize)] +struct PromResponse { + status: String, + data: PromData, +} + +/// Parsed metric value with model/fold labels +#[derive(Debug, Clone)] +pub struct MetricSample { + pub name: String, + pub model: String, + pub fold: String, + pub value: f64, +} + +struct PrometheusClient { + http: reqwest::Client, + base_url: String, +} + +impl PrometheusClient { + fn new(base_url: &str) -> Self { + let http = reqwest::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .unwrap_or_else(|_| reqwest::Client::new()); + Self { + http, + base_url: base_url.trim_end_matches('/').to_owned(), + } + } + + /// Execute an instant query against Prometheus + async fn query(&self, promql: &str) -> Result> { + let url = format!("{}/api/v1/query", self.base_url); + let resp = self + .http + .get(&url) + .query(&[("query", promql)]) + .send() + .await + .context("Prometheus HTTP request failed")?; + + if !resp.status().is_success() { + anyhow::bail!("Prometheus returned HTTP {}", resp.status()); + } + + let body: PromResponse = resp + .json() + .await + .context("Failed to parse Prometheus response")?; + + if body.status != "success" { + anyhow::bail!("Prometheus query status: {}", body.status); + } + + Ok(body.data.result) + } + + /// Fetch all training + hyperopt metrics + async fn fetch_training_metrics(&self) -> Result> { + let results = self + .query(r#"{__name__=~"foxhunt_training_.*|foxhunt_hyperopt_.*"}"#) + .await?; + Ok(parse_samples(results)) + } + + /// Fetch GPU metrics from DCGM exporter + async fn fetch_gpu_metrics(&self) -> Result> { + let results = self + .query(r#"{__name__=~"dcgm_gpu_utilization|dcgm_fb_used|dcgm_fb_free|dcgm_gpu_temp|dcgm_power_usage"}"#) + .await?; + Ok(parse_samples(results)) + } + + /// Fetch active training worker count (covers both K8s Jobs and CI runner pods) + async fn fetch_active_jobs(&self) -> Result { + let results = self.query("foxhunt_training_active_workers").await?; + let count: f64 = results + .iter() + .filter_map(|r| r.value.1.parse::().ok()) + .sum(); + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let count = count as u32; + Ok(count) + } + + /// Query node-level CPU and memory usage from Prometheus. + /// + /// Returns `(cpu_usage_pct, mem_usage_pct, disk_usage_pct)`. + /// Falls back to zeros on any query failure (best-effort). + async fn query_system_metrics(&self) -> (f64, f64, f64) { + // CPU: 1 − idle fraction across all cores, averaged over 1m + let cpu = self + .query( + r#"100 * (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[1m])))"#, + ) + .await + .ok() + .and_then(|r| r.first().and_then(|v| v.value.1.parse::().ok())) + .unwrap_or(0.0); + + // Memory: (total − available) / total × 100 + let mem = self + .query( + r#"100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)"#, + ) + .await + .ok() + .and_then(|r| r.first().and_then(|v| v.value.1.parse::().ok())) + .unwrap_or(0.0); + + // Disk: (total − free) / total × 100 on root mount + let disk = self + .query( + r#"100 * (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"})"#, + ) + .await + .ok() + .and_then(|r| r.first().and_then(|v| v.value.1.parse::().ok())) + .unwrap_or(0.0); + + (cpu, mem, disk) + } + + /// Query cluster-level CPU and memory from node-exporter metrics. + /// + /// Returns `(cpu_usage_pct, mem_used_mb, mem_total_mb)`. + /// Best-effort: returns `(0.0, 0.0, 0.0)` on any query failure. + #[allow(clippy::cast_possible_truncation)] + async fn query_cluster_resources(&self) -> (f32, f32, f32) { + let cpu_pct = self + .query( + r#"100 * (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[1m])))"#, + ) + .await + .ok() + .and_then(|r| r.first().and_then(|v| v.value.1.parse::().ok())) + .unwrap_or(0.0) as f32; + + let mem_used_mb = self + .query( + r#"sum(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / 1024 / 1024"#, + ) + .await + .ok() + .and_then(|r| r.first().and_then(|v| v.value.1.parse::().ok())) + .unwrap_or(0.0) as f32; + + let mem_total_mb = self + .query(r#"sum(node_memory_MemTotal_bytes) / 1024 / 1024"#) + .await + .ok() + .and_then(|r| r.first().and_then(|v| v.value.1.parse::().ok())) + .unwrap_or(0.0) as f32; + + (cpu_pct, mem_used_mb, mem_total_mb) + } +} + +fn parse_samples(results: Vec) -> Vec { + results + .into_iter() + .filter_map(|r| { + let value: f64 = r.value.1.parse().ok()?; + Some(MetricSample { + name: r.metric.get("__name__")?.clone(), + model: r.metric.get("model").cloned().unwrap_or_default(), + fold: r.metric.get("fold").cloned().unwrap_or_default(), + value, + }) + }) + .collect() +} + +// ============================================================================ +// Monitoring Service Handler +// ============================================================================ + +const MAX_EPOCH_HISTORY: usize = 50; + +/// Monitoring Service Handler +/// +/// Directly scrapes Prometheus for training metrics, GPU stats, and active jobs. +/// Replaces the former proxy pattern that forwarded to monitoring-service backend. +pub struct MonitoringServiceHandler { + prom: Arc, + default_interval: u32, + epoch_histories: Arc>>>, + last_epochs: Arc>>, + /// Backend service URLs for health checking: (name, url) + backends: Arc>, + start_time: std::time::Instant, +} + +impl MonitoringServiceHandler { + pub fn new(prometheus_url: &str, default_interval_secs: u32) -> Self { + let prom = PrometheusClient::new(prometheus_url); + info!( + "MonitoringServiceHandler: Prometheus={}, interval={}s", + prometheus_url, default_interval_secs + ); + Self { + prom: Arc::new(prom), + default_interval: default_interval_secs, + epoch_histories: Arc::new(RwLock::new(HashMap::new())), + last_epochs: Arc::new(RwLock::new(HashMap::new())), + backends: Arc::new(Vec::new()), + start_time: std::time::Instant::now(), + } + } + + /// Set the list of backend service URLs for health checking + pub fn with_backends(mut self, backends: Vec<(String, String)>) -> Self { + self.backends = Arc::new(backends); + self + } + + /// Check health of a single backend via gRPC health check protocol. + /// + /// `uptime_secs` is the gateway's own uptime — used as an approximation + /// for the backend service uptime (individual services don't expose this). + async fn check_backend_health( + name: &str, + url: &str, + uptime_secs: i64, + ) -> ServiceStatus { + let channel = match tonic::transport::Channel::from_shared(url.to_string()) { + Ok(c) => c.connect_lazy(), + Err(_) => { + return ServiceStatus { + service_name: name.to_string(), + health: ServiceHealth::Unhealthy.into(), + state: ServiceState::Error.into(), + error_message: Some(format!("Invalid URL: {url}")), + version: Some(common::build_info::version().to_owned()), + ..Default::default() + }; + } + }; + + let mut hc = tonic_health::pb::health_client::HealthClient::new(channel); + let req = tonic_health::pb::HealthCheckRequest { + service: String::new(), + }; + + let start = std::time::Instant::now(); + let (health, state, msg) = match tokio::time::timeout( + Duration::from_millis(2000), + hc.check(req), + ) + .await + { + Ok(Ok(_)) => ( + ServiceHealth::Healthy, + ServiceState::Running, + None, + ), + Ok(Err(s)) if s.code() == tonic::Code::Unimplemented => ( + ServiceHealth::Healthy, + ServiceState::Running, + Some("Serving (no health service)".to_string()), + ), + Ok(Err(e)) => ( + ServiceHealth::Unhealthy, + ServiceState::Error, + Some(format!("gRPC error: {e}")), + ), + Err(_) => ( + ServiceHealth::Unhealthy, + ServiceState::Error, + Some("Unreachable (2s timeout)".to_string()), + ), + }; + let latency_ms = start.elapsed().as_secs_f64() * 1000.0; + + let now = chrono::Utc::now().timestamp(); + let mut metadata = std::collections::HashMap::new(); + metadata.insert("latency_ms".to_string(), format!("{latency_ms:.1}")); + + ServiceStatus { + service_name: name.to_string(), + health: health.into(), + state: state.into(), + error_message: msg, + version: Some(common::build_info::version().to_owned()), + last_health_check: now, + uptime_seconds: uptime_secs, + metadata, + ..Default::default() + } + } + + pub fn into_server(self) -> MonitoringServiceServer { + MonitoringServiceServer::new(self) + } + + /// Build a full training metrics snapshot from Prometheus and update epoch history. + async fn build_response( + prom: &PrometheusClient, + model_filter: &str, + epoch_histories: &RwLock>>, + last_epochs: &RwLock>, + ) -> Result { + // Fetch training/GPU/jobs (fail-fast) and cluster resources (best-effort) concurrently. + let (training_result, (cpu_pct, mem_used_mb, mem_total_mb)) = tokio::join!( + async { + tokio::try_join!( + prom.fetch_training_metrics(), + prom.fetch_gpu_metrics(), + prom.fetch_active_jobs(), + ) + }, + prom.query_cluster_resources(), + ); + + let (training, gpu, jobs) = training_result + .map_err(|e| Status::internal(format!("Prometheus query failed: {e}")))?; + + let sessions = group_into_sessions(&training, model_filter); + let gpu_snapshot = build_gpu_snapshot(&gpu); + + // Record epoch history snapshots for sessions with new epoch data + { + let mut last = last_epochs.write().await; + let mut histories = epoch_histories.write().await; + for session in &sessions { + let key = format!("{}/{}", session.model, session.fold); + let prev_epoch = last.get(&key).copied().unwrap_or_default(); + if session.current_epoch > prev_epoch && session.epoch_sharpe != 0.0 { + last.insert(key.clone(), session.current_epoch); + let snapshot = EpochFinancialSnapshot { + epoch: session.current_epoch as u32, + sharpe: session.epoch_sharpe, + sortino: session.epoch_sortino, + win_rate: session.epoch_win_rate, + max_drawdown: session.epoch_max_drawdown, + profit_factor: session.epoch_profit_factor, + total_return: session.epoch_total_return, + avg_return: session.epoch_avg_return, + total_trades: session.epoch_total_trades, + loss: session.epoch_loss, + val_loss: session.validation_loss, + learning_rate: session.learning_rate, + action_buy_pct: session.action_buy_pct, + action_sell_pct: session.action_sell_pct, + action_hold_pct: session.action_hold_pct, + }; + let history = histories + .entry(key) + .or_insert_with(|| VecDeque::with_capacity(MAX_EPOCH_HISTORY)); + if history.len() >= MAX_EPOCH_HISTORY { + history.pop_front(); + } + history.push_back(snapshot); + } + } + } + + Ok(GetLiveTrainingMetricsResponse { + sessions, + gpu: Some(gpu_snapshot), + active_k8s_jobs: jobs, + timestamp: chrono::Utc::now().timestamp(), + cpu_percent: cpu_pct, + memory_used_mb: mem_used_mb, + memory_total_mb: mem_total_mb, + }) + } +} + +#[tonic::async_trait] +impl MonitoringService for MonitoringServiceHandler { + // ======================================================================== + // Training metrics RPCs (implemented -- direct Prometheus scraping) + // ======================================================================== + + type StreamTrainingMetricsStream = + Pin> + Send>>; + + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn get_live_training_metrics( + &self, + request: Request, + ) -> Result, Status> { + info!("GetLiveTrainingMetrics: scraping Prometheus"); + let filter = &request.into_inner().model_filter; + let resp = + Self::build_response(&self.prom, filter, &self.epoch_histories, &self.last_epochs) + .await?; + Ok(Response::new(resp)) + } + + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn stream_training_metrics( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let interval_secs = if req.interval_seconds == 0 { + self.default_interval + } else { + req.interval_seconds.clamp(1, 60) + }; + let filter = req.model_filter; + let prom = self.prom.clone(); + let epoch_histories = self.epoch_histories.clone(); + let last_epochs = self.last_epochs.clone(); + + info!( + "StreamTrainingMetrics: interval={}s, filter={:?}", + interval_secs, + if filter.is_empty() { + "" + } else { + &filter + } + ); + + let stream = async_stream::stream! { + let mut interval = tokio::time::interval(Duration::from_secs(u64::from(interval_secs))); + loop { + interval.tick().await; + match Self::build_response(&prom, &filter, &epoch_histories, &last_epochs).await { + Ok(resp) => yield Ok(resp), + Err(e) => { + error!("Stream tick failed: {}", e); + yield Err(e); + break; + } + } + } + }; + + Ok(Response::new(Box::pin(stream))) + } + + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn get_epoch_history( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let key = format!("{}/{}", req.model, req.fold); + let histories = self.epoch_histories.read().await; + let epochs = match histories.get(&key) { + Some(deque) => { + let max = if req.max_epochs == 0 { + MAX_EPOCH_HISTORY + } else { + req.max_epochs as usize + }; + deque.iter().rev().take(max).rev().cloned().collect() + } + None => vec![], + }; + Ok(Response::new(GetEpochHistoryResponse { + model: req.model, + fold: req.fold, + epochs, + })) + } + + // ======================================================================== + // System health RPCs (stubs -- will be implemented when real health checks are wired) + // ======================================================================== + + type StreamSystemStatusStream = + Pin> + Send>>; + + #[instrument(skip(self, request), err)] + async fn get_system_status( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let backends = self.backends.clone(); + #[allow(clippy::cast_possible_truncation)] + let uptime_secs = self.start_time.elapsed().as_secs() as i64; + + // Fan-out health checks + system metrics concurrently + let prom = self.prom.clone(); + let checks: Vec<_> = backends + .iter() + .filter(|(name, _)| { + req.service_names.is_empty() || req.service_names.contains(name) + }) + .map(|(name, url)| { + let name = name.clone(); + let url = url.clone(); + async move { Self::check_backend_health(&name, &url, uptime_secs).await } + }) + .collect(); + + let (statuses, (cpu, mem, disk)) = tokio::join!( + futures::future::join_all(checks), + prom.query_system_metrics(), + ); + + let healthy_count = statuses + .iter() + .filter(|s| s.health == i32::from(ServiceHealth::Healthy)) + .count(); + let total = statuses.len(); + + #[allow(clippy::cast_possible_truncation)] + let overall_health = if healthy_count == total { + SystemHealth::Healthy + } else if healthy_count == 0 { + SystemHealth::Critical + } else if healthy_count * 2 >= total { + SystemHealth::Degraded + } else { + SystemHealth::Unhealthy + }; + + let critical_issues: Vec = statuses + .iter() + .filter(|s| s.health != i32::from(ServiceHealth::Healthy)) + .map(|s| { + format!( + "{}: {}", + s.service_name, + s.error_message.as_deref().unwrap_or("unhealthy") + ) + }) + .collect(); + + #[allow(clippy::cast_possible_truncation)] + let response = GetSystemStatusResponse { + overall_status: Some(SystemStatus { + overall_health: overall_health.into(), + healthy_services: healthy_count as i32, + total_services: total as i32, + critical_issues, + system_uptime_seconds: uptime_secs, + system_metrics: Some(SystemMetrics { + cpu_usage_percent: cpu, + memory_usage_percent: mem, + disk_usage_percent: disk, + ..Default::default() + }), + }), + service_statuses: statuses, + timestamp: chrono::Utc::now().timestamp(), + }; + + Ok(Response::new(response)) + } + + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn stream_system_status( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let interval_secs = req.update_frequency_seconds.unwrap_or(0); + let interval_secs = if interval_secs == 0 { + 5u32 + } else { + (interval_secs as u32).clamp(1, 60) + }; + let backends = self.backends.clone(); + let start_time = self.start_time; + let service_filter = req.service_names; + + info!( + "StreamSystemStatus: interval={}s, filter={:?}", + interval_secs, + if service_filter.is_empty() { + "" + } else { + "filtered" + } + ); + + let prom = self.prom.clone(); + + let stream = async_stream::stream! { + let mut interval = tokio::time::interval(Duration::from_secs(u64::from(interval_secs))); + loop { + interval.tick().await; + + #[allow(clippy::cast_possible_truncation)] + let uptime_secs = start_time.elapsed().as_secs() as i64; + let checks: Vec<_> = backends + .iter() + .filter(|(name, _)| { + service_filter.is_empty() || service_filter.contains(name) + }) + .map(|(name, url)| { + let name = name.clone(); + let url = url.clone(); + async move { Self::check_backend_health(&name, &url, uptime_secs).await } + }) + .collect(); + + let (statuses, (cpu, mem, disk)) = tokio::join!( + futures::future::join_all(checks), + prom.query_system_metrics(), + ); + + let healthy_count = statuses + .iter() + .filter(|s| s.health == i32::from(ServiceHealth::Healthy)) + .count(); + let total = statuses.len(); + + #[allow(clippy::cast_possible_truncation)] + let overall_health = if healthy_count == total { + SystemHealth::Healthy + } else if healthy_count == 0 { + SystemHealth::Critical + } else if healthy_count * 2 >= total { + SystemHealth::Degraded + } else { + SystemHealth::Unhealthy + }; + + let critical_issues: Vec = statuses + .iter() + .filter(|s| s.health != i32::from(ServiceHealth::Healthy)) + .map(|s| { + format!( + "{}: {}", + s.service_name, + s.error_message.as_deref().unwrap_or("unhealthy") + ) + }) + .collect(); + + #[allow(clippy::cast_possible_truncation)] + let system_status = SystemStatus { + overall_health: overall_health.into(), + healthy_services: healthy_count as i32, + total_services: total as i32, + critical_issues, + system_uptime_seconds: start_time.elapsed().as_secs() as i64, + system_metrics: Some(SystemMetrics { + cpu_usage_percent: cpu, + memory_usage_percent: mem, + disk_usage_percent: disk, + ..Default::default() + }), + }; + + let event = SystemStatusEvent { + system_status: Some(system_status), + change_type: SystemStatusChangeType::Unspecified.into(), + timestamp: chrono::Utc::now().timestamp(), + }; + + yield Ok(event); + } + }; + + Ok(Response::new(Box::pin(stream))) + } + + #[instrument(skip(self, request), err)] + async fn get_health_check( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let backends = self.backends.clone(); + #[allow(clippy::cast_possible_truncation)] + let uptime_secs = self.start_time.elapsed().as_secs() as i64; + + // Fan-out health checks + let checks: Vec<_> = backends + .iter() + .filter(|(name, _)| { + req.service_name.is_none() + || req.service_name.as_deref() == Some(name.as_str()) + }) + .map(|(name, url)| { + let name = name.clone(); + let url = url.clone(); + async move { + let start = std::time::Instant::now(); + let status = Self::check_backend_health(&name, &url, uptime_secs).await; + let elapsed = start.elapsed(); + let healthy = status.health == i32::from(ServiceHealth::Healthy); + HealthCheck { + check_name: name, + status: if healthy { + HealthStatus::Healthy.into() + } else { + HealthStatus::Unhealthy.into() + }, + message: status.error_message, + response_time_ms: Some(elapsed.as_secs_f64() * 1000.0), + last_checked: chrono::Utc::now().timestamp(), + details: HashMap::new(), + } + } + }) + .collect(); + + let health_checks = futures::future::join_all(checks).await; + + let all_healthy = health_checks + .iter() + .all(|c| c.status == i32::from(HealthStatus::Healthy)); + + let response = GetHealthCheckResponse { + health_status: if all_healthy { + HealthStatus::Healthy.into() + } else { + HealthStatus::Degraded.into() + }, + health_checks, + timestamp: chrono::Utc::now().timestamp(), + }; + + Ok(Response::new(response)) + } + + type StreamMetricsStream = + Pin> + Send>>; + + #[instrument(skip(self, request), err)] + async fn get_metrics( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + + // If specific metric names requested, query each; otherwise return empty + let mut metrics = Vec::new(); + for name in &req.metric_names { + match self.prom.query(name).await { + Ok(results) => { + for r in results { + if let Ok(value) = r.value.1.parse::() { + metrics.push(Metric { + name: r + .metric + .get("__name__") + .cloned() + .unwrap_or_else(|| name.clone()), + metric_type: MetricType::Gauge.into(), + value, + unit: String::new(), + labels: r.metric, + timestamp: chrono::Utc::now().timestamp(), + statistics: None, + }); + } + } + } + Err(e) => { + error!("Prometheus query for '{}' failed: {}", name, e); + } + } + } + + Ok(Response::new(GetMetricsResponse { + metrics, + timestamp: chrono::Utc::now().timestamp(), + })) + } + + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn stream_metrics( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let interval_secs = req.update_frequency_seconds.unwrap_or(0); + let interval_secs = if interval_secs == 0 { + 5u32 + } else { + (interval_secs as u32).clamp(1, 60) + }; + let metric_names: Vec = if req.metric_names.is_empty() { + vec![ + "cpu_usage".to_owned(), + "memory_usage".to_owned(), + "gpu_utilization".to_owned(), + ] + } else { + req.metric_names + }; + let prom = self.prom.clone(); + + info!( + "StreamMetrics: interval={}s, metrics={:?}", + interval_secs, metric_names + ); + + let stream = async_stream::stream! { + let mut interval = tokio::time::interval(Duration::from_secs(u64::from(interval_secs))); + loop { + interval.tick().await; + + let mut metrics = Vec::new(); + for name in &metric_names { + match prom.query(name).await { + Ok(results) => { + for r in results { + if let Ok(value) = r.value.1.parse::() { + metrics.push(Metric { + name: r + .metric + .get("__name__") + .cloned() + .unwrap_or_else(|| name.clone()), + metric_type: MetricType::Gauge.into(), + value, + unit: String::new(), + labels: r.metric, + timestamp: chrono::Utc::now().timestamp(), + statistics: None, + }); + } + } + } + Err(e) => { + error!("StreamMetrics: Prometheus query for '{}' failed: {}", name, e); + } + } + } + + let event = MetricsEvent { + metrics, + timestamp: chrono::Utc::now().timestamp(), + }; + + yield Ok(event); + } + }; + + Ok(Response::new(Box::pin(stream))) + } + + async fn get_latency_metrics( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "GetLatencyMetrics not yet implemented", + )) + } + + async fn get_throughput_metrics( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "GetThroughputMetrics not yet implemented", + )) + } + + type StreamAlertsStream = + Pin> + Send>>; + + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn stream_alerts( + &self, + request: Request, + ) -> Result, Status> { + let _req = request.into_inner(); + Err(Status::unimplemented( + "StreamAlerts will be wired to Prometheus Alertmanager", + )) + } + + async fn acknowledge_alert( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "AcknowledgeAlert not yet implemented", + )) + } + + async fn get_active_alerts( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "GetActiveAlerts not yet implemented", + )) + } + + // ======================================================================== + // Cluster pods RPC (real -- queries Kubernetes API via pods_handler) + // ======================================================================== + + type SubscribeClusterPodsStream = + Pin> + Send>>; + + async fn subscribe_cluster_pods( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let interval_secs = if req.interval_seconds == 0 { + 5 + } else { + req.interval_seconds.clamp(1, 60) + }; + + info!( + "SubscribeClusterPods: interval={}s, namespace=foxhunt", + interval_secs + ); + + let stream = async_stream::stream! { + let mut interval = tokio::time::interval(Duration::from_secs(u64::from(interval_secs))); + loop { + interval.tick().await; + match super::pods_handler::list_pods("foxhunt").await { + Ok(pods) => { + yield Ok(ClusterPodsResponse { + pods, + timestamp: chrono::Utc::now().timestamp(), + }); + } + Err(e) => { + tracing::warn!("Pod listing failed: {e}"); + yield Ok(ClusterPodsResponse { + pods: vec![], + timestamp: chrono::Utc::now().timestamp(), + }); + } + } + } + }; + + Ok(Response::new(Box::pin(stream))) + } +} + +// ============================================================================ +// Helper functions (ported from monitoring_service/src/service.rs) +// ============================================================================ + +/// Group flat metric samples into TrainingSession structs keyed by (model, fold) +fn group_into_sessions(samples: &[MetricSample], model_filter: &str) -> Vec { + let mut map: HashMap<(String, String), TrainingSession> = HashMap::new(); + + for s in samples { + // Skip metrics without a model label (e.g. foxhunt_training_active_workers) + if s.model.is_empty() { + continue; + } + if !model_filter.is_empty() && s.model != model_filter { + continue; + } + + let key = (s.model.clone(), s.fold.clone()); + let session = map.entry(key).or_insert_with(|| TrainingSession { + model: s.model.clone(), + fold: s.fold.clone(), + ..Default::default() + }); + + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + match s.name.as_str() { + "foxhunt_training_current_epoch" => session.current_epoch = s.value as f32, + "foxhunt_training_epoch_loss" => session.epoch_loss = s.value as f32, + "foxhunt_training_validation_loss" => session.validation_loss = s.value as f32, + "foxhunt_training_batches_per_second" => session.batches_per_second = s.value as f32, + "foxhunt_training_batches_processed" => session.batches_processed = s.value as f32, + "foxhunt_training_iteration_seconds" => session.iteration_seconds = s.value as f32, + "foxhunt_training_eval_accuracy" => session.eval_accuracy = s.value as f32, + "foxhunt_training_eval_precision" => session.eval_precision = s.value as f32, + "foxhunt_training_eval_recall" => session.eval_recall = s.value as f32, + "foxhunt_training_eval_f1" => session.eval_f1 = s.value as f32, + "foxhunt_training_checkpoint_size_bytes" => { + session.checkpoint_size_bytes = s.value as f32; + } + "foxhunt_training_checkpoint_saves_total" => { + session.checkpoint_saves = s.value as u32; + } + "foxhunt_training_checkpoint_failures_total" => { + session.checkpoint_failures = s.value as u32; + } + "foxhunt_training_nan_detected_total" => session.nan_detected = s.value as u32, + "foxhunt_training_gradient_explosion_total" => { + session.gradient_explosions = s.value as u32; + } + "foxhunt_training_feature_errors_total" => session.feature_errors = s.value as u32, + "foxhunt_hyperopt_trial_current" => { + session.hyperopt_trial_current = s.value as u32; + session.is_hyperopt = true; + } + "foxhunt_hyperopt_trial_total" => session.hyperopt_trial_total = s.value as u32, + "foxhunt_hyperopt_best_objective" => { + session.hyperopt_best_objective = s.value as f32; + } + "foxhunt_hyperopt_trials_failed_total" => { + session.hyperopt_trials_failed = s.value as u32; + } + "foxhunt_hyperopt_mode" => { + if s.value > 0.5 { + session.is_hyperopt = true; + } + } + // RL diagnostics + "foxhunt_training_q_value_mean" => session.q_value_mean = s.value as f32, + "foxhunt_training_q_value_max" => session.q_value_max = s.value as f32, + "foxhunt_training_policy_entropy" => session.policy_entropy = s.value as f32, + "foxhunt_training_kl_divergence" => session.kl_divergence = s.value as f32, + "foxhunt_training_advantage_mean" => session.advantage_mean = s.value as f32, + "foxhunt_training_replay_buffer_size" => { + session.replay_buffer_size = s.value as u32; + } + // Gradient & training health + "foxhunt_training_gradient_norm" => session.gradient_norm = s.value as f32, + "foxhunt_training_learning_rate" => session.learning_rate = s.value as f32, + "foxhunt_training_epoch_duration_seconds" => { + session.epoch_duration_seconds = s.value as f32; + } + // Hyperopt intra-trial + "foxhunt_hyperopt_trial_epoch" => session.hyperopt_trial_epoch = s.value as u32, + "foxhunt_hyperopt_trial_best_loss" => { + session.hyperopt_trial_best_loss = s.value as f32; + } + "foxhunt_hyperopt_elapsed_seconds" => { + session.hyperopt_elapsed_seconds = s.value as f32; + } + // Epoch-level financial metrics + "foxhunt_training_epoch_sharpe" => session.epoch_sharpe = s.value as f32, + "foxhunt_training_epoch_sortino" => session.epoch_sortino = s.value as f32, + "foxhunt_training_epoch_win_rate" => session.epoch_win_rate = s.value as f32, + "foxhunt_training_epoch_max_drawdown" => { + session.epoch_max_drawdown = s.value as f32; + } + "foxhunt_training_epoch_profit_factor" => { + session.epoch_profit_factor = s.value as f32; + } + "foxhunt_training_epoch_total_return" => { + session.epoch_total_return = s.value as f32; + } + "foxhunt_training_epoch_avg_return" => { + session.epoch_avg_return = s.value as f32; + } + "foxhunt_training_epoch_total_trades" => { + session.epoch_total_trades = s.value as u32; + } + // Action distribution + "foxhunt_training_epoch_action_buy_pct" => session.action_buy_pct = s.value as f32, + "foxhunt_training_epoch_action_sell_pct" => session.action_sell_pct = s.value as f32, + "foxhunt_training_epoch_action_hold_pct" => session.action_hold_pct = s.value as f32, + _ => {} + } + } + + let mut sessions: Vec<_> = map.into_values().collect(); + sessions.sort_by(|a, b| (&a.model, &a.fold).cmp(&(&b.model, &b.fold))); + sessions +} + +fn build_gpu_snapshot(samples: &[MetricSample]) -> GpuSnapshot { + let mut snap = GpuSnapshot::default(); + let mut fb_free: f32 = 0.0; + #[allow(clippy::cast_possible_truncation)] + for s in samples { + match s.name.as_str() { + "dcgm_gpu_utilization" => snap.utilization_percent = s.value as f32, + "dcgm_fb_used" => snap.memory_used_mb = s.value as f32, + "dcgm_fb_free" => fb_free = s.value as f32, + "dcgm_gpu_temp" => snap.temperature_celsius = s.value as f32, + "dcgm_power_usage" => snap.power_watts = s.value as f32, + _ => {} + } + } + // dcgm_fb_free + dcgm_fb_used = total + snap.memory_total_mb = snap.memory_used_mb + fb_free; + snap +} + +// ============================================================================ +// Tests (ported from monitoring_service/src/service.rs) +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_handler_creation() { + let handler = MonitoringServiceHandler::new("http://localhost:9090", 3); + assert_eq!(handler.default_interval, 3); + } + + #[test] + fn test_group_empty_samples() { + let sessions = group_into_sessions(&[], ""); + assert!(sessions.is_empty()); + } + + #[test] + fn test_group_with_filter() { + let samples = vec![ + MetricSample { + name: "foxhunt_training_current_epoch".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 5.0, + }, + MetricSample { + name: "foxhunt_training_current_epoch".to_owned(), + model: "ppo".to_owned(), + fold: "0".to_owned(), + value: 3.0, + }, + ]; + let sessions = group_into_sessions(&samples, "dqn"); + assert_eq!(sessions.len(), 1); + assert_eq!(sessions.first().map(|s| s.model.as_str()), Some("dqn")); + assert_eq!(sessions.first().map(|s| s.current_epoch), Some(5.0)); + } + + #[test] + fn test_group_sets_hyperopt_flag() { + let samples = vec![MetricSample { + name: "foxhunt_hyperopt_mode".to_owned(), + model: "dqn".to_owned(), + fold: "".to_owned(), + value: 1.0, + }]; + let sessions = group_into_sessions(&samples, ""); + assert_eq!(sessions.len(), 1); + assert!(sessions.first().map(|s| s.is_hyperopt).unwrap_or(false)); + } + + #[test] + fn test_group_new_diagnostic_metrics() { + let samples = vec![ + MetricSample { + name: "foxhunt_training_current_epoch".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 25.0, + }, + MetricSample { + name: "foxhunt_training_q_value_mean".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 12.3, + }, + MetricSample { + name: "foxhunt_training_q_value_max".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 45.6, + }, + MetricSample { + name: "foxhunt_training_gradient_norm".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 1.23, + }, + MetricSample { + name: "foxhunt_training_learning_rate".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 0.001, + }, + MetricSample { + name: "foxhunt_training_epoch_duration_seconds".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 12.5, + }, + MetricSample { + name: "foxhunt_training_replay_buffer_size".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 50000.0, + }, + ]; + let sessions = group_into_sessions(&samples, ""); + assert_eq!(sessions.len(), 1); + let s = &sessions[0]; + assert_eq!(s.current_epoch, 25.0); + assert!((s.q_value_mean - 12.3).abs() < 0.01); + assert!((s.q_value_max - 45.6).abs() < 0.01); + assert!((s.gradient_norm - 1.23).abs() < 0.01); + assert!((s.learning_rate - 0.001).abs() < 0.0001); + assert!((s.epoch_duration_seconds - 12.5).abs() < 0.01); + assert_eq!(s.replay_buffer_size, 50000); + } + + #[test] + fn test_group_ppo_diagnostics() { + let samples = vec![ + MetricSample { + name: "foxhunt_training_policy_entropy".to_owned(), + model: "ppo".to_owned(), + fold: "0".to_owned(), + value: 1.23, + }, + MetricSample { + name: "foxhunt_training_kl_divergence".to_owned(), + model: "ppo".to_owned(), + fold: "0".to_owned(), + value: 0.008, + }, + MetricSample { + name: "foxhunt_training_advantage_mean".to_owned(), + model: "ppo".to_owned(), + fold: "0".to_owned(), + value: 0.001, + }, + ]; + let sessions = group_into_sessions(&samples, ""); + assert_eq!(sessions.len(), 1); + let s = &sessions[0]; + assert!((s.policy_entropy - 1.23).abs() < 0.01); + assert!((s.kl_divergence - 0.008).abs() < 0.001); + assert!((s.advantage_mean - 0.001).abs() < 0.001); + } + + #[test] + fn test_group_hyperopt_intra_trial() { + let samples = vec![ + MetricSample { + name: "foxhunt_hyperopt_mode".to_owned(), + model: "dqn".to_owned(), + fold: "".to_owned(), + value: 1.0, + }, + MetricSample { + name: "foxhunt_hyperopt_trial_epoch".to_owned(), + model: "dqn".to_owned(), + fold: "".to_owned(), + value: 12.0, + }, + MetricSample { + name: "foxhunt_hyperopt_trial_best_loss".to_owned(), + model: "dqn".to_owned(), + fold: "".to_owned(), + value: 0.042, + }, + MetricSample { + name: "foxhunt_hyperopt_elapsed_seconds".to_owned(), + model: "dqn".to_owned(), + fold: "".to_owned(), + value: 123.4, + }, + ]; + let sessions = group_into_sessions(&samples, ""); + assert_eq!(sessions.len(), 1); + let s = &sessions[0]; + assert!(s.is_hyperopt); + assert_eq!(s.hyperopt_trial_epoch, 12); + assert!((s.hyperopt_trial_best_loss - 0.042).abs() < 0.001); + assert!((s.hyperopt_elapsed_seconds - 123.4).abs() < 0.1); + } + + #[test] + fn test_group_financial_metrics() { + let samples = vec![ + MetricSample { + name: "foxhunt_training_epoch_sharpe".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 2.31, + }, + MetricSample { + name: "foxhunt_training_epoch_win_rate".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 0.552, + }, + MetricSample { + name: "foxhunt_training_epoch_max_drawdown".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 0.081, + }, + MetricSample { + name: "foxhunt_training_epoch_action_buy_pct".to_owned(), + model: "dqn".to_owned(), + fold: "0".to_owned(), + value: 0.35, + }, + ]; + let sessions = group_into_sessions(&samples, ""); + assert_eq!(sessions.len(), 1); + let s = &sessions[0]; + assert!((s.epoch_sharpe - 2.31).abs() < 0.01); + assert!((s.epoch_win_rate - 0.552).abs() < 0.001); + assert!((s.epoch_max_drawdown - 0.081).abs() < 0.001); + assert!((s.action_buy_pct - 0.35).abs() < 0.01); + } + + #[test] + fn test_build_gpu_snapshot() { + let samples = vec![ + MetricSample { + name: "dcgm_gpu_utilization".to_owned(), + model: String::new(), + fold: String::new(), + value: 87.0, + }, + MetricSample { + name: "dcgm_fb_used".to_owned(), + model: String::new(), + fold: String::new(), + value: 38200.0, + }, + MetricSample { + name: "dcgm_fb_free".to_owned(), + model: String::new(), + fold: String::new(), + value: 9800.0, + }, + MetricSample { + name: "dcgm_gpu_temp".to_owned(), + model: String::new(), + fold: String::new(), + value: 62.0, + }, + MetricSample { + name: "dcgm_power_usage".to_owned(), + model: String::new(), + fold: String::new(), + value: 245.0, + }, + ]; + let snap = build_gpu_snapshot(&samples); + assert_eq!(snap.utilization_percent, 87.0); + assert_eq!(snap.memory_used_mb, 38200.0); + assert_eq!(snap.memory_total_mb, 48000.0); + assert_eq!(snap.temperature_celsius, 62.0); + assert_eq!(snap.power_watts, 245.0); + } +} diff --git a/services/api/src/grpc/pods_handler.rs b/services/api/src/grpc/pods_handler.rs new file mode 100644 index 000000000..8fdc144fe --- /dev/null +++ b/services/api/src/grpc/pods_handler.rs @@ -0,0 +1,114 @@ +//! Kubernetes pod listing for the Pods cockpit. +//! +//! Queries the in-cluster Kubernetes API (via `kube` crate) to list pods in a +//! given namespace, converting them into proto `PodInfo` messages. The handler +//! is called periodically by the `subscribe_cluster_pods` streaming RPC. + +use anyhow::Result; +use k8s_openapi::api::core::v1::Pod; +use kube::{Api, Client}; + +/// Fetch all pods in a namespace and convert to proto `PodInfo`. +pub async fn list_pods(namespace: &str) -> Result> { + let client = Client::try_default().await?; + let pods: Api = Api::namespaced(client, namespace); + let pod_list = pods.list(&Default::default()).await?; + + let mut infos = Vec::with_capacity(pod_list.items.len()); + for pod in pod_list.items { + let meta = &pod.metadata; + let spec = pod.spec.as_ref(); + let status = pod.status.as_ref(); + + let name = meta.name.clone().unwrap_or_default(); + let labels = meta.labels.as_ref(); + let service = labels + .and_then(|l| l.get("app.kubernetes.io/name")) + .cloned() + .unwrap_or_default(); + + let phase = status + .and_then(|s| s.phase.clone()) + .unwrap_or_else(|| "Unknown".into()); + + let ready = status + .and_then(|s| s.conditions.as_ref()) + .map(|conds| { + conds + .iter() + .any(|c| c.type_ == "Ready" && c.status == "True") + }) + .unwrap_or(false); + + let restarts: i32 = status + .and_then(|s| s.container_statuses.as_ref()) + .map(|cs| cs.iter().map(|c| c.restart_count).sum()) + .unwrap_or(0); + + let node = spec + .and_then(|s| s.node_name.clone()) + .unwrap_or_default(); + + let age = meta + .creation_timestamp + .as_ref() + .map(|t| format_age(&t.0)) + .unwrap_or_else(|| "?".into()); + + // Try to get version from FOXHUNT_BUILD_VERSION env var in first container + let version = spec + .and_then(|s| s.containers.first()) + .and_then(|c| c.env.as_ref()) + .and_then(|envs| { + envs.iter() + .find(|e| e.name == "FOXHUNT_BUILD_VERSION") + .and_then(|e| e.value.clone()) + }) + .unwrap_or_default(); + + infos.push(crate::monitoring::PodInfo { + name, + service, + namespace: namespace.into(), + status: phase, + restarts, + age, + node: short_node_name(&node), + ready, + version, + }); + } + + infos.sort_by(|a, b| (&a.service, &a.name).cmp(&(&b.service, &b.name))); + Ok(infos) +} + +/// Shorten Scaleway node names: "scw-foxhunt-pool-default-abc123" -> "pool-default-abc123" +fn short_node_name(full: &str) -> String { + full.strip_prefix("scw-foxhunt-") + .unwrap_or(full) + .split('-') + .take(3) + .collect::>() + .join("-") + .chars() + .take(20) + .collect() +} + +/// Format a duration since `created` as a human-readable age string. +/// +/// k8s-openapi 0.27+ uses `jiff::Timestamp` for `creation_timestamp`. +fn format_age(created: &k8s_openapi::jiff::Timestamp) -> String { + let now = k8s_openapi::jiff::Timestamp::now(); + let secs = now.as_second() - created.as_second(); + if secs < 60 { + format!("{secs}s") + } else if secs < 3600 { + format!("{}m", secs / 60) + } else if secs < 86400 { + format!("{}h{}m", secs / 3600, (secs % 3600) / 60) + } else { + format!("{}d{}h", secs / 86400, (secs % 86400) / 3600) + } +} diff --git a/services/api/src/grpc/risk_proxy.rs b/services/api/src/grpc/risk_proxy.rs new file mode 100644 index 000000000..25a15ddda --- /dev/null +++ b/services/api/src/grpc/risk_proxy.rs @@ -0,0 +1,230 @@ +//! Risk Service Proxy - Zero-copy gRPC forwarding for risk.RiskService +//! +//! Forwards to trading-service backend (risk runs in the same process). + +use futures::Stream; +use std::pin::Pin; +use std::time::Duration; +use tonic::{Request, Response, Status}; +use tracing::{error, instrument}; + +use crate::risk::risk_service_client::RiskServiceClient; +use crate::risk::risk_service_server::RiskService; +use crate::risk::{ + EmergencyStopRequest, EmergencyStopResponse, GetCircuitBreakerStatusRequest, + GetCircuitBreakerStatusResponse, GetPositionRiskRequest, GetPositionRiskResponse, + GetRiskMetricsRequest, GetRiskMetricsResponse, GetVaRRequest, GetVaRResponse, + RiskAlertEvent, StreamCircuitBreakerStatusRequest, StreamRiskAlertsRequest, + StreamRiskMetricsRequest, StreamVaRRequest, VaREvent, ValidateOrderRequest, + ValidateOrderResponse, +}; + +#[derive(Debug, Clone)] +pub struct RiskServiceProxy { + client: RiskServiceClient, +} + +impl RiskServiceProxy { + pub fn new(client: RiskServiceClient) -> Self { + Self { client } + } +} + +#[tonic::async_trait] +impl RiskService for RiskServiceProxy { + type StreamVaRUpdatesStream = Pin> + Send>>; + type StreamRiskAlertsStream = + Pin> + Send>>; + type StreamCircuitBreakerStatusStream = + Pin> + Send>>; + type StreamRiskMetricsStream = + Pin> + Send>>; + + #[instrument(skip(self, request), err)] + async fn get_va_r( + &self, + request: Request, + ) -> Result, Status> { + self.client.clone().get_va_r(request).await.map_err(|e| { + error!("Backend GetVaR failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn stream_va_r_updates( + &self, + request: Request, + ) -> Result, Status> { + let stream = self + .client + .clone() + .stream_va_r_updates(request) + .await + .map_err(|e| { + error!("Backend StreamVaRUpdates failed: {}", e); + e + })?; + Ok(Response::new(Box::pin(stream.into_inner()))) + } + + #[instrument(skip(self, request), err)] + async fn get_position_risk( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_position_risk(request) + .await + .map_err(|e| { + error!("Backend GetPositionRisk failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn validate_order( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .validate_order(request) + .await + .map_err(|e| { + error!("Backend ValidateOrder failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_risk_metrics( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_risk_metrics(request) + .await + .map_err(|e| { + error!("Backend GetRiskMetrics failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn stream_risk_alerts( + &self, + request: Request, + ) -> Result, Status> { + let stream = self + .client + .clone() + .stream_risk_alerts(request) + .await + .map_err(|e| { + error!("Backend StreamRiskAlerts failed: {}", e); + e + })?; + Ok(Response::new(Box::pin(stream.into_inner()))) + } + + #[instrument(skip(self, request), err)] + async fn emergency_stop( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .emergency_stop(request) + .await + .map_err(|e| { + error!("Backend EmergencyStop failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_circuit_breaker_status( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_circuit_breaker_status(request) + .await + .map_err(|e| { + error!("Backend GetCircuitBreakerStatus failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn stream_circuit_breaker_status( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let interval_secs = if req.interval_seconds == 0 { + 2 + } else { + req.interval_seconds.clamp(1, 60) + }; + let symbol = req.symbol; + let mut client = self.client.clone(); + + let stream = async_stream::stream! { + let mut interval = tokio::time::interval(Duration::from_secs(u64::from(interval_secs))); + loop { + interval.tick().await; + match client.get_circuit_breaker_status(Request::new(GetCircuitBreakerStatusRequest { + symbol: symbol.clone(), + })).await { + Ok(resp) => yield Ok(resp.into_inner()), + Err(e) => { + error!("Backend GetCircuitBreakerStatus failed: {}", e); + yield Err(e); + break; + } + } + } + }; + + Ok(Response::new(Box::pin(stream))) + } + + #[instrument(skip(self, request), err)] + async fn stream_risk_metrics( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let interval_secs = if req.interval_seconds == 0 { + 3 + } else { + req.interval_seconds.clamp(1, 60) + }; + let portfolio_id = req.portfolio_id; + let mut client = self.client.clone(); + + let stream = async_stream::stream! { + let mut interval = tokio::time::interval(Duration::from_secs(u64::from(interval_secs))); + loop { + interval.tick().await; + match client.get_risk_metrics(Request::new(GetRiskMetricsRequest { + portfolio_id: portfolio_id.clone(), + })).await { + Ok(resp) => yield Ok(resp.into_inner()), + Err(e) => { + error!("Backend GetRiskMetrics failed: {}", e); + yield Err(e); + break; + } + } + } + }; + + Ok(Response::new(Box::pin(stream))) + } +} diff --git a/services/api/src/grpc/server.rs b/services/api/src/grpc/server.rs new file mode 100644 index 000000000..04becc63b --- /dev/null +++ b/services/api/src/grpc/server.rs @@ -0,0 +1,443 @@ +//! gRPC Server Setup with Circuit Breakers and Connection Pooling +//! +//! This module provides utilities for setting up backend service clients +//! with circuit breakers, connection pooling, and health checking. + +use anyhow::{Context, Result}; +use std::time::Duration; +use tonic::transport::{Certificate, Channel, ClientTlsConfig, Endpoint, Identity}; +use tracing::{error, info}; + +use super::ml_training_proxy::MlTrainingProxy; +use super::trading_agent_proxy::TradingAgentProxy; +use crate::ml_training::ml_training_service_client::MlTrainingServiceClient; +use crate::trading_agent::trading_agent_service_client::TradingAgentServiceClient; + +/// Configuration for ML Training Service backend +#[derive(Debug, Clone)] +pub struct MlTrainingBackendConfig { + /// Backend service address (e.g., "http://ml-training-service:50053") + pub address: String, + /// Connection timeout in milliseconds + pub connect_timeout_ms: u64, + /// Request timeout in milliseconds + pub request_timeout_ms: u64, + /// Circuit breaker: consecutive failures before opening + pub circuit_breaker_failures: u64, + /// Circuit breaker: reset timeout in seconds + pub circuit_breaker_reset_secs: u64, + /// TLS CA certificate path (optional, for HTTPS) + pub tls_ca_cert_path: Option, + /// TLS client certificate path (optional, for mTLS) + pub tls_client_cert_path: Option, + /// TLS client key path (optional, for mTLS) + pub tls_client_key_path: Option, +} + +impl Default for MlTrainingBackendConfig { + fn default() -> Self { + Self { + address: "http://localhost:50053".to_string(), + connect_timeout_ms: 5000, + request_timeout_ms: 30000, + circuit_breaker_failures: 5, + circuit_breaker_reset_secs: 30, + tls_ca_cert_path: None, + tls_client_cert_path: None, + tls_client_key_path: None, + } + } +} + +/// Setup ML Training Service client with circuit breaker and connection pooling +/// +/// # Arguments +/// * `config` - Backend service configuration +/// +/// # Returns +/// * Configured ML Training Service client with circuit breaker +/// +/// # Performance +/// - Connection pooling via tonic::transport::Channel (shared Arc) +/// +/// - Circuit breaker with <10μs overhead per request +/// - Zero-copy request forwarding +pub async fn setup_ml_training_client( + config: MlTrainingBackendConfig, +) -> Result> { + info!( + "Setting up ML Training Service client for {}", + config.address + ); + + // Parse and configure endpoint + let mut endpoint = Endpoint::from_shared(config.address.clone())? + .connect_timeout(Duration::from_millis(config.connect_timeout_ms)) + .timeout(Duration::from_millis(config.request_timeout_ms)) + .tcp_keepalive(Some(Duration::from_secs(60))) + .http2_keep_alive_interval(Duration::from_secs(30)) + .keep_alive_while_idle(true); + + // Configure TLS if HTTPS URL and certificate paths provided + if config.address.starts_with("https://") { + match ( + &config.tls_ca_cert_path, + &config.tls_client_cert_path, + &config.tls_client_key_path, + ) { + (Some(ca_path), Some(cert_path), Some(key_path)) => { + info!("Configuring TLS with mTLS (client certificates)"); + info!(" CA cert: {}", ca_path); + info!(" Client cert: {}", cert_path); + info!(" Client key: {}", key_path); + + // Load certificates from files + info!("Reading CA certificate..."); + let ca_pem = tokio::fs::read_to_string(ca_path).await.context(format!( + "Failed to read ML Training TLS CA cert at {}", + ca_path + ))?; + info!("CA certificate loaded ({} bytes)", ca_pem.len()); + + info!("Reading client certificate..."); + let client_cert_pem = + tokio::fs::read_to_string(cert_path).await.context(format!( + "Failed to read ML Training TLS client cert at {}", + cert_path + ))?; + info!( + "Client certificate loaded ({} bytes)", + client_cert_pem.len() + ); + + info!("Reading client key..."); + let client_key_pem = tokio::fs::read_to_string(key_path).await.context(format!( + "Failed to read ML Training TLS client key at {}", + key_path + ))?; + info!("Client key loaded ({} bytes)", client_key_pem.len()); + + // Extract hostname from backend URL for SNI (Server Name Indication) + // Use "foxhunt-services" to match the certificate CN + let hostname = if let Some(host) = config.address.strip_prefix("https://") { + host.split(':').next().unwrap_or("foxhunt-services") + } else { + "foxhunt-services" + }; + + // Create TLS configuration with mTLS + let tls_config = ClientTlsConfig::new() + .ca_certificate(Certificate::from_pem(&ca_pem)) + .identity(Identity::from_pem(&client_cert_pem, &client_key_pem)) + .domain_name(hostname); // Use actual hostname for SNI + + info!("TLS SNI hostname: {}", hostname); + + endpoint = endpoint + .tls_config(tls_config) + .context("Failed to apply ML Training TLS configuration")?; + info!("TLS configuration with mTLS applied successfully"); + }, + (Some(ca_path), None, None) => { + info!("Configuring TLS with server verification only (no client cert)"); + let ca_pem = tokio::fs::read_to_string(ca_path).await.context(format!( + "Failed to read ML Training TLS CA cert at {}", + ca_path + ))?; + + let tls_config = ClientTlsConfig::new() + .ca_certificate(Certificate::from_pem(&ca_pem)) + .domain_name("foxhunt-services"); // Match server cert CN + + endpoint = endpoint + .tls_config(tls_config) + .context("Failed to apply ML Training TLS configuration")?; + info!("TLS configuration with server verification applied successfully"); + }, + _ => { + error!("HTTPS URL provided but certificate paths incomplete - connection may fail"); + error!( + " CA: {:?}, Client cert: {:?}, Client key: {:?}", + config.tls_ca_cert_path, + config.tls_client_cert_path, + config.tls_client_key_path + ); + }, + } + } else { + info!("Using HTTP (no TLS) for ML Training Service connection"); + } + + info!("Connecting to ML Training Service at {}...", config.address); + + // Establish connection (connection pool managed by Channel) + let channel = endpoint.connect().await.map_err(|e| { + error!("Failed to connect to ML Training Service: {}", e); + anyhow::anyhow!("ML Training Service connection failed: {}", e) + })?; + + info!("✓ Connected to ML Training Service"); + + // Note: Circuit breaker configuration is stored but not applied yet + // Full implementation requires tower-layer custom circuit breaker + info!( + "Circuit breaker config: {} failures, {}s reset (to be implemented)", + config.circuit_breaker_failures, config.circuit_breaker_reset_secs + ); + + // Create client from channel + let client = MlTrainingServiceClient::new(channel); + + info!("✓ ML Training Service client ready"); + + Ok(client) +} + +/// Setup ML Training Service proxy +/// +/// # Arguments +/// * `config` - Backend service configuration +/// +/// # Returns +/// * ML Training Service proxy ready for serving +pub async fn setup_ml_training_proxy(config: MlTrainingBackendConfig) -> Result { + info!("Setting up ML Training Service proxy..."); + + // Setup client with circuit breaker + let client = setup_ml_training_client(config).await?; + + // Create proxy + let proxy = MlTrainingProxy::new(client); + + info!("✓ ML Training Service proxy initialized"); + + Ok(proxy) +} + + +/// Configuration for Trading Agent Service backend +#[derive(Debug, Clone)] +pub struct TradingAgentBackendConfig { + /// Backend service address (e.g., "http://trading-agent-service:50055") + pub address: String, + /// Connection timeout in milliseconds + pub connect_timeout_ms: u64, + /// Request timeout in milliseconds + pub request_timeout_ms: u64, + /// Circuit breaker: consecutive failures before opening + pub circuit_breaker_failures: u64, + /// Circuit breaker: reset timeout in seconds + pub circuit_breaker_reset_secs: u64, + /// TLS CA certificate path (optional, for HTTPS) + pub tls_ca_cert_path: Option, + /// TLS client certificate path (optional, for mTLS) + pub tls_client_cert_path: Option, + /// TLS client key path (optional, for mTLS) + pub tls_client_key_path: Option, +} + +impl Default for TradingAgentBackendConfig { + fn default() -> Self { + Self { + address: "http://localhost:50055".to_string(), + connect_timeout_ms: 5000, + request_timeout_ms: 30000, + circuit_breaker_failures: 5, + circuit_breaker_reset_secs: 30, + tls_ca_cert_path: None, + tls_client_cert_path: None, + tls_client_key_path: None, + } + } +} + +/// Setup Trading Agent Service client with circuit breaker and connection pooling +/// +/// # Arguments +/// * `config` - Backend service configuration +/// +/// # Returns +/// * Configured Trading Agent Service client with circuit breaker +/// +/// # Performance +/// - Connection pooling via tonic::transport::Channel (shared Arc) +/// - Circuit breaker with <10μs overhead per request +/// - Zero-copy request forwarding +pub async fn setup_trading_agent_client( + config: TradingAgentBackendConfig, +) -> Result> { + info!( + "Setting up Trading Agent Service client for {}", + config.address + ); + + // Parse and configure endpoint + let mut endpoint = Endpoint::from_shared(config.address.clone())? + .connect_timeout(Duration::from_millis(config.connect_timeout_ms)) + .timeout(Duration::from_millis(config.request_timeout_ms)) + .tcp_keepalive(Some(Duration::from_secs(60))) + .http2_keep_alive_interval(Duration::from_secs(30)) + .keep_alive_while_idle(true); + + // Configure TLS if HTTPS URL and certificate paths provided + if config.address.starts_with("https://") { + match ( + &config.tls_ca_cert_path, + &config.tls_client_cert_path, + &config.tls_client_key_path, + ) { + (Some(ca_path), Some(cert_path), Some(key_path)) => { + info!("Configuring TLS with mTLS (client certificates)"); + info!(" CA cert: {}", ca_path); + info!(" Client cert: {}", cert_path); + info!(" Client key: {}", key_path); + + // Load certificates from files + info!("Reading CA certificate..."); + let ca_pem = tokio::fs::read_to_string(ca_path).await.context(format!( + "Failed to read Trading Agent TLS CA cert at {}", + ca_path + ))?; + info!("CA certificate loaded ({} bytes)", ca_pem.len()); + + info!("Reading client certificate..."); + let client_cert_pem = + tokio::fs::read_to_string(cert_path).await.context(format!( + "Failed to read Trading Agent TLS client cert at {}", + cert_path + ))?; + info!( + "Client certificate loaded ({} bytes)", + client_cert_pem.len() + ); + + info!("Reading client key..."); + let client_key_pem = tokio::fs::read_to_string(key_path).await.context(format!( + "Failed to read Trading Agent TLS client key at {}", + key_path + ))?; + info!("Client key loaded ({} bytes)", client_key_pem.len()); + + // Extract hostname from backend URL for SNI (Server Name Indication) + let hostname = if let Some(host) = config.address.strip_prefix("https://") { + host.split(':').next().unwrap_or("foxhunt-services") + } else { + "foxhunt-services" + }; + + // Create TLS configuration with mTLS + let tls_config = ClientTlsConfig::new() + .ca_certificate(Certificate::from_pem(&ca_pem)) + .identity(Identity::from_pem(&client_cert_pem, &client_key_pem)) + .domain_name(hostname); + + info!("TLS SNI hostname: {}", hostname); + + endpoint = endpoint + .tls_config(tls_config) + .context("Failed to apply Trading Agent TLS configuration")?; + info!("TLS configuration with mTLS applied successfully"); + }, + (Some(ca_path), None, None) => { + info!("Configuring TLS with server verification only (no client cert)"); + let ca_pem = tokio::fs::read_to_string(ca_path).await.context(format!( + "Failed to read Trading Agent TLS CA cert at {}", + ca_path + ))?; + + let tls_config = ClientTlsConfig::new() + .ca_certificate(Certificate::from_pem(&ca_pem)) + .domain_name("foxhunt-services"); + + endpoint = endpoint + .tls_config(tls_config) + .context("Failed to apply Trading Agent TLS configuration")?; + info!("TLS configuration with server verification applied successfully"); + }, + _ => { + error!("HTTPS URL provided but certificate paths incomplete - connection may fail"); + error!( + " CA: {:?}, Client cert: {:?}, Client key: {:?}", + config.tls_ca_cert_path, + config.tls_client_cert_path, + config.tls_client_key_path + ); + }, + } + } else { + info!("Using HTTP (no TLS) for Trading Agent Service connection"); + } + + info!( + "Connecting to Trading Agent Service at {}...", + config.address + ); + + // Establish connection (connection pool managed by Channel) + let channel = endpoint.connect().await.map_err(|e| { + error!("Failed to connect to Trading Agent Service: {}", e); + anyhow::anyhow!("Trading Agent Service connection failed: {}", e) + })?; + + info!("✓ Connected to Trading Agent Service"); + + // Note: Circuit breaker configuration is stored but not applied yet + info!( + "Circuit breaker config: {} failures, {}s reset (to be implemented)", + config.circuit_breaker_failures, config.circuit_breaker_reset_secs + ); + + // Create client from channel + let client = TradingAgentServiceClient::new(channel); + + info!("✓ Trading Agent Service client ready"); + + Ok(client) +} + +/// Setup Trading Agent Service proxy +/// +/// # Arguments +/// * `config` - Backend service configuration +/// +/// # Returns +/// * Trading Agent Service proxy ready for serving +pub async fn setup_trading_agent_proxy( + config: TradingAgentBackendConfig, +) -> Result { + info!("Setting up Trading Agent Service proxy..."); + + // Setup client with circuit breaker + let client = setup_trading_agent_client(config).await?; + + // Create proxy + let proxy = TradingAgentProxy::new(client); + + info!("✓ Trading Agent Service proxy initialized"); + + Ok(proxy) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_default_config() { + let config = MlTrainingBackendConfig::default(); + assert_eq!(config.address, "http://localhost:50053"); + assert_eq!(config.connect_timeout_ms, 5000); + assert_eq!(config.circuit_breaker_failures, 5); + } + + #[tokio::test] + async fn test_client_setup_invalid_address() { + let config = MlTrainingBackendConfig { + address: "invalid://address".to_string(), + ..Default::default() + }; + + let result = setup_ml_training_client(config).await; + assert!(result.is_err()); + } +} diff --git a/services/api/src/grpc/trading_agent_proxy.rs b/services/api/src/grpc/trading_agent_proxy.rs new file mode 100644 index 000000000..3cff1a7e8 --- /dev/null +++ b/services/api/src/grpc/trading_agent_proxy.rs @@ -0,0 +1,551 @@ +//! Trading Agent Service Proxy - Zero-copy gRPC forwarding for trading agent operations +//! +//! This module implements a high-performance proxy for the Trading Agent Service with: +//! - Zero-copy message forwarding (routing overhead <10μs) +//! - Connection pooling via tonic::transport::Channel +//! - Circuit breaker integration for backend failures +//! - Efficient streaming support for agent activity events +//! - Health checking integration + +use std::pin::Pin; +use std::time::Duration; + +use futures::Stream; +use tonic::{Request, Response, Status}; +use tracing::{error, info, instrument, warn}; + +// Import the generated Trading Agent service protobuf definitions from lib.rs +use crate::trading_agent::trading_agent_service_client::TradingAgentServiceClient; +use crate::trading_agent::trading_agent_service_server::{ + TradingAgentService, TradingAgentServiceServer, +}; +use crate::trading_agent::{ + AgentActivityEvent, + // Portfolio Allocation + AllocatePortfolioRequest, + AllocatePortfolioResponse, + // Order Generation + GenerateOrdersRequest, + GenerateOrdersResponse, + GetAgentPerformanceRequest, + GetAgentPerformanceResponse, + + // Agent Monitoring + GetAgentStatusRequest, + GetAgentStatusResponse, + GetAllocationRequest, + GetAllocationResponse, + GetSelectedAssetsRequest, + GetSelectedAssetsResponse, + + GetUniverseRequest, + GetUniverseResponse, + // Service Health + HealthCheckRequest, + HealthCheckResponse, + ListStrategiesRequest, + ListStrategiesResponse, + RebalancePortfolioRequest, + RebalancePortfolioResponse, + + // Strategy Coordination + RegisterStrategyRequest, + RegisterStrategyResponse, + // Asset Selection + SelectAssetsRequest, + SelectAssetsResponse, + // Universe Management + SelectUniverseRequest, + SelectUniverseResponse, + StreamAgentActivityRequest, + StreamAgentStatusRequest, + SubmitAgentOrdersRequest, + SubmitAgentOrdersResponse, + + UpdateStrategyStatusRequest, + UpdateStrategyStatusResponse, + + UpdateUniverseCriteriaRequest, + UpdateUniverseCriteriaResponse, +}; + +/// Trading Agent Service Proxy +/// +/// Provides zero-copy forwarding of gRPC requests to the backend Trading Agent Service. +/// +/// Uses connection pooling and circuit breakers for high availability and performance. +#[derive(Debug, Clone)] +pub struct TradingAgentProxy { + /// Backend Trading Agent Service client with connection pooling + client: TradingAgentServiceClient, +} + +impl TradingAgentProxy { + /// Create a new Trading Agent Service proxy + /// + /// # Arguments + /// * `client` - Pre-configured Trading Agent Service client with circuit breaker + /// + /// # Performance + /// - Uses Arc-based channel cloning for zero-copy client reuse + /// - Connection pooling managed by tonic::transport::Channel + pub fn new(client: TradingAgentServiceClient) -> Self { + Self { client } + } + + /// Convert proxy into a tonic server instance + pub fn into_server(self) -> TradingAgentServiceServer { + TradingAgentServiceServer::new(self) + } +} + +#[tonic::async_trait] +impl TradingAgentService for TradingAgentProxy { + /// Server streaming type for agent activity events + type StreamAgentActivityStream = + Pin> + Send>>; + /// Server streaming type for poll-based agent status + type StreamAgentStatusStream = + Pin> + Send>>; + + // ===== Universe Management Methods ===== + + /// Select tradable universe based on liquidity, volatility, and ML signals + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn select_universe( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying SelectUniverse request"); + + let mut client = self.client.clone(); + let response = client.select_universe(request).await.map_err(|e| { + error!("Backend SelectUniverse failed: {}", e); + e + })?; + + info!("SelectUniverse request forwarded successfully"); + Ok(response) + } + + /// Get current trading universe configuration + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn get_universe( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying GetUniverse request"); + + let mut client = self.client.clone(); + let response = client.get_universe(request).await.map_err(|e| { + error!("Backend GetUniverse failed: {}", e); + e + })?; + + info!("GetUniverse request forwarded successfully"); + Ok(response) + } + + /// Update universe selection criteria + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn update_universe_criteria( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying UpdateUniverseCriteria request"); + + let mut client = self.client.clone(); + let response = client + .update_universe_criteria(request) + .await + .map_err(|e| { + error!("Backend UpdateUniverseCriteria failed: {}", e); + e + })?; + + info!("UpdateUniverseCriteria request forwarded successfully"); + Ok(response) + } + + // ===== Asset Selection Methods ===== + + /// Select specific assets to trade within universe + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn select_assets( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying SelectAssets request"); + + let mut client = self.client.clone(); + let response = client.select_assets(request).await.map_err(|e| { + error!("Backend SelectAssets failed: {}", e); + e + })?; + + info!("SelectAssets request forwarded successfully"); + Ok(response) + } + + /// Get current asset selection with scores + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn get_selected_assets( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying GetSelectedAssets request"); + + let mut client = self.client.clone(); + let response = client.get_selected_assets(request).await.map_err(|e| { + error!("Backend GetSelectedAssets failed: {}", e); + e + })?; + + info!("GetSelectedAssets request forwarded successfully"); + Ok(response) + } + + // ===== Portfolio Allocation Methods ===== + + /// Allocate capital across selected assets + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn allocate_portfolio( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying AllocatePortfolio request"); + + let mut client = self.client.clone(); + let response = client.allocate_portfolio(request).await.map_err(|e| { + error!("Backend AllocatePortfolio failed: {}", e); + e + })?; + + info!("AllocatePortfolio request forwarded successfully"); + Ok(response) + } + + /// Get current portfolio allocation + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn get_allocation( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying GetAllocation request"); + + let mut client = self.client.clone(); + let response = client.get_allocation(request).await.map_err(|e| { + error!("Backend GetAllocation failed: {}", e); + e + })?; + + info!("GetAllocation request forwarded successfully"); + Ok(response) + } + + /// Rebalance portfolio based on target allocation + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn rebalance_portfolio( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying RebalancePortfolio request"); + + let mut client = self.client.clone(); + let response = client.rebalance_portfolio(request).await.map_err(|e| { + error!("Backend RebalancePortfolio failed: {}", e); + e + })?; + + info!("RebalancePortfolio request forwarded successfully"); + Ok(response) + } + + // ===== Order Generation Methods ===== + + /// Generate orders based on allocation and ML signals + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn generate_orders( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying GenerateOrders request"); + + let mut client = self.client.clone(); + let response = client.generate_orders(request).await.map_err(|e| { + error!("Backend GenerateOrders failed: {}", e); + e + })?; + + info!("GenerateOrders request forwarded successfully"); + Ok(response) + } + + /// Submit generated orders to Trading Service + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn submit_agent_orders( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying SubmitAgentOrders request"); + + let mut client = self.client.clone(); + let response = client.submit_agent_orders(request).await.map_err(|e| { + error!("Backend SubmitAgentOrders failed: {}", e); + e + })?; + + info!("SubmitAgentOrders request forwarded successfully"); + Ok(response) + } + + // ===== Strategy Coordination Methods ===== + + /// Register a trading strategy with the agent + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn register_strategy( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying RegisterStrategy request"); + + let mut client = self.client.clone(); + let response = client.register_strategy(request).await.map_err(|e| { + error!("Backend RegisterStrategy failed: {}", e); + e + })?; + + info!("RegisterStrategy request forwarded successfully"); + Ok(response) + } + + /// Get list of active strategies + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn list_strategies( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying ListStrategies request"); + + let mut client = self.client.clone(); + let response = client.list_strategies(request).await.map_err(|e| { + error!("Backend ListStrategies failed: {}", e); + e + })?; + + info!("ListStrategies request forwarded successfully"); + Ok(response) + } + + /// Enable/disable a strategy + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn update_strategy_status( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying UpdateStrategyStatus request"); + + let mut client = self.client.clone(); + let response = client.update_strategy_status(request).await.map_err(|e| { + error!("Backend UpdateStrategyStatus failed: {}", e); + e + })?; + + info!("UpdateStrategyStatus request forwarded successfully"); + Ok(response) + } + + // ===== Agent Monitoring Methods ===== + + /// Get comprehensive agent status and performance + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn get_agent_status( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying GetAgentStatus request"); + + let mut client = self.client.clone(); + let response = client.get_agent_status(request).await.map_err(|e| { + error!("Backend GetAgentStatus failed: {}", e); + e + })?; + + info!("GetAgentStatus request forwarded successfully"); + Ok(response) + } + + /// Stream real-time agent decisions and actions (server streaming) + /// + /// # Performance + /// - Zero-copy stream forwarding + /// - No intermediate buffering + /// - Direct stream passthrough from backend + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn stream_agent_activity( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying StreamAgentActivity streaming request"); + + let mut client = self.client.clone(); + + // Get backend stream response + let stream_response = client.stream_agent_activity(request).await.map_err(|e| { + error!("Backend StreamAgentActivity failed: {}", e); + e + })?; + + // Extract inner stream and forward directly (zero-copy) + let stream = stream_response.into_inner(); + let boxed_stream = Box::pin(stream) as Self::StreamAgentActivityStream; + + info!("StreamAgentActivity streaming request forwarded successfully"); + Ok(Response::new(boxed_stream)) + } + + /// Get agent performance metrics + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn get_agent_performance( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying GetAgentPerformance request"); + + let mut client = self.client.clone(); + let response = client.get_agent_performance(request).await.map_err(|e| { + error!("Backend GetAgentPerformance failed: {}", e); + e + })?; + + info!("GetAgentPerformance request forwarded successfully"); + Ok(response) + } + + // ===== Service Health Methods ===== + + /// Health check for Trading Agent Service backend + /// + /// # Performance + /// - Zero-copy message forwarding + /// - Routing overhead target: <10μs + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn health_check( + &self, + request: Request, + ) -> Result, Status> { + info!("Proxying HealthCheck request"); + + let mut client = self.client.clone(); + let response = client.health_check(request).await.map_err(|e| { + warn!("Backend HealthCheck failed: {}", e); + e + })?; + + info!("HealthCheck request forwarded successfully"); + Ok(response) + } + + /// Poll-to-stream adapter for agent status + /// + /// Polls GetAgentStatus at a configurable interval and yields results as a server stream. + #[instrument(skip(self, request), fields(request_id = %uuid::Uuid::new_v4()), err)] + async fn stream_agent_status( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let interval_secs = if req.interval_seconds == 0 { + 3 + } else { + req.interval_seconds.clamp(1, 60) + }; + let mut client = self.client.clone(); + + let stream = async_stream::stream! { + let mut interval = tokio::time::interval(Duration::from_secs(u64::from(interval_secs))); + loop { + interval.tick().await; + match client.get_agent_status(Request::new(GetAgentStatusRequest::default())).await { + Ok(resp) => yield Ok(resp.into_inner()), + Err(e) => { + error!("Backend GetAgentStatus failed: {}", e); + yield Err(e); + break; + } + } + } + }; + + Ok(Response::new(Box::pin(stream))) + } +} + +#[cfg(test)] +mod tests { + #[test] + fn test_proxy_creation() { + // This test validates the proxy struct can be created + // Full integration tests require running backend service + } +} diff --git a/services/api/src/grpc/trading_direct_proxy.rs b/services/api/src/grpc/trading_direct_proxy.rs new file mode 100644 index 000000000..12b5b4a81 --- /dev/null +++ b/services/api/src/grpc/trading_direct_proxy.rs @@ -0,0 +1,336 @@ +//! Trading Service Direct Proxy - Zero-copy gRPC forwarding for trading.TradingService +//! +//! Forwards requests from fxt CLI (package: trading) directly to trading-service backend. +//! Distinct from TradingServiceProxy which serves the monolithic foxhunt.tli.TradingService. + +use std::pin::Pin; +use std::time::Duration; + +use futures::Stream; +use tonic::{Request, Response, Status}; +use tracing::{error, instrument}; + +use crate::trading_backend::trading_service_client::TradingServiceClient; +use crate::trading_backend::trading_service_server::TradingService; +use crate::trading_backend::{ + CancelOrderRequest, CancelOrderResponse, ExecutionEvent, GetExecutionHistoryRequest, + GetExecutionHistoryResponse, GetOrderBookRequest, GetOrderBookResponse, + GetOrderStatusRequest, GetOrderStatusResponse, GetPositionsRequest, GetPositionsResponse, + GetPortfolioSummaryRequest, GetPortfolioSummaryResponse, GetRegimeStateRequest, + GetRegimeStateResponse, GetRegimeTransitionsRequest, GetRegimeTransitionsResponse, + MarketDataEvent, MlOrderRequest, MlOrderResponse, MlPerformanceRequest, + MlPerformanceResponse, MlPredictionsRequest, MlPredictionsResponse, OrderEvent, + PositionEvent, StreamExecutionsRequest, StreamMarketDataRequest, StreamOrderBookRequest, + StreamOrdersRequest, StreamPortfolioSummaryRequest, StreamPositionsRequest, + SubmitOrderRequest, SubmitOrderResponse, +}; + +#[derive(Debug, Clone)] +pub struct TradingDirectProxy { + client: TradingServiceClient, +} + +impl TradingDirectProxy { + pub fn new(client: TradingServiceClient) -> Self { + Self { client } + } +} + +#[tonic::async_trait] +impl TradingService for TradingDirectProxy { + type StreamOrdersStream = Pin> + Send>>; + type StreamPositionsStream = + Pin> + Send>>; + type StreamMarketDataStream = + Pin> + Send>>; + type StreamExecutionsStream = + Pin> + Send>>; + type StreamPortfolioSummaryStream = + Pin> + Send>>; + type StreamOrderBookStream = + Pin> + Send>>; + + #[instrument(skip(self, request), err)] + async fn submit_order( + &self, + request: Request, + ) -> Result, Status> { + self.client.clone().submit_order(request).await.map_err(|e| { + error!("Backend SubmitOrder failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn cancel_order( + &self, + request: Request, + ) -> Result, Status> { + self.client.clone().cancel_order(request).await.map_err(|e| { + error!("Backend CancelOrder failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_order_status( + &self, + request: Request, + ) -> Result, Status> { + self.client.clone().get_order_status(request).await.map_err(|e| { + error!("Backend GetOrderStatus failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn stream_orders( + &self, + request: Request, + ) -> Result, Status> { + let stream = self.client.clone().stream_orders(request).await.map_err(|e| { + error!("Backend StreamOrders failed: {}", e); + e + })?; + Ok(Response::new(Box::pin(stream.into_inner()))) + } + + #[instrument(skip(self, request), err)] + async fn get_positions( + &self, + request: Request, + ) -> Result, Status> { + self.client.clone().get_positions(request).await.map_err(|e| { + error!("Backend GetPositions failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn stream_positions( + &self, + request: Request, + ) -> Result, Status> { + let stream = self.client.clone().stream_positions(request).await.map_err(|e| { + error!("Backend StreamPositions failed: {}", e); + e + })?; + Ok(Response::new(Box::pin(stream.into_inner()))) + } + + #[instrument(skip(self, request), err)] + async fn get_portfolio_summary( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_portfolio_summary(request) + .await + .map_err(|e| { + error!("Backend GetPortfolioSummary failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn stream_market_data( + &self, + request: Request, + ) -> Result, Status> { + let stream = self + .client + .clone() + .stream_market_data(request) + .await + .map_err(|e| { + error!("Backend StreamMarketData failed: {}", e); + e + })?; + Ok(Response::new(Box::pin(stream.into_inner()))) + } + + #[instrument(skip(self, request), err)] + async fn get_order_book( + &self, + request: Request, + ) -> Result, Status> { + self.client.clone().get_order_book(request).await.map_err(|e| { + error!("Backend GetOrderBook failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn stream_executions( + &self, + request: Request, + ) -> Result, Status> { + let stream = self + .client + .clone() + .stream_executions(request) + .await + .map_err(|e| { + error!("Backend StreamExecutions failed: {}", e); + e + })?; + Ok(Response::new(Box::pin(stream.into_inner()))) + } + + #[instrument(skip(self, request), err)] + async fn get_execution_history( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_execution_history(request) + .await + .map_err(|e| { + error!("Backend GetExecutionHistory failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn submit_ml_order( + &self, + request: Request, + ) -> Result, Status> { + self.client.clone().submit_ml_order(request).await.map_err(|e| { + error!("Backend SubmitMLOrder failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_ml_predictions( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_ml_predictions(request) + .await + .map_err(|e| { + error!("Backend GetMLPredictions failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_ml_performance( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_ml_performance(request) + .await + .map_err(|e| { + error!("Backend GetMLPerformance failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_regime_state( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_regime_state(request) + .await + .map_err(|e| { + error!("Backend GetRegimeState failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn get_regime_transitions( + &self, + request: Request, + ) -> Result, Status> { + self.client + .clone() + .get_regime_transitions(request) + .await + .map_err(|e| { + error!("Backend GetRegimeTransitions failed: {}", e); + e + }) + } + + #[instrument(skip(self, request), err)] + async fn stream_portfolio_summary( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let interval_secs = if req.interval_seconds == 0 { + 3 + } else { + req.interval_seconds.clamp(1, 60) + }; + let account_id = req.account_id; + let mut client = self.client.clone(); + + let stream = async_stream::stream! { + let mut interval = tokio::time::interval(Duration::from_secs(u64::from(interval_secs))); + loop { + interval.tick().await; + match client.get_portfolio_summary(Request::new(GetPortfolioSummaryRequest { + account_id: account_id.clone(), + })).await { + Ok(resp) => yield Ok(resp.into_inner()), + Err(e) => { + error!("Backend GetPortfolioSummary failed: {}", e); + yield Err(e); + break; + } + } + } + }; + + Ok(Response::new(Box::pin(stream))) + } + + #[instrument(skip(self, request), err)] + async fn stream_order_book( + &self, + request: Request, + ) -> Result, Status> { + let req = request.into_inner(); + let interval_secs = if req.interval_seconds == 0 { + 1 + } else { + req.interval_seconds.clamp(1, 60) + }; + let symbol = req.symbol; + let depth = req.depth; + let mut client = self.client.clone(); + + let stream = async_stream::stream! { + let mut interval = tokio::time::interval(Duration::from_secs(u64::from(interval_secs))); + loop { + interval.tick().await; + match client.get_order_book(Request::new(GetOrderBookRequest { + symbol: symbol.clone(), + depth: if depth == 0 { None } else { Some(depth) }, + })).await { + Ok(resp) => yield Ok(resp.into_inner()), + Err(e) => { + error!("Backend GetOrderBook failed: {}", e); + yield Err(e); + break; + } + } + } + }; + + Ok(Response::new(Box::pin(stream))) + } +} diff --git a/services/api/src/grpc/trading_proxy.rs b/services/api/src/grpc/trading_proxy.rs new file mode 100644 index 000000000..9f1f13503 --- /dev/null +++ b/services/api/src/grpc/trading_proxy.rs @@ -0,0 +1,2158 @@ +//! Protocol Translation Layer for Trading Service +//! +//! Translates between TLI proto (foxhunt.tli) and Trading Service proto (trading). +//! Acts as API Gateway's adapter to bridge client-facing and backend interfaces. +//! +//! Architecture: +//! - Receives TLI proto requests from clients +//! - Translates to Trading Service proto +//! - Forwards to backend Trading Service +//! - Translates responses back to TLI proto +//! +//! Features: +//! - Zero-allocation translations where possible +//! - Circuit breaker with atomic health state +//! - Connection pooling with tonic::Channel +//! - Metadata extraction and forwarding +//! - Target: <10μs translation overhead + +use futures::Stream; +use std::collections::HashMap; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Instant; +use tokio::sync::watch; +use tonic::transport::Channel; +use tonic::{Request, Response, Status}; +use tracing::{debug, error, warn, instrument}; + +// Import TLI proto (client-facing interface) +use crate::foxhunt::tli::trading_service_server::TradingService as TliTradingService; + +// Import Trading Service backend proto +use crate::trading_backend::trading_service_client::TradingServiceClient as BackendTradingClient; + +// Import Risk Service backend proto +use crate::risk::risk_service_client::RiskServiceClient as BackendRiskClient; + +// Import Config Service backend proto +use crate::config_backend::config_service_client::ConfigServiceClient as BackendConfigClient; + +/// Health checker for backend trading service +/// +/// Uses atomic operations for lock-free health state management. +/// +/// Minimal overhead: ~1-2ns per health check (single atomic load). +#[derive(Debug)] +pub struct HealthChecker { + /// Last successful health check timestamp (nanoseconds since epoch) + last_check: Arc, + /// Current health state (true = healthy, false = unhealthy) + is_healthy: Arc, + /// Health check interval in seconds + check_interval_secs: u64, +} + +impl HealthChecker { + /// Create new health checker with specified check interval + pub fn new(check_interval_secs: u64) -> Self { + Self { + last_check: Arc::new(AtomicU64::new(0)), + is_healthy: Arc::new(AtomicBool::new(true)), // Optimistically start healthy + check_interval_secs, + } + } + + /// Check if backend is healthy (lock-free atomic read) + /// + /// Latency: ~1-2ns (single atomic load) + #[inline(always)] + pub fn is_healthy(&self) -> bool { + self.is_healthy.load(Ordering::Relaxed) + } + + /// Check if health check is needed based on interval + fn should_check(&self) -> bool { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + + let last = self.last_check.load(Ordering::Relaxed); + now.saturating_sub(last) >= self.check_interval_secs + } + + /// Perform health check against backend service + pub async fn check_health(&self, client: &mut BackendTradingClient) { + if !self.should_check() { + return; + } + + let start = Instant::now(); + + // Simple health check with GetPositions + let health_result = tokio::time::timeout( + std::time::Duration::from_millis(100), + client.get_positions(Request::new(crate::trading_backend::GetPositionsRequest { + account_id: Some("health_check".to_string()), + symbol: None, + })), + ) + .await; + + let is_healthy = health_result.is_ok(); + self.is_healthy.store(is_healthy, Ordering::Relaxed); + + let now_secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + self.last_check.store(now_secs, Ordering::Relaxed); + + let elapsed = start.elapsed(); + if is_healthy { + debug!("Health check passed in {:?}", elapsed); + } else { + warn!("Health check failed in {:?}", elapsed); + } + } + + /// Mark backend as unhealthy (used by circuit breaker) + pub fn mark_unhealthy(&self) { + self.is_healthy.store(false, Ordering::Relaxed); + } +} + +/// Per-service health entry streamed to clients via SubscribeSystemStatus. +#[derive(Debug, Clone)] +pub struct ServiceHealthEntry { + pub name: String, + pub healthy: bool, + pub message: String, +} + +/// Shared health snapshot updated by a background task and consumed by +/// `subscribe_system_status`. Uses a `watch` channel for efficient broadcast. +#[derive(Clone)] +pub struct BackendHealthState { + rx: watch::Receiver>, +} + +impl BackendHealthState { + /// Create a new health state and its update sender. + pub fn new() -> (watch::Sender>, Self) { + let (tx, rx) = watch::channel(Vec::new()); + (tx, Self { rx }) + } +} + +/// Protocol translation proxy for Trading Service +/// +/// Bridges TLI proto (client interface) and Trading Service proto (backend). +pub struct TradingServiceProxy { + /// Backend trading service client + backend_client: BackendTradingClient, + /// Backend risk service client + risk_client: BackendRiskClient, + /// Backend config service client + config_client: BackendConfigClient, + /// Health checker with circuit breaker + health_checker: Arc, + /// Shared backend health state for SubscribeSystemStatus + health_state: Option, +} + +impl TradingServiceProxy { + /// Create new trading service proxy with protocol translation + pub async fn new(backend_url: &str) -> Result> { + let channel = Channel::from_shared(backend_url.to_string())? + .connect() + .await?; + + let backend_client = BackendTradingClient::new(channel.clone()); + let risk_client = BackendRiskClient::new(channel.clone()); + let config_client = BackendConfigClient::new(channel); + + Ok(Self { + backend_client, + risk_client, + config_client, + health_checker: Arc::new(HealthChecker::new(30)), + health_state: None, + }) + } + + /// Create proxy with lazy connection + pub fn new_lazy(backend_url: &str) -> Result> { + let channel = Channel::from_shared(backend_url.to_string())?.connect_lazy(); + + let backend_client = BackendTradingClient::new(channel.clone()); + let risk_client = BackendRiskClient::new(channel.clone()); + let config_client = BackendConfigClient::new(channel); + + Ok(Self { + backend_client, + risk_client, + config_client, + health_checker: Arc::new(HealthChecker::new(30)), + health_state: None, + }) + } + + /// Inject shared backend health state for SubscribeSystemStatus streaming. + pub fn set_health_state(&mut self, state: BackendHealthState) { + self.health_state = Some(state); + } + + /// Check circuit breaker before forwarding request + #[inline(always)] + fn check_circuit_breaker(&self) -> Result<(), Status> { + if !self.health_checker.is_healthy() { + return Err(Status::unavailable( + "Trading service is unavailable (circuit breaker open)", + )); + } + Ok(()) + } + + /// Extract user_id from metadata (injected by upstream AuthInterceptor) + #[inline(always)] + fn extract_user_id(request: &Request) -> Result { + request + .metadata() + .get("x-user-id") + .ok_or_else(|| Status::internal("Missing user context (x-user-id)"))? + .to_str() + .map(|s| s.to_string()) + .map_err(|_| Status::internal("Invalid user_id encoding")) + } + + /// Background health check + pub async fn background_health_check(&mut self) { + self.health_checker + .check_health(&mut self.backend_client) + .await; + } + + // ======================================================================== + // Translation Helper Functions + // ======================================================================== + + /// Translate TLI OrderSide to Backend OrderSide (enum values match, just pass through) + #[inline] + fn translate_order_side(tli_side: i32) -> i32 { + tli_side // Enum values are identical between TLI and backend proto + } + + /// Translate Backend OrderSide to TLI OrderSide (enum values match, just pass through) + #[inline] + fn translate_order_side_back(backend_side: i32) -> i32 { + backend_side // Enum values are identical between TLI and backend proto + } + + /// Translate TLI OrderType to Backend OrderType (enum values match, just pass through) + #[inline] + fn translate_order_type(tli_type: i32) -> i32 { + tli_type // Enum values are identical between TLI and backend proto + } + + /// Translate Backend OrderType to TLI OrderType (enum values match, just pass through) + #[inline] + fn translate_order_type_back(backend_type: i32) -> i32 { + backend_type // Enum values are identical between TLI and backend proto + } + + /// Translate Backend OrderStatus to TLI OrderStatus (enum values match, just pass through) + #[inline] + fn translate_order_status(backend_status: i32) -> i32 { + backend_status // Enum values are identical between TLI and backend proto + } + + /// Translate Backend `Position` to TLI Position + fn translate_position( + backend_pos: crate::trading_backend::Position, + ) -> crate::foxhunt::tli::Position { + // Calculate market_price from market_value and quantity + let market_price = if backend_pos.quantity != 0.0 { + backend_pos.market_value / backend_pos.quantity + } else { + 0.0 + }; + + crate::foxhunt::tli::Position { + symbol: backend_pos.symbol, + quantity: backend_pos.quantity, + market_price, + market_value: backend_pos.market_value, + average_cost: backend_pos.average_price, + unrealized_pnl: backend_pos.unrealized_pnl, + realized_pnl: backend_pos.realized_pnl, + } + } + + /// Translate Backend MarketDataEvent to TLI MarketDataEvent + fn translate_market_data_event( + backend_event: crate::trading_backend::MarketDataEvent, + ) -> crate::foxhunt::tli::MarketDataEvent { + use crate::trading_backend::market_data_event::Data as BackendData; + + // Extract the event data based on backend oneof type + let tli_event = match backend_event.data { + Some(BackendData::Trade(backend_trade)) => { + // Backend Trade → TLI TradeData + crate::foxhunt::tli::market_data_event::Event::Trade( + crate::foxhunt::tli::TradeData { + symbol: backend_event.symbol.clone(), + timestamp_unix_nanos: backend_trade.timestamp, + price: backend_trade.price, + size: backend_trade.volume as u64, + trade_id: format!("{}", backend_trade.timestamp), // Generate trade_id from timestamp + exchange: "".to_string(), // Backend doesn't have exchange field + }, + ) + }, + Some(BackendData::Quote(backend_quote)) => { + // Backend Quote → TLI QuoteData + crate::foxhunt::tli::market_data_event::Event::Quote( + crate::foxhunt::tli::QuoteData { + symbol: backend_event.symbol.clone(), + timestamp_unix_nanos: backend_quote.timestamp, + bid_price: backend_quote.bid_price, + bid_size: backend_quote.bid_size as u64, + ask_price: backend_quote.ask_price, + ask_size: backend_quote.ask_size as u64, + exchange: "".to_string(), // Backend doesn't have exchange field + }, + ) + }, + Some(BackendData::OrderBook(backend_book)) => { + // Backend OrderBook → TLI TickData (use last trade from order book) + // Note: TLI doesn't have OrderBook, so we convert to a tick with mid price + let mid_price = if let (Some(bid), Some(ask)) = + (backend_book.bids.first(), backend_book.asks.first()) + { + (bid.price + ask.price) / 2.0 + } else if let Some(bid) = backend_book.bids.first() { + bid.price + } else if let Some(ask) = backend_book.asks.first() { + ask.price + } else { + 0.0 + }; + + let size = if let Some(bid) = backend_book.bids.first() { + bid.quantity as u64 + } else if let Some(ask) = backend_book.asks.first() { + ask.quantity as u64 + } else { + 0_u64 + }; + + crate::foxhunt::tli::market_data_event::Event::Tick(crate::foxhunt::tli::TickData { + symbol: backend_event.symbol.clone(), + timestamp_unix_nanos: backend_book.timestamp, + price: mid_price, + size, + exchange: "".to_string(), + }) + }, + None => { + // No data available, create empty tick + warn!("Backend MarketDataEvent has no data, creating empty tick"); + crate::foxhunt::tli::market_data_event::Event::Tick(crate::foxhunt::tli::TickData { + symbol: backend_event.symbol.clone(), + timestamp_unix_nanos: backend_event.timestamp, + price: 0.0, + size: 0_u64, + exchange: "".to_string(), + }) + }, + }; + + crate::foxhunt::tli::MarketDataEvent { + event: Some(tli_event), + } + } + + /// Translate Backend OrderEvent to TLI OrderUpdateEvent + fn translate_order_event( + backend_event: crate::trading_backend::OrderEvent, + ) -> crate::foxhunt::tli::OrderUpdateEvent { + // Extract order from backend event + let backend_order = backend_event.order.unwrap_or_else(|| { + warn!("Backend OrderEvent missing order details"); + crate::trading_backend::Order { + order_id: backend_event.order_id.clone(), + symbol: "".to_string(), + side: 0, + order_type: 0, + quantity: 0.0, + filled_quantity: 0.0, + price: None, + stop_price: None, + status: 0, + created_at: backend_event.timestamp, + updated_at: Some(backend_event.timestamp), + account_id: "".to_string(), + metadata: Default::default(), + } + }); + + crate::foxhunt::tli::OrderUpdateEvent { + order_id: backend_order.order_id, + symbol: backend_order.symbol, + status: backend_order.status, // Enum values match + filled_quantity: backend_order.filled_quantity, + remaining_quantity: backend_order.quantity - backend_order.filled_quantity, + last_fill_price: backend_order.price.unwrap_or(0.0), + last_fill_quantity: 0_u64, // Backend doesn't have last_fill_quantity + timestamp_unix_nanos: backend_event.timestamp, + message: backend_event.message, + } + } +} + +// ============================================================================ +// TradingService trait implementation with protocol translation +// ============================================================================ + +#[tonic::async_trait] +impl TliTradingService for TradingServiceProxy { + // Streaming response types + type SubscribeMarketDataStream = + Pin> + Send>>; + type SubscribeOrderUpdatesStream = + Pin> + Send>>; + type SubscribeRiskAlertsStream = + Pin> + Send>>; + type SubscribeMetricsStream = + Pin> + Send>>; + type SubscribeConfigStream = + Pin> + Send>>; + type SubscribeSystemStatusStream = + Pin> + Send>>; + + /// Submit order with protocol translation + #[instrument(skip_all)] + async fn submit_order( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + let user_id = Self::extract_user_id(&request)?; + debug!("Translating submit_order for user: {}", user_id); + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + debug!("Client metadata count: {}", client_metadata.keys().count()); + debug!("Checking for authorization header..."); + if let Some(auth) = client_metadata.get("authorization") { + debug!(" authorization: present (value: {:?})", auth); + } else { + warn!(" authorization: MISSING"); + } + if let Some(user_id) = client_metadata.get("x-user-id") { + debug!(" x-user-id: {:?}", user_id); + } else { + warn!(" x-user-id: MISSING"); + } + let tli_req = request.into_inner(); + + // Translate TLI proto → Trading proto + let backend_req = crate::trading_backend::SubmitOrderRequest { + symbol: tli_req.symbol, + side: Self::translate_order_side(tli_req.side), + quantity: tli_req.quantity, + order_type: Self::translate_order_type(tli_req.order_type), + price: tli_req.price, + stop_price: tli_req.stop_price, + account_id: user_id, + metadata: vec![ + ("client_order_id".to_string(), tli_req.client_order_id), + ("time_in_force".to_string(), tli_req.time_in_force), + ] + .into_iter() + .collect(), + }; + + // Forward to backend with auth metadata + let mut client = self.backend_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + debug!("Forwarding authorization header to backend"); + backend_metadata.insert("authorization", auth_token.clone()); + } else { + warn!("No authorization header in client metadata!"); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + debug!("Forwarding x-user-id header to backend"); + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } else { + warn!("No x-user-id header in client metadata!"); + } + + let backend_resp = match client.submit_order(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in submit_order: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate Trading proto → TLI proto + let tli_resp = crate::foxhunt::tli::SubmitOrderResponse { + success: backend_resp.status == 2, // OrderStatusSubmitted = 2 + order_id: backend_resp.order_id, + message: backend_resp.message, + timestamp_unix_nanos: backend_resp.timestamp, + }; + + Ok(Response::new(tli_resp)) + } + + /// Cancel order with protocol translation + #[instrument(skip_all)] + async fn cancel_order( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + let user_id = Self::extract_user_id(&request)?; + debug!("Translating cancel_order for user: {}", user_id); + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Translate request + let backend_req = crate::trading_backend::CancelOrderRequest { + order_id: tli_req.order_id, + account_id: user_id, + }; + + // Forward to backend with auth metadata + let mut client = self.backend_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.cancel_order(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in cancel_order: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate response + let tli_resp = crate::foxhunt::tli::CancelOrderResponse { + success: backend_resp.success, + message: backend_resp.message, + timestamp_unix_nanos: backend_resp.timestamp, + }; + + Ok(Response::new(tli_resp)) + } + + /// Get order status with protocol translation + #[instrument(skip_all)] + async fn get_order_status( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Translate request + let backend_req = crate::trading_backend::GetOrderStatusRequest { + order_id: tli_req.order_id.clone(), + }; + + // Forward to backend with auth metadata + let mut client = self.backend_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.get_order_status(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in get_order_status: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Extract order from backend response + let backend_order = backend_resp + .order + .ok_or_else(|| Status::internal("Backend returned empty order"))?; + + // Translate response + let tli_resp = crate::foxhunt::tli::GetOrderStatusResponse { + order_id: backend_order.order_id, + symbol: backend_order.symbol, + side: Self::translate_order_side_back(backend_order.side), + order_type: Self::translate_order_type_back(backend_order.order_type), + quantity: backend_order.quantity, + filled_quantity: backend_order.filled_quantity, + remaining_quantity: backend_order.quantity - backend_order.filled_quantity, + average_price: backend_order.price.unwrap_or(0.0), + status: Self::translate_order_status(backend_order.status), + created_at_unix_nanos: backend_order.created_at, + updated_at_unix_nanos: backend_order.updated_at.unwrap_or(backend_order.created_at), + }; + + Ok(Response::new(tli_resp)) + } + + /// Get account info with protocol translation + #[instrument(skip_all)] + async fn get_account_info( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + let _user_id = Self::extract_user_id(&request)?; + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // TLI GetAccountInfo maps to Backend GetPortfolioSummary + let backend_req = crate::trading_backend::GetPortfolioSummaryRequest { + account_id: tli_req.account_id.clone(), + }; + + // Forward to backend with auth metadata + let mut client = self.backend_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.get_portfolio_summary(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in get_portfolio_summary: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate response + let tli_resp = crate::foxhunt::tli::GetAccountInfoResponse { + account_id: tli_req.account_id, + total_value: backend_resp.total_value, + cash_balance: backend_resp.buying_power, // Map buying_power to cash_balance + buying_power: backend_resp.buying_power, + maintenance_margin: backend_resp.margin_used, + day_trading_buying_power: backend_resp.buying_power, // Approximation + }; + + Ok(Response::new(tli_resp)) + } + + /// Get positions with protocol translation + #[instrument(skip_all)] + async fn get_positions( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + let user_id = Self::extract_user_id(&request)?; + debug!("Translating get_positions for user: {}", user_id); + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Translate request + let backend_req = crate::trading_backend::GetPositionsRequest { + account_id: Some(user_id), + symbol: tli_req.symbol, + }; + + // Forward to backend with auth metadata + let mut client = self.backend_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.get_positions(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in get_positions: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate positions + let tli_positions = backend_resp + .positions + .into_iter() + .map(Self::translate_position) + .collect(); + + let tli_resp = crate::foxhunt::tli::GetPositionsResponse { + positions: tli_positions, + }; + + Ok(Response::new(tli_resp)) + } + + // ======================================================================== + // Streaming Methods (Phase 4 implementation) + // ======================================================================== + + /// Subscribe to market data stream with protocol translation + #[instrument(skip_all)] + async fn subscribe_market_data( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + let _user_id = Self::extract_user_id(&request)?; + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + debug!( + "Translating subscribe_market_data for symbols: {:?}, data_types: {:?}", + tli_req.symbols, tli_req.data_types + ); + + // Translate TLI proto → Trading proto + let backend_req = crate::trading_backend::StreamMarketDataRequest { + symbols: tli_req.symbols, + data_types: tli_req.data_types, // Enum values match between protos + }; + + // Connect to backend stream with auth metadata + let mut client = self.backend_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_stream = match client.stream_market_data(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in subscribe_market_data: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Create translation stream: Backend MarketDataEvent → TLI MarketDataEvent + let tli_stream = futures::stream::unfold(backend_stream, |mut stream| async move { + match stream.message().await { + Ok(Some(backend_event)) => { + let tli_event = Self::translate_market_data_event(backend_event); + Some((Ok(tli_event), stream)) + }, + Ok(None) => None, // Stream ended + Err(e) => { + error!("Error in market data stream: {}", e); + Some((Err(e), stream)) + }, + } + }); + + Ok(Response::new(Box::pin(tli_stream))) + } + + /// Subscribe to order updates stream with protocol translation + #[instrument(skip_all)] + async fn subscribe_order_updates( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + let user_id = Self::extract_user_id(&request)?; + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + debug!( + "Translating subscribe_order_updates for account: {:?}", + tli_req.account_id + ); + + // Translate TLI proto → Trading proto + let backend_req = crate::trading_backend::StreamOrdersRequest { + account_id: tli_req.account_id.or(Some(user_id)), + symbol: None, // TLI doesn't have symbol filter for order updates + }; + + // Connect to backend stream with auth metadata + let mut client = self.backend_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_stream = match client.stream_orders(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in subscribe_order_updates: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Create translation stream: Backend OrderEvent → TLI OrderUpdateEvent + let tli_stream = futures::stream::unfold(backend_stream, |mut stream| async move { + match stream.message().await { + Ok(Some(backend_event)) => { + let tli_event = Self::translate_order_event(backend_event); + Some((Ok(tli_event), stream)) + }, + Ok(None) => None, // Stream ended + Err(e) => { + error!("Error in order updates stream: {}", e); + Some((Err(e), stream)) + }, + } + }); + + Ok(Response::new(Box::pin(tli_stream))) + } + + // ======================================================================== + // Risk Management Methods (connect to Risk Service backend) + // ======================================================================== + + #[instrument(skip_all)] + async fn get_va_r( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Translate TLI proto → Risk proto + let backend_req = crate::risk::GetVaRRequest { + symbols: tli_req.symbols, + confidence_level: tli_req.confidence_level, + lookback_days: i32::try_from(tli_req.lookback_days).map_err(|_| { + Status::invalid_argument(format!( + "lookback_days {} exceeds i32 range", + tli_req.lookback_days + )) + })?, + method: tli_req.methodology, // TLI uses 'methodology', backend uses 'method' + }; + + // Forward to Risk backend with auth metadata + let mut client = self.risk_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.get_va_r(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in get_va_r: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate response - Backend uses different field names than TLI + let tli_resp = crate::foxhunt::tli::GetVaRResponse { + portfolio_var: backend_resp.portfolio_var, + symbol_vars: backend_resp + .symbol_vars + .into_iter() + .map(|sv| { + crate::foxhunt::tli::SymbolVaR { + symbol: sv.symbol, + var_amount: sv.var_value, // Backend: var_value → TLI: var_amount + contribution_percent: sv.contribution_pct, // Backend: contribution_pct → TLI: contribution_percent + } + }) + .collect(), + timestamp_unix_nanos: backend_resp.calculated_at, // Backend: calculated_at (i64) → TLI: timestamp_unix_nanos (i64) + methodology_used: format!("{:?}", backend_resp.method), // Backend: method (enum) → TLI: methodology_used (string) + // Note: TLI doesn't have confidence_level or lookback_days in response + // These are request parameters that TLI clients already know + }; + + Ok(Response::new(tli_resp)) + } + + #[instrument(skip_all)] + async fn get_position_risk( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Translate TLI proto → Risk proto + // Note: TLI doesn't have account_id in GetPositionRiskRequest + let backend_req = crate::risk::GetPositionRiskRequest { + symbol: tli_req.symbol, + account_id: None, // TLI proto doesn't include account_id + }; + + // Forward to Risk backend with auth metadata + let mut client = self.risk_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.get_position_risk(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in get_position_risk: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate response: Backend has more fields than TLI + // Backend PositionRisk → TLI PositionRisk field mappings: + // concentration_risk (f64) → concentration_percent (f64) + // overall_score.risk_level (enum) → risk_level (enum) + // liquidity_risk, metrics → discarded (TLI doesn't have these fields) + + let positions: Vec<_> = backend_resp + .position_risks + .into_iter() + .map(|pr| { + crate::foxhunt::tli::PositionRisk { + symbol: pr.symbol, + position_size: pr.position_size, + market_value: pr.market_value, + var_contribution: pr.var_contribution, + concentration_percent: pr.concentration_risk, // Backend concentration_risk → TLI concentration_percent + risk_level: pr.overall_score.map(|rs| rs.risk_level).unwrap_or(0), // Extract risk_level from overall_score + } + }) + .collect(); + + // Calculate total_exposure and concentration_risk for TLI response + let total_exposure: f64 = positions.iter().map(|p| p.market_value.abs()).sum(); + let max_concentration: f64 = positions + .iter() + .map(|p| p.concentration_percent) + .fold(0.0, f64::max); + + let tli_resp = crate::foxhunt::tli::GetPositionRiskResponse { + positions, // Backend position_risks → TLI positions + total_exposure, + concentration_risk: max_concentration, + timestamp_unix_nanos: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + .try_into() + .unwrap_or_else(|_| { + warn!("Timestamp exceeds i64::MAX nanoseconds"); + i64::MAX + }), + }; + + Ok(Response::new(tli_resp)) + } + + #[instrument(skip_all)] + async fn validate_order( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Convert TLI OrderSide enum to string for backend + let side_str = match tli_req.side { + 1 => "buy", // ORDER_SIDE_BUY + 2 => "sell", // ORDER_SIDE_SELL + _ => "unknown", + } + .to_string(); + + // Translate TLI proto → Risk proto + let backend_req = crate::risk::ValidateOrderRequest { + symbol: tli_req.symbol, + quantity: tli_req.quantity, + price: tli_req.price, + side: side_str, + account_id: tli_req.account_id, + }; + + // Forward to Risk backend with auth metadata + let mut client = self.risk_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.validate_order(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in validate_order: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate response + // Backend has: is_valid, violations, risk_score, message + // TLI has: approved, reason, violations, projected_exposure, margin_impact + let tli_resp = crate::foxhunt::tli::ValidateOrderResponse { + approved: backend_resp.is_valid, + reason: backend_resp.message, + violations: backend_resp + .violations + .into_iter() + .map(|v| { + // Backend RiskViolation has: violation_type, description, current_value, limit_value, severity + // TLI RiskViolation has: type, description, limit_value, current_value, severity + crate::foxhunt::tli::RiskViolation { + r#type: v.violation_type, + description: v.description, + limit_value: v.limit_value, + current_value: v.current_value, + severity: v.severity, + } + }) + .collect(), + projected_exposure: 0.0, // Backend doesn't provide this, use placeholder + margin_impact: 0.0, // Backend doesn't provide this, use placeholder + }; + + Ok(Response::new(tli_resp)) + } + + #[instrument(skip_all)] + async fn get_risk_metrics( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Translate TLI proto → Risk proto + let backend_req = crate::risk::GetRiskMetricsRequest { + portfolio_id: tli_req.portfolio_id, + }; + + // Forward to Risk backend with auth metadata + let mut client = self.risk_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.get_risk_metrics(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in get_risk_metrics: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate response - backend has nested RiskMetrics, TLI has flat fields + let tli_resp = crate::foxhunt::tli::GetRiskMetricsResponse { + // Extract fields from nested metrics + sharpe_ratio: backend_resp + .metrics + .as_ref() + .map(|m| m.sharpe_ratio) + .unwrap_or(0.0), + max_drawdown: backend_resp + .metrics + .as_ref() + .map(|m| m.max_drawdown) + .unwrap_or(0.0), + current_drawdown: backend_resp + .metrics + .as_ref() + .map(|m| m.current_drawdown) + .unwrap_or(0.0), + volatility: backend_resp + .metrics + .as_ref() + .map(|m| m.volatility) + .unwrap_or(0.0), + beta: backend_resp.metrics.as_ref().map(|m| m.beta).unwrap_or(0.0), + alpha: backend_resp + .metrics + .as_ref() + .map(|m| m.alpha) + .unwrap_or(0.0), + value_at_risk: backend_resp + .metrics + .as_ref() + .map(|m| m.portfolio_var_1d) + .unwrap_or(0.0), + expected_shortfall: backend_resp + .metrics + .as_ref() + .map(|m| { + // Calculate expected shortfall as average of VaR values (approximation) + (m.portfolio_var_1d + m.portfolio_var_5d + m.portfolio_var_30d) / 3.0 + }) + .unwrap_or(0.0), + timestamp_unix_nanos: backend_resp.calculated_at, + }; + + Ok(Response::new(tli_resp)) + } + + #[instrument(skip_all)] + async fn subscribe_risk_alerts( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Translate TLI proto → Risk proto + let backend_req = crate::risk::StreamRiskAlertsRequest { + min_severity: tli_req.min_severity.first().copied().unwrap_or(0), + alert_types: Vec::new(), // TLI doesn't expose alert_types, backend requires it + }; + // Connect to Risk backend stream with auth metadata + let mut client = self.risk_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_stream = match client.stream_risk_alerts(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in subscribe_risk_alerts: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Create translation stream + let tli_stream = futures::stream::unfold(backend_stream, |mut stream| async move { + match stream.message().await { + Ok(Some(backend_event)) => { + let tli_event = crate::foxhunt::tli::RiskAlertEvent { + alert_id: backend_event.alert_id, + // Map backend RiskAlertSeverity to TLI RiskSeverity (enum values may differ) + severity: backend_event.severity, // Pass through, assuming compatible + message: backend_event.message, + symbol: backend_event.symbol.unwrap_or_default(), // TLI requires string + // TLI doesn't have threshold_value, current_value - set to 0.0 + threshold_value: 0.0, + current_value: 0.0, + timestamp_unix_nanos: backend_event.timestamp, + // TLI doesn't have account_id, metadata, alert_type + // Map severity to requires_action (critical/emergency = true) + requires_action: backend_event.severity >= 3, // CRITICAL or EMERGENCY + }; + Some((Ok(tli_event), stream)) + }, + Ok(None) => None, + Err(e) => { + error!("Error in risk alerts stream: {}", e); + Some((Err(e), stream)) + }, + } + }); + + Ok(Response::new(Box::pin(tli_stream))) + } + + #[instrument(skip_all)] + async fn emergency_stop( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Translate TLI proto → Risk proto + // TLI has repeated symbols, backend takes single optional symbol (use first if provided) + let backend_req = crate::risk::EmergencyStopRequest { + stop_type: tli_req.stop_type, + reason: tli_req.reason, + symbol: tli_req.symbols.first().cloned(), + account_id: None, // TLI doesn't provide account_id in request + }; + + // Forward to Risk backend with auth metadata + let mut client = self.risk_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.emergency_stop(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in emergency_stop: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate response + // Backend provides affected_orders list, TLI expects counts + let orders_cancelled = + u32::try_from(backend_resp.affected_orders.len()).unwrap_or_else(|_| { + warn!( + "Number of affected orders {} exceeds u32::MAX", + backend_resp.affected_orders.len() + ); + u32::MAX + }); + + let tli_resp = crate::foxhunt::tli::EmergencyStopResponse { + success: backend_resp.success, + message: backend_resp.message, + orders_cancelled, + positions_closed: 0_u32, // Backend doesn't track positions_closed + timestamp_unix_nanos: backend_resp.timestamp, + }; + + Ok(Response::new(tli_resp)) + } + + // ======================================================================== + // Monitoring Methods — DEPRECATED: Use MonitoringService directly + // These TLI monitoring RPCs targeted the stale trading_service monitoring + // proto (10 RPCs, never implemented). The real MonitoringService (2 RPCs: + // GetLiveTrainingMetrics, StreamTrainingMetrics) is now served directly + // by the API gateway via MonitoringServiceHandler. + // ======================================================================== + + #[instrument(skip_all)] + async fn get_metrics( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("Use MonitoringService directly")) + } + + #[instrument(skip_all)] + async fn get_latency( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("Use MonitoringService directly")) + } + + #[instrument(skip_all)] + async fn get_throughput( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("Use MonitoringService directly")) + } + + #[instrument(skip_all)] + async fn subscribe_metrics( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("Use MonitoringService directly")) + } + + // ======================================================================== + // Configuration Methods (connect to Config Service backend) + // ======================================================================== + + #[instrument(skip_all)] + async fn update_parameters( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Extract user_id for changed_by field + let user_id = client_metadata + .get("x-user-id") + .and_then(|v| v.to_str().ok()) + .unwrap_or("unknown") + .to_string(); + + // Translate TLI proto → Config proto + // TLI has map parameters + // Backend expects individual UpdateConfigurationRequest per parameter + // For simplicity, we'll update each parameter individually + // Note: TLI UpdateParametersRequest has: + // - parameters: map + // - persist: bool + // Backend UpdateConfigurationRequest has: + // - category, key, value, changed_by, change_reason, environment + + // Since TLI doesn't specify category/key structure, we'll parse keys as "category.key" + // For now, treat the first parameter (if multiple, use first one only for single backend call) + let (key, value) = tli_req + .parameters + .iter() + .next() + .ok_or_else(|| Status::invalid_argument("No parameters provided"))?; + + let backend_req = crate::config_backend::UpdateConfigurationRequest { + category: "runtime".to_string(), // Default category since TLI doesn't specify + key: key.clone(), + value: value.clone(), + changed_by: user_id.clone(), + change_reason: Some(format!("Updated via TLI (persist={})", tli_req.persist)), + environment: None, + }; + + // Forward to Config backend with auth metadata + let mut client = self.config_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.update_configuration(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in update_parameters: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate response + let tli_resp = crate::foxhunt::tli::UpdateParametersResponse { + success: backend_resp.success, + message: backend_resp.message, + updated_keys: vec![key.clone()], // Return the updated key + }; + + Ok(Response::new(tli_resp)) + } + + #[instrument(skip_all)] + async fn get_config( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Translate TLI proto → Config proto + // TLI has: repeated string keys + // Backend expects: optional category, optional key, optional environment + // Strategy: If keys is empty, get all configs (category=None, key=None) + // If keys has values, we need to query each key individually + // For simplicity, we'll get all configs and filter by keys if provided + let backend_req = crate::config_backend::GetConfigurationRequest { + category: None, + key: None, + environment: None, + }; + + // Forward to Config backend with auth metadata + let mut client = self.config_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.get_configuration(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in get_config: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate response: Backend ConfigurationSetting → TLI map + // TLI expects: map config, version, last_updated_unix_nanos + // Backend provides: repeated ConfigurationSetting with many fields + + // Build map from settings, filtering by requested keys if provided + let mut config_map = std::collections::HashMap::new(); + let mut max_modified_at = 0i64; + + for setting in backend_resp.settings { + // If keys filter provided, only include matching keys + if !tli_req.keys.is_empty() && !tli_req.keys.contains(&setting.key) { + continue; + } + + // Use "category.key" as the map key for uniqueness + let full_key = format!("{}.{}", setting.category, setting.key); + config_map.insert(full_key, setting.value); + + // Track latest modification time + if setting.modified_at > max_modified_at { + max_modified_at = setting.modified_at; + } + } + + let tli_resp = crate::foxhunt::tli::GetConfigResponse { + config: config_map, + version: 1_i64, // Backend doesn't provide version, use 1 as default + last_updated_unix_nanos: max_modified_at, + }; + + Ok(Response::new(tli_resp)) + } + + #[instrument(skip_all)] + async fn subscribe_config( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Translate TLI proto → Config proto (StreamConfigChanges) + let backend_req = crate::config_backend::StreamConfigChangesRequest { + categories: Vec::new(), // TLI doesn't have categories, backend requires it + keys: tli_req.keys, + }; + + // Connect to Config backend stream with auth metadata + let mut client = self.config_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_stream = match client.stream_config_changes(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in subscribe_config: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Create translation stream + let tli_stream = futures::stream::unfold(backend_stream, |mut stream| async move { + match stream.message().await { + Ok(Some(backend_event)) => { + // Translate Backend ConfigChangeEvent → TLI ConfigEvent (simplified structure) + let tli_event = crate::foxhunt::tli::ConfigEvent { + key: backend_event.key, + value: backend_event.new_value, + old_value: backend_event.old_value, + timestamp_unix_nanos: backend_event.timestamp, + }; + Some((Ok(tli_event), stream)) + }, + Ok(None) => None, + Err(e) => { + error!("Error in config stream: {}", e); + Some((Err(e), stream)) + }, + } + }); + + Ok(Response::new(Box::pin(tli_stream))) + } + + #[instrument(skip_all)] + async fn get_system_status( + &self, + _request: Request, + ) -> Result, Status> { + let health_state = self + .health_state + .as_ref() + .ok_or_else(|| Status::unavailable("System health monitoring not configured"))?; + + let entries = health_state.rx.borrow().clone(); + let all_healthy = entries.iter().all(|e| e.healthy); + let overall = if all_healthy { 1 } else { 2 }; // HEALTHY=1, DEGRADED=2 + + let services: Vec = entries + .iter() + .map(|e| crate::foxhunt::tli::ServiceStatus { + name: e.name.clone(), + status: if e.healthy { 1 } else { 3 }, + message: e.message.clone(), + last_check_unix_nanos: 0, + details: HashMap::new(), + }) + .collect(); + + Ok(Response::new( + crate::foxhunt::tli::GetSystemStatusResponse { + overall_status: overall, + services, + timestamp_unix_nanos: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as i64) + .unwrap_or(0), + }, + )) + } + + #[instrument(skip_all)] + async fn subscribe_system_status( + &self, + _request: Request, + ) -> Result, Status> { + let health_state = self + .health_state + .as_ref() + .ok_or_else(|| Status::unavailable("System health monitoring not configured"))? + .clone(); + + let stream = async_stream::stream! { + let mut rx = health_state.rx.clone(); + // Emit initial snapshot immediately. + { + let entries = rx.borrow().clone(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as i64) + .unwrap_or(0); + for entry in &entries { + let status = if entry.healthy { 1 } else { 3 }; // HEALTHY=1, UNHEALTHY=3 + yield Ok(crate::foxhunt::tli::SystemStatusEvent { + service_name: entry.name.clone(), + status, + previous_status: 0, + message: entry.message.clone(), + timestamp_unix_nanos: now, + }); + } + } + // Then emit updates whenever the health snapshot changes. + loop { + if rx.changed().await.is_err() { + break; // Sender dropped — gateway shutting down. + } + let entries = rx.borrow_and_update().clone(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as i64) + .unwrap_or(0); + for entry in &entries { + let status = if entry.healthy { 1 } else { 3 }; + yield Ok(crate::foxhunt::tli::SystemStatusEvent { + service_name: entry.name.clone(), + status, + previous_status: 0, + message: entry.message.clone(), + timestamp_unix_nanos: now, + }); + } + } + }; + + Ok(Response::new(Box::pin(stream))) + } + + // ======================================================================== + // ML Trading Operations + // ======================================================================== + + /// Submit ML-powered trading order with ensemble predictions + #[instrument(skip_all)] + async fn submit_ml_order( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + let user_id = Self::extract_user_id(&request)?; + debug!("Translating submit_ml_order for user: {}", user_id); + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Translate TLI proto → Trading proto + let backend_req = crate::trading_backend::MlOrderRequest { + symbol: tli_req.symbol.clone(), + account_id: user_id, + use_ensemble: tli_req.model_filter.is_none(), + model_name: tli_req.model_filter.clone(), + features: vec![], // Features are extracted in the trading service + }; + + // Forward to backend with auth metadata + let mut client = self.backend_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.submit_ml_order(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in submit_ml_order: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate Trading proto → TLI proto + let tli_resp = crate::foxhunt::tli::SubmitMlOrderResponse { + order_id: backend_resp.order_id, + symbol: tli_req.symbol, + model_used: if backend_resp.executed { + if tli_req.model_filter.is_some() { + tli_req + .model_filter + .unwrap_or_else(|| "Ensemble".to_string()) + } else { + "Ensemble".to_string() + } + } else { + "None".to_string() + }, + predicted_action: backend_resp.action, + confidence: backend_resp.confidence, + // BLOCKER: The Trading-service MlOrderResponse proto does not include a + // quantity field — it only returns order_id, prediction_id, action, + // confidence, message, and executed. To expose real order quantity here, + // either: + // 1. Add an `int32 quantity` field to MlOrderResponse in trading.proto + // and populate it from the SubmitOrderResponse in the trading service, or + // 2. Perform a follow-up GetOrderStatus RPC using the returned order_id + // to fetch the filled quantity (adds latency). + // Until then, we report 1 (executed) or 0 (not executed) as a boolean proxy. + quantity: if backend_resp.executed { 1 } else { 0 }, + executed: backend_resp.executed, + message: backend_resp.message, + }; + + Ok(Response::new(tli_resp)) + } + + /// Get ML prediction history with outcomes + #[instrument(skip_all)] + async fn get_ml_predictions( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + debug!("Translating get_ml_predictions"); + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Translate TLI proto → Trading proto + let backend_req = crate::trading_backend::MlPredictionsRequest { + symbol: tli_req.symbol, + model_name: tli_req.model_filter, + limit: tli_req.limit.unwrap_or(10), + start_time: None, + end_time: None, + }; + + // Forward to backend with auth metadata + let mut client = self.backend_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.get_ml_predictions(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in get_ml_predictions: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate Trading proto → TLI proto + let predictions = backend_resp + .predictions + .into_iter() + .map(|pred| crate::foxhunt::tli::MlPrediction { + timestamp: format!("{}", pred.timestamp), // Convert nanos to ISO 8601 if needed + model_id: pred.ensemble_action.clone(), // Use action as model_id for simplicity + symbol: pred.symbol, + predicted_action: pred.ensemble_action, + confidence: pred.ensemble_confidence, + actual_return: pred.actual_pnl, + }) + .collect(); + + let tli_resp = crate::foxhunt::tli::GetMlPredictionsResponse { predictions }; + + Ok(Response::new(tli_resp)) + } + + /// Get ML model performance metrics + #[instrument(skip_all)] + async fn get_ml_performance( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + debug!("Translating get_ml_performance"); + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Translate TLI proto → Trading proto + let backend_req = crate::trading_backend::MlPerformanceRequest { + model_name: tli_req.model_filter, + start_time: None, + end_time: None, + }; + + // Forward to backend with auth metadata + let mut client = self.backend_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.get_ml_performance(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in get_ml_performance: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate Trading proto → TLI proto + let active_models = backend_resp.models.len() as i32; + let models = backend_resp + .models + .into_iter() + .map(|model| crate::foxhunt::tli::ModelPerformance { + model_id: model.model_name, + accuracy: model.accuracy, + total_predictions: model.total_predictions, + sharpe_ratio: model.sharpe_ratio, + avg_return: model.avg_pnl, + max_drawdown: 0.0, // Backend doesn't provide this field + }) + .collect(); + + let tli_resp = crate::foxhunt::tli::GetMlPerformanceResponse { + models, + ensemble_threshold: 0.6, // Default threshold, should come from config + active_models, + total_models: 4, // DQN, MAMBA2, PPO, TFT + }; + + Ok(Response::new(tli_resp)) + } + + /// Get current regime state for a symbol (Wave D) + #[instrument(skip_all)] + async fn get_regime_state( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + debug!("Translating get_regime_state"); + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Translate TLI proto → Trading proto + let backend_req = crate::trading_backend::GetRegimeStateRequest { + symbol: tli_req.symbol, + }; + + // Forward to backend with auth metadata + let mut client = self.backend_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.get_regime_state(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in get_regime_state: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate Trading proto → TLI proto + let tli_resp = crate::foxhunt::tli::GetRegimeStateResponse { + symbol: backend_resp.symbol, + current_regime: backend_resp.current_regime, + confidence: backend_resp.confidence, + cusum_s_plus: backend_resp.cusum_s_plus, + cusum_s_minus: backend_resp.cusum_s_minus, + adx: backend_resp.adx, + stability: backend_resp.stability, + entropy: backend_resp.entropy, + updated_at_unix_nanos: backend_resp.updated_at, + }; + + Ok(Response::new(tli_resp)) + } + + /// Get regime transition history for a symbol (Wave D) + #[instrument(skip_all)] + async fn get_regime_transitions( + &self, + request: Request, + ) -> Result, Status> { + self.check_circuit_breaker()?; + + debug!("Translating get_regime_transitions"); + + // Extract metadata BEFORE into_inner() consumes the request + let client_metadata = request.metadata().clone(); + let tli_req = request.into_inner(); + + // Translate TLI proto → Trading proto + let backend_req = crate::trading_backend::GetRegimeTransitionsRequest { + symbol: tli_req.symbol, + limit: tli_req.limit, + }; + + // Forward to backend with auth metadata + let mut client = self.backend_client.clone(); + let mut backend_request = Request::new(backend_req); + + // Forward authorization and user context from client metadata + let backend_metadata = backend_request.metadata_mut(); + if let Some(auth_token) = client_metadata.get("authorization") { + backend_metadata.insert("authorization", auth_token.clone()); + } + if let Some(user_id_meta) = client_metadata.get("x-user-id") { + backend_metadata.insert("x-user-id", user_id_meta.clone()); + } + + let backend_resp = match client.get_regime_transitions(backend_request).await { + Ok(resp) => resp.into_inner(), + Err(e) => { + error!("Backend error in get_regime_transitions: {}", e); + if matches!( + e.code(), + tonic::Code::Unavailable | tonic::Code::DeadlineExceeded + ) { + self.health_checker.mark_unhealthy(); + } + return Err(e); + }, + }; + + // Translate Trading proto → TLI proto + let transitions = backend_resp + .transitions + .into_iter() + .map(|trans| crate::foxhunt::tli::RegimeTransition { + from_regime: trans.from_regime, + to_regime: trans.to_regime, + duration_bars: trans.duration_bars, + transition_probability: trans.transition_probability, + timestamp_unix_nanos: trans.timestamp, + }) + .collect(); + + let tli_resp = crate::foxhunt::tli::GetRegimeTransitionsResponse { transitions }; + + Ok(Response::new(tli_resp)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_health_checker_creation() { + let checker = HealthChecker::new(30); + assert!(checker.is_healthy()); + } + + #[test] + fn test_health_checker_mark_unhealthy() { + let checker = HealthChecker::new(30); + checker.mark_unhealthy(); + assert!(!checker.is_healthy()); + } + + #[test] + fn test_order_side_translation() { + assert_eq!(TradingServiceProxy::translate_order_side(1), 1); // Buy + assert_eq!(TradingServiceProxy::translate_order_side(2), 2); // Sell + } + + #[test] + fn test_order_type_translation() { + assert_eq!(TradingServiceProxy::translate_order_type(1), 1); // Market + assert_eq!(TradingServiceProxy::translate_order_type(2), 2); // Limit + } +} diff --git a/services/api/src/health.rs b/services/api/src/health.rs new file mode 100644 index 000000000..3097fa006 --- /dev/null +++ b/services/api/src/health.rs @@ -0,0 +1,133 @@ +//! Health check endpoints for API Gateway service +//! +//! Provides HTTP-based health and readiness endpoints for Kubernetes probes. +//! These endpoints don't require authentication to allow load balancers and +//! orchestrators to check service health. + +use axum::{extract::State, http::StatusCode, routing::get, Json, Router}; +use serde::Serialize; +use std::sync::Arc; +use sqlx::PgPool; +use redis::aio::ConnectionManager; + +/// Health status response +#[derive(Serialize)] +pub struct HealthResponse { + /// Service status + pub status: &'static str, + /// Service name + pub service: &'static str, + /// Service version + pub version: &'static str, +} + +/// Detailed health status with dependency checks +#[derive(Serialize)] +pub struct DetailedHealthResponse { + /// Service status + pub status: &'static str, + /// Service name + pub service: &'static str, + /// Service version + pub version: &'static str, + /// Database connectivity + pub database: bool, + /// Redis connectivity + pub redis: bool, +} + +/// Shared health check state +#[derive(Clone)] +pub struct HealthState { + /// PostgreSQL connection pool + pub db_pool: Option, + /// Redis connection manager + pub redis: Option, +} + +/// Creates the health check router with /health and /ready endpoints +pub fn health_router(state: HealthState) -> Router { + Router::new() + .route("/health", get(health_check)) + .route("/ready", get(readiness_check)) + .with_state(Arc::new(state)) +} + +/// Liveness probe - indicates service is running +/// Returns 200 OK if the service process is alive +async fn health_check() -> Json { + Json(HealthResponse { + status: "healthy", + service: "api", + version: env!("CARGO_PKG_VERSION"), + }) +} + +/// Readiness probe - indicates service is ready to accept requests +/// Checks database and Redis connectivity +async fn readiness_check( + State(state): State>, +) -> Result, StatusCode> { + let mut db_ok = false; + let mut redis_ok = false; + + // Check database connectivity + if let Some(ref pool) = state.db_pool { + db_ok = sqlx::query("SELECT 1") + .fetch_one(pool) + .await + .is_ok(); + } + + // Check Redis connectivity + if let Some(ref mut redis_conn) = state.redis.clone() { + use redis::AsyncCommands; + redis_ok = redis_conn.ping::<()>().await.is_ok(); + } + + // Service is ready only if all dependencies are healthy + let is_ready = db_ok && redis_ok; + let status = if is_ready { "ready" } else { "not_ready" }; + let http_status = if is_ready { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }; + + let response = DetailedHealthResponse { + status, + service: "api", + version: env!("CARGO_PKG_VERSION"), + database: db_ok, + redis: redis_ok, + }; + + if is_ready { + Ok(Json(response)) + } else { + Err(http_status) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_health_check() { + let response = health_check().await; + assert_eq!(response.status, "healthy"); + assert_eq!(response.service, "api"); + } + + #[tokio::test] + async fn test_readiness_check_without_deps() { + let state = Arc::new(HealthState { + db_pool: None, + redis: None, + }); + + let result = readiness_check(State(state)).await; + assert!(result.is_err()); + } +} diff --git a/services/api/src/health_router.rs b/services/api/src/health_router.rs new file mode 100644 index 000000000..8382023a3 --- /dev/null +++ b/services/api/src/health_router.rs @@ -0,0 +1,285 @@ +//! Health and Resilience HTTP Endpoints +//! +//! Provides Kubernetes-compatible health probes and resilience status endpoints: +//! - /health/liveness - Liveness probe (always OK if service is running) +//! - /health/readiness - Readiness probe (checks backend services) +//! - /health/startup - Startup probe (initialization complete) +//! - /resilience/circuit-breaker/status - Circuit breaker state +//! - /resilience/rate-limit/status - Rate limiter status +//! - /resilience/timeout/config - Timeout configuration +//! - /resilience/retry/config - Retry policy + +use axum::{extract::State, http::StatusCode, routing::get, Json, Router}; +use serde_json::{json, Value}; +use std::sync::Arc; + +/// Application state for health endpoints +#[derive(Clone)] +pub struct HealthState { + pub startup_complete: Arc, + pub service_healthy: Arc, +} + +impl HealthState { + pub fn new() -> Self { + Self { + startup_complete: Arc::new(std::sync::atomic::AtomicBool::new(true)), + service_healthy: Arc::new(std::sync::atomic::AtomicBool::new(true)), + } + } + + pub fn mark_startup_complete(&self) { + self.startup_complete + .store(true, std::sync::atomic::Ordering::SeqCst); + } + + pub fn set_healthy(&self, healthy: bool) { + self.service_healthy + .store(healthy, std::sync::atomic::Ordering::SeqCst); + } + + pub fn is_startup_complete(&self) -> bool { + self.startup_complete + .load(std::sync::atomic::Ordering::SeqCst) + } + + pub fn is_healthy(&self) -> bool { + self.service_healthy + .load(std::sync::atomic::Ordering::SeqCst) + } +} + +impl Default for HealthState { + fn default() -> Self { + Self::new() + } +} + +/// Liveness probe - always returns OK if service is running +async fn liveness() -> &'static str { + "OK" +} + +/// Simple health check endpoint - returns JSON status +async fn health() -> Json { + Json(json!({"status": "healthy"})) +} + +/// Readiness probe - checks if service is ready to accept traffic +async fn readiness(State(state): State) -> Result { + if state.is_healthy() { + Ok("READY".to_string()) + } else { + Err(StatusCode::SERVICE_UNAVAILABLE) + } +} + +/// Startup probe - checks if initialization is complete +async fn startup(State(state): State) -> Result { + if state.is_startup_complete() { + Ok("READY".to_string()) + } else { + Err(StatusCode::SERVICE_UNAVAILABLE) + } +} + +/// Circuit breaker status +async fn circuit_breaker_status() -> Json { + Json(json!({ + "state": "closed", + "failure_count": 0, + "success_count": 100, + "threshold": 5, + "timeout_seconds": 60 + })) +} + +/// Rate limiter status +async fn rate_limit_status() -> Json { + Json(json!({ + "enabled": true, + "requests_per_minute": 1000, + "current_usage": 0, + "burst_size": 100 + })) +} + +/// Timeout configuration +async fn timeout_config() -> Json { + Json(json!({ + "request_timeout_ms": 5000, + "connection_timeout_ms": 3000, + "idle_timeout_ms": 60000 + })) +} + +/// Retry policy configuration +async fn retry_config() -> Json { + Json(json!({ + "max_retries": 3, + "retry_delay_ms": 100, + "backoff_multiplier": 2.0, + "max_backoff_ms": 10000 + })) +} + +/// Create health and resilience router +pub fn health_router(state: HealthState) -> Router { + Router::new() + // Simple health endpoint + .route("/health", get(health)) + // Health probes (Kubernetes-compatible) + .route("/health/liveness", get(liveness)) + .route("/health/readiness", get(readiness)) + .route("/health/startup", get(startup)) + // Resilience status endpoints + .route("/resilience/circuit-breaker/status", get(circuit_breaker_status)) + .route("/resilience/rate-limit/status", get(rate_limit_status)) + .route("/resilience/timeout/config", get(timeout_config)) + .route("/resilience/retry/config", get(retry_config)) + .with_state(state) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::Body; + use axum::http::{Request, StatusCode}; + use tower::ServiceExt; + + #[tokio::test] + async fn test_liveness_probe() { + let state = HealthState::new(); + let app = health_router(state); + + let response = app + .oneshot( + Request::builder() + .uri("/health/liveness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_readiness_probe_healthy() { + let state = HealthState::new(); + state.set_healthy(true); + let app = health_router(state); + + let response = app + .oneshot( + Request::builder() + .uri("/health/readiness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_readiness_probe_unhealthy() { + let state = HealthState::new(); + state.set_healthy(false); + let app = health_router(state); + + let response = app + .oneshot( + Request::builder() + .uri("/health/readiness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + } + + #[tokio::test] + async fn test_startup_probe() { + let state = HealthState::new(); + state.mark_startup_complete(); + let app = health_router(state); + + let response = app + .oneshot( + Request::builder() + .uri("/health/startup") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_circuit_breaker_status() { + let state = HealthState::new(); + let app = health_router(state); + + let response = app + .oneshot( + Request::builder() + .uri("/resilience/circuit-breaker/status") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_rate_limit_status() { + let state = HealthState::new(); + let app = health_router(state); + + let response = app + .oneshot( + Request::builder() + .uri("/resilience/rate-limit/status") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + } + + #[tokio::test] + async fn test_health_endpoint() { + let state = HealthState::new(); + let app = health_router(state); + + let response = app + .oneshot( + Request::builder() + .uri("/health") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + + // Verify JSON response + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let json: serde_json::Value = serde_json::from_slice(&body).expect("INVARIANT: Deserialization should succeed for valid JSON"); + assert_eq!(json["status"], "healthy"); + } +} diff --git a/services/api/src/lib.rs b/services/api/src/lib.rs new file mode 100644 index 000000000..fc8cbf9f5 --- /dev/null +++ b/services/api/src/lib.rs @@ -0,0 +1,129 @@ +//! Foxhunt API Service -- unified gRPC gateway with tonic-web for browser access +//! +//! Provides zero-copy gRPC proxying with: +//! - RBAC authorization with sub-100ns cached permission checks +//! - <10us routing overhead +//! - Connection pooling +//! - Circuit breaker pattern +//! - Health checking +//! - Performance monitoring +//! - Token bucket rate limiting with <50ns cache hits +//! - gRPC-Web support for browser clients via tonic-web + +#![deny(clippy::unwrap_used, clippy::expect_used)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))] +// API gateway domain lints: generated protobuf code and routing logic +#![allow(clippy::mixed_attributes_style)] // Generated protobuf code uses mixed attributes +#![allow(clippy::type_complexity)] // Complex types in async proxy handlers +#![allow(clippy::doc_lazy_continuation)] // Doc formatting acceptable +#![allow(clippy::manual_strip)] // Explicit prefix stripping for clarity + +// File descriptor sets for gRPC reflection +pub const TLI_FILE_DESCRIPTOR_SET: &[u8] = + tonic::include_file_descriptor_set!("tli_descriptor"); +pub const MONITORING_FILE_DESCRIPTOR_SET: &[u8] = + tonic::include_file_descriptor_set!("monitoring_descriptor"); +pub const ML_TRAINING_FILE_DESCRIPTOR_SET: &[u8] = + tonic::include_file_descriptor_set!("ml_training_descriptor"); +pub const TRADING_AGENT_FILE_DESCRIPTOR_SET: &[u8] = + tonic::include_file_descriptor_set!("trading_agent_descriptor"); +pub const TRADING_FILE_DESCRIPTOR_SET: &[u8] = + tonic::include_file_descriptor_set!("trading_descriptor"); +pub const RISK_FILE_DESCRIPTOR_SET: &[u8] = + tonic::include_file_descriptor_set!("risk_descriptor"); +pub const CONFIG_FILE_DESCRIPTOR_SET: &[u8] = + tonic::include_file_descriptor_set!("config_descriptor"); +pub const BROKER_GATEWAY_FILE_DESCRIPTOR_SET: &[u8] = + tonic::include_file_descriptor_set!("broker_gateway_descriptor"); +pub const DATA_ACQUISITION_FILE_DESCRIPTOR_SET: &[u8] = + tonic::include_file_descriptor_set!("data_acquisition_descriptor"); +pub const ML_FILE_DESCRIPTOR_SET: &[u8] = + tonic::include_file_descriptor_set!("ml_descriptor"); + +// Include generated protobuf code FIRST (needed by other modules) +pub mod foxhunt { + pub mod tli { + tonic::include_proto!("foxhunt.tli"); + } +} + +pub mod ml_training { + tonic::include_proto!("ml_training"); +} + +// Trading Agent Service proto (for proxy) +pub mod trading_agent { + tonic::include_proto!("trading_agent"); +} + +// Trading Service backend proto (for protocol translation) +pub mod trading_backend { + tonic::include_proto!("trading"); +} + +// Risk Service backend proto +pub mod risk { + tonic::include_proto!("risk"); +} + +// Monitoring Service backend proto +pub mod monitoring { + tonic::include_proto!("monitoring"); +} + +// Config Service backend proto +pub mod config_backend { + tonic::include_proto!("config"); +} + +// Broker Gateway Service proto +pub mod broker_gateway { + tonic::include_proto!("broker_gateway"); +} + +// Data Acquisition Service proto +pub mod data_acquisition { + tonic::include_proto!("data_acquisition"); +} + +// ML Inference Service proto +pub mod ml_inference { + tonic::include_proto!("ml"); +} + +// Error module MUST come before modules that use it +pub mod error; + +// Now other modules can use error types and proto definitions +pub mod auth; +pub mod config; +pub mod grpc; +pub mod health_router; +pub mod metrics; +pub mod routing; + +// Re-export error types +pub use error::{GatewayConfigError, GatewayConfigResult}; + +// Re-export configuration management types +pub use config::{ + AuthzMetrics, AuthzService, ConfigItem, ConfigValidator, ConfigurationManager, + GatewayConfigService, PermissionResult, +}; + +// Re-export routing and rate limiting types +pub use routing::{CacheStats, RateLimitConfig, RateLimiter}; + +// Re-export gRPC proxy and handler types +pub use grpc::{ + setup_ml_training_client, setup_ml_training_proxy, setup_trading_agent_client, + setup_trading_agent_proxy, BackendHealthState, BacktestingServiceProxy, BrokerGatewayProxy, + ConfigServiceProxy, DataAcquisitionProxy, HealthChecker, MlServiceProxy, MlTradingProxy, + MlTrainingBackendConfig, MlTrainingProxy, MonitoringServiceHandler, RiskServiceProxy, + ServiceHealthEntry, TradingAgentBackendConfig, TradingAgentProxy, TradingDirectProxy, + TradingServiceProxy, +}; + +// Re-export health router types +pub use health_router::{health_router, HealthState}; + diff --git a/services/api/src/main.rs b/services/api/src/main.rs new file mode 100644 index 000000000..edf3323bd --- /dev/null +++ b/services/api/src/main.rs @@ -0,0 +1,773 @@ +#![deny(clippy::unwrap_used, clippy::expect_used)] + +//! Foxhunt API Service +//! +//! Unified gRPC gateway with tonic-web for browser access and 6-layer authentication. +//! Optimized for HFT requirements with <10us authentication overhead. + +use anyhow::Result; +use clap::Parser; +use std::sync::Arc; +use std::time::Duration; +use tracing::{error, info, warn}; + +use common::metrics::GrpcMetricsLayer; +use tonic_web::GrpcWebLayer; +use tower_http::cors::{AllowHeaders, CorsLayer}; + +// Import all needed types from the library +use api::auth::jwt::JwtConfig; +use api::auth::{ + AuditLogger, AuthInterceptor, AuthzService, JwtService, RateLimiter, RevocationService, +}; + +#[derive(Parser, Debug)] +#[command(name = "api", about = "Foxhunt API Service")] +struct Args { + /// gRPC server bind address + #[arg(long, env = "GATEWAY_BIND_ADDR", default_value = "0.0.0.0:50051")] + bind_addr: String, + + /// JWT secret (or use JWT_SECRET_FILE for production) + #[arg(long, env = "JWT_SECRET")] + jwt_secret: Option, + + /// JWT issuer + #[arg(long, env = "JWT_ISSUER", default_value = "foxhunt-trading")] + jwt_issuer: String, + + /// JWT audience + #[arg(long, env = "JWT_AUDIENCE", default_value = "trading-api")] + jwt_audience: String, + + /// Redis URL for JWT revocation + #[arg(long, env = "REDIS_URL", default_value = "redis://localhost:6379")] + redis_url: String, + + /// Rate limit (requests per second per user) + #[arg(long, env = "RATE_LIMIT_RPS", default_value = "100")] + rate_limit_rps: u32, + + /// Enable audit logging + #[arg(long, env = "ENABLE_AUDIT_LOGGING", default_value = "true")] + enable_audit_logging: bool, +} + +#[tokio::main] +async fn main() -> Result<()> { + // Install ring as the default rustls CryptoProvider (required by kube-rs for in-cluster TLS) + if let Err(_already) = rustls::crypto::ring::default_provider().install_default() { + // Another provider already installed -- fine + } + + let args = Args::parse(); + + // Initialize observability (JSON logging + OpenTelemetry tracing via OTLP) + let otlp_endpoint = std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT") + .unwrap_or_else(|_| "http://localhost:4317".to_string()); + if let Err(e) = common::observability::init_observability("api", Some(&otlp_endpoint)) { + eprintln!("Failed to initialize observability: {}", e); + } + + info!("Starting Foxhunt API Service"); + info!("Bind address: {}", args.bind_addr); + info!("JWT issuer: {}", args.jwt_issuer); + info!("JWT audience: {}", args.jwt_audience); + info!("Redis URL: {}", args.redis_url); + info!("Rate limit: {} req/s per user", args.rate_limit_rps); + info!("Audit logging: {}", args.enable_audit_logging); + + // Load JWT configuration (Vault-based with fallback) + info!("Loading JWT configuration..."); + let jwt_config = match load_jwt_config().await { + Ok(config) => { + info!("JWT configuration loaded successfully"); + config + }, + Err(e) => { + error!("Failed to load JWT configuration: {}", e); + return Err(e); + }, + }; + + // Initialize authentication components + info!("Initializing authentication services..."); + + let jwt_service = JwtService::new( + jwt_config.jwt_secret.clone(), + jwt_config.jwt_issuer.clone(), + jwt_config.jwt_audience.clone(), + ); + info!("JWT service initialized with cached decoding key"); + + let revocation_service = RevocationService::new(&args.redis_url) + .await + .map_err(|e| anyhow::anyhow!("Failed to connect to Redis for revocation service: {}", e))?; + info!("JWT revocation service connected to Redis"); + + let authz_service = AuthzService::new(); + info!("Authorization service initialized with permission cache"); + + let rate_limiter = RateLimiter::new(args.rate_limit_rps) + .map_err(|e| anyhow::anyhow!("Failed to create rate limiter: {}", e))?; + info!("Rate limiter initialized ({} req/s)", args.rate_limit_rps); + + let audit_logger = AuditLogger::new(args.enable_audit_logging); + info!("Audit logger initialized"); + + // Create authentication interceptor + let auth_interceptor = AuthInterceptor::new( + jwt_service, + revocation_service, + authz_service, + rate_limiter, + audit_logger, + ); + info!("6-layer authentication interceptor ready"); + + info!("API Service initialization complete"); + info!("Ready to accept gRPC requests with <10us auth overhead"); + + // Initialize backend service proxies + info!("Connecting to backend services..."); + + let trading_backend_url = std::env::var("TRADING_SERVICE_URL") + .unwrap_or_else(|_| "http://localhost:50052".to_string()); + let backtesting_backend_url = std::env::var("BACKTESTING_SERVICE_URL") + .unwrap_or_else(|_| "http://localhost:50053".to_string()); + let ml_training_backend_url = std::env::var("ML_TRAINING_SERVICE_URL") + .unwrap_or_else(|_| "http://localhost:50054".to_string()); + let broker_gw_url = std::env::var("BROKER_GATEWAY_SERVICE_URL") + .unwrap_or_else(|_| "http://localhost:50056".to_string()); + let data_acq_url = std::env::var("DATA_ACQUISITION_SERVICE_URL") + .unwrap_or_else(|_| "http://localhost:50057".to_string()); + let ml_service_url = std::env::var("ML_SERVICE_URL") + .unwrap_or_else(|_| trading_backend_url.clone()); + let trading_agent_url = std::env::var("TRADING_AGENT_SERVICE_URL") + .unwrap_or_else(|_| "http://localhost:50055".to_string()); + let prometheus_url = std::env::var("PROMETHEUS_URL") + .unwrap_or_else(|_| "http://prometheus.foxhunt.svc:80".to_string()); + let monitoring_stream_interval: u32 = std::env::var("MONITORING_STREAM_INTERVAL") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(3); + + // Load TLS certificate paths for backtesting service (mTLS) + let backtesting_tls_ca_cert = std::env::var("BACKTESTING_TLS_CA_CERT").ok(); + let backtesting_tls_client_cert = std::env::var("BACKTESTING_TLS_CLIENT_CERT").ok(); + let backtesting_tls_client_key = std::env::var("BACKTESTING_TLS_CLIENT_KEY").ok(); + + // Load TLS certificate paths for ML training service (mTLS) + let ml_training_tls_ca_cert = std::env::var("ML_TRAINING_TLS_CA_CERT").ok(); + let ml_training_tls_client_cert = std::env::var("ML_TRAINING_TLS_CLIENT_CERT").ok(); + let ml_training_tls_client_key = std::env::var("ML_TRAINING_TLS_CLIENT_KEY").ok(); + + // Initialize trading service proxy + let mut trading_proxy = api::grpc::TradingServiceProxy::new_lazy(&trading_backend_url) + .map_err(|e| anyhow::anyhow!("Failed to create trading service proxy: {}", e))?; + info!( + "Trading service proxy initialized ({})", + trading_backend_url + ); + + // Initialize backtesting service proxy (optional - graceful degradation) + info!("Attempting to initialize Backtesting Service proxy..."); + info!(" Backend URL: {}", backtesting_backend_url); + info!(" CA cert: {:?}", backtesting_tls_ca_cert); + info!(" Client cert: {:?}", backtesting_tls_client_cert); + info!(" Client key: {:?}", backtesting_tls_client_key); + + let backtesting_proxy = match api::grpc::BacktestingServiceProxy::new( + &backtesting_backend_url, + backtesting_tls_ca_cert.as_deref(), + backtesting_tls_client_cert.as_deref(), + backtesting_tls_client_key.as_deref(), + ) + .await + { + Ok(proxy) => { + info!( + "Backtesting service proxy initialized ({})", + backtesting_backend_url + ); + Some(Arc::new(proxy)) + }, + Err(e) => { + error!("Backtesting service initialization failed!"); + error!(" Error type: {:?}", e); + error!(" Error message: {}", e); + warn!("Backtesting service unavailable: {}. API will run without backtesting endpoints.", e); + warn!(" Backtesting endpoints will return 503 Service Unavailable"); + None + }, + }; + + // Spawn background health check task for backtesting service + if let Some(proxy) = backtesting_proxy.as_ref() { + let proxy_clone = Arc::clone(proxy); + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(10)); + loop { + interval.tick().await; + proxy_clone.background_health_check().await; + } + }); + info!("Backtesting service health check task started (10s interval)"); + } + + // Initialize ML training service proxy (optional - graceful degradation) + info!("Attempting to initialize ML Training Service proxy..."); + info!(" Backend URL: {}", ml_training_backend_url); + info!(" CA cert: {:?}", ml_training_tls_ca_cert); + info!(" Client cert: {:?}", ml_training_tls_client_cert); + info!(" Client key: {:?}", ml_training_tls_client_key); + + let ml_config = api::grpc::MlTrainingBackendConfig { + address: ml_training_backend_url.clone(), + connect_timeout_ms: 5000, + request_timeout_ms: 30000, + circuit_breaker_failures: 5, + circuit_breaker_reset_secs: 30, + tls_ca_cert_path: ml_training_tls_ca_cert.clone(), + tls_client_cert_path: ml_training_tls_client_cert.clone(), + tls_client_key_path: ml_training_tls_client_key.clone(), + }; + let ml_training_proxy = match api::grpc::setup_ml_training_proxy(ml_config).await { + Ok(proxy) => { + info!( + "ML training service proxy initialized ({})", + ml_training_backend_url + ); + Some(proxy) + }, + Err(e) => { + error!("ML Training service initialization failed!"); + error!(" Error type: {:?}", e); + error!(" Error message: {}", e); + warn!( + "ML Training service unavailable: {}. API will run without ML endpoints.", + e + ); + warn!(" ML training endpoints will return 503 Service Unavailable"); + None + }, + }; + + // Initialize monitoring service handler (direct Prometheus scraping, no backend needed) + let monitoring_backends = vec![ + ("trading-service".to_string(), trading_backend_url.clone()), + ("backtesting-service".to_string(), backtesting_backend_url.clone()), + ("ml-training-service".to_string(), ml_training_backend_url.clone()), + ("broker-gateway".to_string(), broker_gw_url.clone()), + ("data-acquisition-service".to_string(), data_acq_url.clone()), + ("trading-agent-service".to_string(), trading_agent_url.clone()), + ]; + let monitoring_handler = api::MonitoringServiceHandler::new( + &prometheus_url, + monitoring_stream_interval, + ) + .with_backends(monitoring_backends); + info!( + "Monitoring service handler initialized (Prometheus={}, interval={}s, backends=6)", + prometheus_url, monitoring_stream_interval + ); + + // Initialize direct proxy services (trading, risk, config, ML, broker_gw, data_acq, trading_agent) + let trading_channel = tonic::transport::Channel::from_shared(trading_backend_url.clone()) + .map_err(|e| anyhow::anyhow!("Invalid TRADING_SERVICE_URL: {e}"))? + .connect_lazy(); + + let trading_direct_proxy = api::TradingDirectProxy::new( + api::trading_backend::trading_service_client::TradingServiceClient::new( + trading_channel.clone(), + ), + ); + let risk_proxy = api::RiskServiceProxy::new( + api::risk::risk_service_client::RiskServiceClient::new(trading_channel.clone()), + ); + let config_direct_proxy = api::ConfigServiceProxy::new( + api::config_backend::config_service_client::ConfigServiceClient::new( + trading_channel, + ), + ); + + let ml_channel = tonic::transport::Channel::from_shared(ml_service_url.clone()) + .map_err(|e| anyhow::anyhow!("Invalid ML_SERVICE_URL: {e}"))? + .connect_lazy(); + let ml_proxy = api::MlServiceProxy::new( + api::ml_inference::ml_service_client::MlServiceClient::new(ml_channel), + ); + + let broker_channel = tonic::transport::Channel::from_shared(broker_gw_url.clone()) + .map_err(|e| anyhow::anyhow!("Invalid BROKER_GATEWAY_SERVICE_URL: {e}"))? + .connect_lazy(); + let broker_proxy = api::BrokerGatewayProxy::new( + api::broker_gateway::broker_gateway_service_client::BrokerGatewayServiceClient::new( + broker_channel, + ), + ); + + let data_channel = tonic::transport::Channel::from_shared(data_acq_url.clone()) + .map_err(|e| anyhow::anyhow!("Invalid DATA_ACQUISITION_SERVICE_URL: {e}"))? + .connect_lazy(); + let data_proxy = api::DataAcquisitionProxy::new( + api::data_acquisition::data_acquisition_service_client::DataAcquisitionServiceClient::new( + data_channel, + ), + ); + + let agent_channel = tonic::transport::Channel::from_shared(trading_agent_url.clone()) + .map_err(|e| anyhow::anyhow!("Invalid TRADING_AGENT_SERVICE_URL: {e}"))? + .connect_lazy(); + let agent_proxy = api::TradingAgentProxy::new( + api::trading_agent::trading_agent_service_client::TradingAgentServiceClient::new( + agent_channel, + ), + ); + + info!("Direct proxy services initialized:"); + info!(" - trading.TradingService -> {}", trading_backend_url); + info!(" - risk.RiskService -> {}", trading_backend_url); + info!(" - config.ConfigService -> {}", trading_backend_url); + info!(" - ml.MLService -> {}", ml_service_url); + info!(" - broker_gateway.BrokerGatewayService -> {}", broker_gw_url); + info!(" - data_acquisition.DataAcquisitionService -> {}", data_acq_url); + info!(" - trading_agent.TradingAgentService -> {}", trading_agent_url); + + // Setup backend health monitoring for SubscribeSystemStatus. + // A background task checks gRPC health on each backend and pushes + // updates through a watch channel consumed by the streaming RPC. + let (health_tx, health_rx) = api::BackendHealthState::new(); + trading_proxy.set_health_state(health_rx); + { + use tonic_health::pb::health_client::HealthClient; + + // Build persistent lazy channels (reused across checks) + let backends: Vec<(&str, tonic::transport::Channel)> = vec![ + ("trading-service", tonic::transport::Channel::from_shared(trading_backend_url.clone()) + .map_err(|e| anyhow::anyhow!("invalid trading URL: {}", e))?.connect_lazy()), + ("ml-training-service", tonic::transport::Channel::from_shared(ml_training_backend_url.clone()) + .map_err(|e| anyhow::anyhow!("invalid ml-training URL: {}", e))?.connect_lazy()), + ("backtesting-service", tonic::transport::Channel::from_shared(backtesting_backend_url.clone()) + .map_err(|e| anyhow::anyhow!("invalid backtesting URL: {}", e))?.connect_lazy()), + ("broker-gateway", tonic::transport::Channel::from_shared(broker_gw_url.clone()) + .map_err(|e| anyhow::anyhow!("invalid broker-gateway URL: {}", e))?.connect_lazy()), + ("data-acquisition-service", tonic::transport::Channel::from_shared(data_acq_url.clone()) + .map_err(|e| anyhow::anyhow!("invalid data-acquisition URL: {}", e))?.connect_lazy()), + ("trading-agent-service", tonic::transport::Channel::from_shared(trading_agent_url.clone()) + .map_err(|e| anyhow::anyhow!("invalid trading-agent URL: {}", e))?.connect_lazy()), + ]; + let backends: Vec<(String, tonic::transport::Channel)> = backends + .into_iter() + .map(|(n, c)| (n.to_string(), c)) + .collect(); + + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(5)); + loop { + interval.tick().await; + let mut entries = Vec::with_capacity(backends.len()); + for (name, channel) in &backends { + let mut hc = HealthClient::new(channel.clone()); + let req = tonic_health::pb::HealthCheckRequest { + service: String::new(), // overall server health + }; + let (healthy, msg) = match tokio::time::timeout( + Duration::from_millis(2000), + hc.check(req), + ) + .await + { + Ok(Ok(_)) => (true, "Serving"), + // Unimplemented means gRPC server is UP but has no health service + Ok(Err(s)) if s.code() == tonic::Code::Unimplemented => { + (true, "Serving (no health check)") + } + Ok(Err(_)) => (false, "Error"), + Err(_) => (false, "Unreachable"), + }; + entries.push(api::ServiceHealthEntry { + name: name.clone(), + healthy, + message: msg.to_string(), + }); + } + let _ = health_tx.send(entries); + } + }); + info!("Backend health monitor started (5s interval)"); + } + + // Initialize configuration manager (requires database) + let database_url = std::env::var("DATABASE_URL") + .map_err(|_| anyhow::anyhow!("DATABASE_URL environment variable must be set"))?; + let db_pool = sqlx::PgPool::connect(&database_url) + .await + .map_err(|e| anyhow::anyhow!("Failed to connect to database: {}", e))?; + info!("Database connection established"); + + // Create Redis connection for config manager + let redis_client = redis::Client::open(args.redis_url.clone()) + .map_err(|e| anyhow::anyhow!("Failed to create Redis client: {}", e))?; + let redis_conn = redis::aio::ConnectionManager::new(redis_client) + .await + .map_err(|e| anyhow::anyhow!("Failed to create Redis connection manager: {}", e))?; + + let mut config_manager = api::ConfigurationManager::new(db_pool.clone(), redis_conn) + .await + .map_err(|e| anyhow::anyhow!("Failed to create configuration manager: {}", e))?; + + // Start NOTIFY listener for hot-reload + config_manager + .start_listening() + .await + .map_err(|e| anyhow::anyhow!("Failed to start configuration listener: {}", e))?; + info!("Configuration manager initialized with hot-reload"); + + // Load TLS configuration if enabled (Wave H1 Security Enforcement) + let tls_enabled = std::env::var("TLS_ENABLED") + .unwrap_or_else(|_| "false".to_string()) + .parse::() + .unwrap_or(false); + + let tls_config = if tls_enabled { + info!("TLS/mTLS enabled - initializing TLS 1.3 configuration"); + + let cert_path = std::env::var("TLS_CERT_PATH") + .unwrap_or_else(|_| "./certs/server-cert.pem".to_string()); + let key_path = + std::env::var("TLS_KEY_PATH").unwrap_or_else(|_| "./certs/server-key.pem".to_string()); + let ca_path = + std::env::var("TLS_CA_PATH").unwrap_or_else(|_| "./certs/ca/ca-cert.pem".to_string()); + let require_client_cert = std::env::var("TLS_REQUIRE_CLIENT_CERT") + .unwrap_or_else(|_| "true".to_string()) + .parse::() + .unwrap_or(true); + let enable_revocation = std::env::var("MTLS_ENABLE_REVOCATION_CHECK") + .unwrap_or_else(|_| "false".to_string()) + .parse::() + .unwrap_or(false); + let crl_url = std::env::var("MTLS_CRL_URL").ok(); + + let tls = api::auth::mtls::ApiGatewayTlsConfig::from_files( + &cert_path, + &key_path, + &ca_path, + require_client_cert, + enable_revocation, + crl_url, + ) + .await?; + + info!( + "TLS configuration loaded - Protocol: TLS 1.3, mTLS: {}, Revocation: {}", + require_client_cert, enable_revocation + ); + Some(tls) + } else { + info!("TLS disabled - running in development mode (set TLS_ENABLED=true for production)"); + None + }; + + // Build gRPC server with all services + use api::foxhunt::tli::{ + backtesting_service_server::BacktestingServiceServer, + trading_service_server::TradingServiceServer, + }; + use api::ml_training::ml_training_service_server::MlTrainingServiceServer; + use api::monitoring::monitoring_service_server::MonitoringServiceServer; + + let addr: std::net::SocketAddr = args + .bind_addr + .parse() + .map_err(|e| anyhow::anyhow!("Failed to parse bind address '{}': {}", args.bind_addr, e))?; + + info!("Starting gRPC server on {}", addr); + + // Setup graceful shutdown + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + + // Spawn signal handler for graceful shutdown + tokio::spawn(async move { + match tokio::signal::ctrl_c().await { + Ok(()) => { + info!("Received shutdown signal, draining connections..."); + } + Err(e) => { + error!("Failed to listen for ctrl-c signal: {}", e); + } + } + let _ = shutdown_tx.send(()); + }); + + // Setup health check service + let (health_reporter, health_service) = tonic_health::server::health_reporter(); + + // Always register trading service (required) + health_reporter + .set_serving::>() + .await; + + // Conditionally register optional services + if backtesting_proxy.is_some() { + health_reporter + .set_serving::>() + .await; + } + if ml_training_proxy.is_some() { + health_reporter + .set_serving::>() + .await; + } + // Monitoring handler is always available (direct Prometheus, no backend dependency) + health_reporter + .set_serving::>() + .await; + + // Initialize and start Prometheus metrics HTTP endpoint on port 9091 + let gateway_metrics = api::metrics::GatewayMetrics::new() + .map_err(|e| anyhow::anyhow!("Failed to initialize gateway metrics: {}", e))?; + + // Add service info metric (always present) + use prometheus::{register_gauge_with_registry, Opts}; + let service_info = register_gauge_with_registry!( + Opts::new( + "api_service_info", + "API service information" + ) + .const_label("version", env!("CARGO_PKG_VERSION")) + .const_label("service", "api"), + gateway_metrics.registry().as_ref() + ) + .map_err(|e| anyhow::anyhow!("Failed to register service info: {}", e))?; + service_info.set(1.0); + + let metrics_registry = gateway_metrics.registry(); + + tokio::spawn(async move { + // Use combined router for metrics AND health/resilience endpoints + let combined_app = api::metrics::combined_router(metrics_registry); + + let metrics_addr = "0.0.0.0:9091"; + info!( + "Prometheus metrics endpoint listening on http://{}", + metrics_addr + ); + info!("Health endpoints available:"); + info!(" - GET http://{}/health/liveness", metrics_addr); + info!(" - GET http://{}/health/readiness", metrics_addr); + info!(" - GET http://{}/health/startup", metrics_addr); + info!( + " - GET http://{}/resilience/circuit-breaker/status", + metrics_addr + ); + info!( + " - GET http://{}/resilience/rate-limit/status", + metrics_addr + ); + info!(" - GET http://{}/resilience/timeout/config", metrics_addr); + info!(" - GET http://{}/resilience/retry/config", metrics_addr); + + let listener = match tokio::net::TcpListener::bind(metrics_addr).await { + Ok(l) => l, + Err(e) => { + error!("Failed to bind metrics endpoint on {}: {}", metrics_addr, e); + return; + } + }; + + if let Err(e) = axum::serve(listener, combined_app).await { + error!("Metrics server failed: {}", e); + } + }); + + // Configure CORS for gRPC-Web browser access + let cors_origins = std::env::var("CORS_ORIGINS") + .unwrap_or_else(|_| "http://localhost:5173".to_string()); + let origins: Vec = cors_origins + .split(',') + .filter_map(|s| s.trim().parse().ok()) + .collect(); + + let cors = CorsLayer::new() + .allow_origin(origins) + .allow_headers(AllowHeaders::any()) + .allow_methods([http::Method::POST, http::Method::OPTIONS]) + .expose_headers([ + http::HeaderName::from_static("grpc-status"), + http::HeaderName::from_static("grpc-message"), + ]); + + // Build server with HTTP/2 optimizations and gRPC-Web support + let mut server_builder = if let Some(ref tls) = tls_config { + tonic::transport::Server::builder() + .accept_http1(true) // Required for grpc-web + .layer(GrpcMetricsLayer::new("api")) + .layer(cors.clone()) + .layer(GrpcWebLayer::new()) + .tls_config(tls.to_server_tls_config())? + .max_concurrent_streams(Some(10_000)) + .http2_keepalive_interval(Some(Duration::from_secs(30))) + .http2_keepalive_timeout(Some(Duration::from_secs(10))) + } else { + tonic::transport::Server::builder() + .accept_http1(true) // Required for grpc-web + .layer(GrpcMetricsLayer::new("api")) + .layer(cors.clone()) + .layer(GrpcWebLayer::new()) + .max_concurrent_streams(Some(10_000)) + .http2_keepalive_interval(Some(Duration::from_secs(30))) + .http2_keepalive_timeout(Some(Duration::from_secs(10))) + } + .layer(tower::ServiceBuilder::new().layer(tower::layer::util::Identity::new())); // Placeholder for auth interceptor layer + + // Add gRPC reflection service (unauthenticated -- for grpcurl/debugging) + let reflection_service = tonic_reflection::server::Builder::configure() + .register_encoded_file_descriptor_set(api::TLI_FILE_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(api::MONITORING_FILE_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(api::ML_TRAINING_FILE_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(api::TRADING_AGENT_FILE_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(api::TRADING_FILE_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(api::RISK_FILE_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(api::CONFIG_FILE_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(api::BROKER_GATEWAY_FILE_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(api::DATA_ACQUISITION_FILE_DESCRIPTOR_SET) + .register_encoded_file_descriptor_set(api::ML_FILE_DESCRIPTOR_SET) + .build_v1() + .map_err(|e| anyhow::anyhow!("Failed to build reflection service: {}", e))?; + + // Add health + reflection services + let mut router = server_builder + .add_service(health_service) + .add_service(reflection_service); + + // Always add trading service (required) with authentication + router = router.add_service(TradingServiceServer::with_interceptor( + trading_proxy, + auth_interceptor.clone(), + )); + + // Track service availability for logging + let backtesting_available = backtesting_proxy.is_some(); + let ml_training_available = ml_training_proxy.is_some(); + + // Conditionally add optional services with authentication + if let Some(backtesting) = backtesting_proxy.as_ref() { + // Clone the entire Arc - tonic services can work with Arc-wrapped implementations + router = router.add_service(BacktestingServiceServer::with_interceptor( + Arc::clone(backtesting), + auth_interceptor.clone(), + )); + } + if let Some(ml_training) = ml_training_proxy { + router = router.add_service(MlTrainingServiceServer::with_interceptor( + ml_training, + auth_interceptor.clone(), + )); + } + + // Monitoring handler is always available (direct Prometheus scraping, no backend) + router = router.add_service(MonitoringServiceServer::with_interceptor( + monitoring_handler, + auth_interceptor.clone(), + )); + + // Register all direct proxy services with auth interceptor + { + use api::trading_backend::trading_service_server::TradingServiceServer as TradingDirectServer; + use api::risk::risk_service_server::RiskServiceServer; + use api::broker_gateway::broker_gateway_service_server::BrokerGatewayServiceServer; + use api::data_acquisition::data_acquisition_service_server::DataAcquisitionServiceServer; + use api::ml_inference::ml_service_server::MlServiceServer; + use api::config_backend::config_service_server::ConfigServiceServer as ConfigBackendServer; + use api::trading_agent::trading_agent_service_server::TradingAgentServiceServer; + + router = router + .add_service(TradingDirectServer::with_interceptor( + trading_direct_proxy, + auth_interceptor.clone(), + )) + .add_service(RiskServiceServer::with_interceptor( + risk_proxy, + auth_interceptor.clone(), + )) + .add_service(BrokerGatewayServiceServer::with_interceptor( + broker_proxy, + auth_interceptor.clone(), + )) + .add_service(DataAcquisitionServiceServer::with_interceptor( + data_proxy, + auth_interceptor.clone(), + )) + .add_service(MlServiceServer::with_interceptor( + ml_proxy, + auth_interceptor.clone(), + )) + .add_service(ConfigBackendServer::with_interceptor( + config_direct_proxy, + auth_interceptor.clone(), + )) + .add_service(TradingAgentServiceServer::with_interceptor( + agent_proxy, + auth_interceptor.clone(), + )); + } + + // Log startup information + info!("API Service listening on {}", addr); + info!(" - gRPC-Web: enabled (CORS origins: {})", cors_origins); + info!(" - Trading Service: {} (REQUIRED)", trading_backend_url); + info!( + " - Backtesting Service: {} ({})", + backtesting_backend_url, + if backtesting_available { + "AVAILABLE" + } else { + "UNAVAILABLE" + } + ); + info!( + " - ML Training Service: {} ({})", + ml_training_backend_url, + if ml_training_available { + "AVAILABLE" + } else { + "UNAVAILABLE" + } + ); + info!( + " - Monitoring: direct Prometheus scraping ({})", + prometheus_url + ); + info!(" - trading.TradingService -> {}", trading_backend_url); + info!(" - risk.RiskService -> {}", trading_backend_url); + info!(" - config.ConfigService -> {}", trading_backend_url); + info!(" - ml.MLService -> {}", ml_service_url); + info!(" - broker_gateway.BrokerGatewayService -> {}", broker_gw_url); + info!(" - data_acquisition.DataAcquisitionService -> {}", data_acq_url); + info!(" - trading_agent.TradingAgentService -> {}", trading_agent_url); + info!(" - Health checks: enabled"); + info!(" - Authentication: 6-layer (<10us overhead)"); + info!(" - Rate limiting: {} req/s per user", args.rate_limit_rps); + + // Start server with graceful shutdown + let server = router.serve_with_shutdown(addr, async { + shutdown_rx.await.ok(); + }); + + // Start server + server.await?; + + info!("API Service shutdown complete"); + Ok(()) +} + +/// Load JWT configuration from Vault (production) or environment (development) +/// +/// Priority: +/// 1. Vault (secret/foxhunt/jwt) - Production +/// 2. JWT_SECRET_FILE - File-based secret +/// 3. JWT_SECRET env var - Development fallback +async fn load_jwt_config() -> Result { + JwtConfig::new().await +} diff --git a/services/api/src/metrics/auth_metrics.rs b/services/api/src/metrics/auth_metrics.rs new file mode 100644 index 000000000..fb562539e --- /dev/null +++ b/services/api/src/metrics/auth_metrics.rs @@ -0,0 +1,412 @@ +//! Authentication Metrics for 6-Layer Security +//! +//! Tracks performance and errors for: +//! - JWT extraction and validation +//! - Revocation checking (Redis) +//! - RBAC permission checks (cached) +//! - Rate limiting (token bucket) +//! - MFA verification +//! - Audit logging + +use prometheus::{Counter, CounterVec, Histogram, HistogramOpts, IntGauge, Opts, Registry}; + +/// Authentication metrics for all 6 layers +pub struct AuthMetrics { + // === Request Counters === + /// Total authentication requests + pub auth_requests_total: Counter, + /// Successful authentications + pub auth_requests_success: Counter, + /// Failed authentications + pub auth_requests_failure: Counter, + + // === Layer-Specific Latencies (microseconds) === + /// JWT extraction and parsing latency + pub jwt_extraction_duration_us: Histogram, + /// JWT signature validation latency + pub jwt_validation_duration_us: Histogram, + /// JWT revocation check latency (Redis) + pub revocation_check_duration_us: Histogram, + /// RBAC permission check latency + pub rbac_check_duration_us: Histogram, + /// Rate limit check latency + pub rate_limit_check_duration_us: Histogram, + /// MFA verification latency (TOTP) + pub mfa_verification_duration_us: Histogram, + /// Total auth pipeline latency + pub auth_total_duration_us: Histogram, + + // === Error Counters by Type === + /// Missing Authorization header + pub auth_errors_missing_jwt: Counter, + /// Malformed JWT token + pub auth_errors_invalid_jwt: Counter, + /// JWT signature verification failed + pub auth_errors_signature_failed: Counter, + /// JWT expired + pub auth_errors_expired_jwt: Counter, + /// JWT revoked (blacklisted) + pub auth_errors_revoked_jwt: Counter, + /// RBAC permission denied + pub auth_errors_permission_denied: Counter, + /// Rate limit exceeded + pub auth_errors_rate_limited: Counter, + /// MFA verification failed + pub auth_errors_mfa_failed: Counter, + /// Redis connection errors + pub auth_errors_redis_failure: Counter, + + // === Current State Gauges === + /// Number of active JWT tokens (estimated) + pub active_jwt_tokens: IntGauge, + /// Number of revoked tokens in cache + pub revoked_tokens_cached: IntGauge, + /// RBAC permission cache size + pub rbac_cache_size: IntGauge, + /// Rate limiter entries + pub rate_limiter_entries: IntGauge, + + // === Per-User Metrics === + /// Requests per user (labeled by user_id) + pub requests_by_user: CounterVec, + /// Auth failures per user + pub auth_failures_by_user: CounterVec, + /// Rate limit hits per user + pub rate_limits_by_user: CounterVec, + + // === Cache Performance === + /// JWT decoding key cache hits + pub jwt_cache_hits: Counter, + /// JWT decoding key cache misses + pub jwt_cache_misses: Counter, + /// RBAC permission cache hits + pub rbac_cache_hits: Counter, + /// RBAC permission cache misses + pub rbac_cache_misses: Counter, + + // === Performance Targets === + /// Counter for auth requests meeting <10μs SLA + pub auth_sla_met: Counter, + /// Counter for auth requests exceeding <10μs SLA + pub auth_sla_exceeded: Counter, +} + +impl AuthMetrics { + /// Create new authentication metrics and register with Prometheus + pub fn new(registry: &Registry) -> Result { + // === Request Counters === + let auth_requests_total = Counter::with_opts(Opts::new( + "api_auth_requests_total", + "Total authentication requests", + ))?; + registry.register(Box::new(auth_requests_total.clone()))?; + + let auth_requests_success = Counter::with_opts(Opts::new( + "api_auth_requests_success", + "Successful authentication requests", + ))?; + registry.register(Box::new(auth_requests_success.clone()))?; + + let auth_requests_failure = Counter::with_opts(Opts::new( + "api_auth_requests_failure", + "Failed authentication requests", + ))?; + registry.register(Box::new(auth_requests_failure.clone()))?; + + // === Layer-Specific Latencies (microseconds) === + // Buckets: 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100 microseconds + let latency_buckets = vec![0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0]; + + let jwt_extraction_duration_us = Histogram::with_opts( + HistogramOpts::new( + "api_jwt_extraction_duration_microseconds", + "JWT extraction and parsing latency in microseconds", + ) + .buckets(latency_buckets.clone()), + )?; + registry.register(Box::new(jwt_extraction_duration_us.clone()))?; + + let jwt_validation_duration_us = Histogram::with_opts( + HistogramOpts::new( + "api_jwt_validation_duration_microseconds", + "JWT signature validation latency in microseconds", + ) + .buckets(latency_buckets.clone()), + )?; + registry.register(Box::new(jwt_validation_duration_us.clone()))?; + + let revocation_check_duration_us = Histogram::with_opts( + HistogramOpts::new( + "api_revocation_check_duration_microseconds", + "JWT revocation check latency in microseconds (Redis)", + ) + .buckets(latency_buckets), + )?; + registry.register(Box::new(revocation_check_duration_us.clone()))?; + + let rbac_check_duration_us = Histogram::with_opts( + HistogramOpts::new( + "api_rbac_check_duration_microseconds", + "RBAC permission check latency in microseconds", + ) + .buckets(vec![0.05, 0.1, 0.2, 0.5, 1.0, 2.0, 5.0]), // Sub-microsecond buckets + )?; + registry.register(Box::new(rbac_check_duration_us.clone()))?; + + let rate_limit_check_duration_us = Histogram::with_opts( + HistogramOpts::new( + "api_rate_limit_check_duration_microseconds", + "Rate limit check latency in microseconds", + ) + .buckets(vec![0.01, 0.05, 0.1, 0.2, 0.5, 1.0]), // Ultra-fast atomic operations + )?; + registry.register(Box::new(rate_limit_check_duration_us.clone()))?; + + let mfa_verification_duration_us = Histogram::with_opts( + HistogramOpts::new( + "api_mfa_verification_duration_microseconds", + "MFA (TOTP) verification latency in microseconds", + ) + .buckets(vec![10.0, 50.0, 100.0, 500.0, 1000.0]), // TOTP is slower + )?; + registry.register(Box::new(mfa_verification_duration_us.clone()))?; + + let auth_total_duration_us = Histogram::with_opts( + HistogramOpts::new( + "api_auth_total_duration_microseconds", + "Total authentication pipeline latency in microseconds", + ) + .buckets(vec![1.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0]), + )?; + registry.register(Box::new(auth_total_duration_us.clone()))?; + + // === Error Counters === + let auth_errors_missing_jwt = Counter::with_opts(Opts::new( + "api_auth_errors_missing_jwt", + "Missing Authorization header errors", + ))?; + registry.register(Box::new(auth_errors_missing_jwt.clone()))?; + + let auth_errors_invalid_jwt = Counter::with_opts(Opts::new( + "api_auth_errors_invalid_jwt", + "Malformed JWT token errors", + ))?; + registry.register(Box::new(auth_errors_invalid_jwt.clone()))?; + + let auth_errors_signature_failed = Counter::with_opts(Opts::new( + "api_auth_errors_signature_failed", + "JWT signature verification failures", + ))?; + registry.register(Box::new(auth_errors_signature_failed.clone()))?; + + let auth_errors_expired_jwt = Counter::with_opts(Opts::new( + "api_auth_errors_expired_jwt", + "Expired JWT token errors", + ))?; + registry.register(Box::new(auth_errors_expired_jwt.clone()))?; + + let auth_errors_revoked_jwt = Counter::with_opts(Opts::new( + "api_auth_errors_revoked_jwt", + "Revoked JWT token errors (blacklisted)", + ))?; + registry.register(Box::new(auth_errors_revoked_jwt.clone()))?; + + let auth_errors_permission_denied = Counter::with_opts(Opts::new( + "api_auth_errors_permission_denied", + "RBAC permission denied errors", + ))?; + registry.register(Box::new(auth_errors_permission_denied.clone()))?; + + let auth_errors_rate_limited = Counter::with_opts(Opts::new( + "api_auth_errors_rate_limited", + "Rate limit exceeded errors", + ))?; + registry.register(Box::new(auth_errors_rate_limited.clone()))?; + + let auth_errors_mfa_failed = Counter::with_opts(Opts::new( + "api_auth_errors_mfa_failed", + "MFA verification failures", + ))?; + registry.register(Box::new(auth_errors_mfa_failed.clone()))?; + + let auth_errors_redis_failure = Counter::with_opts(Opts::new( + "api_auth_errors_redis_failure", + "Redis connection/operation failures", + ))?; + registry.register(Box::new(auth_errors_redis_failure.clone()))?; + + // === Current State Gauges === + let active_jwt_tokens = IntGauge::with_opts(Opts::new( + "api_active_jwt_tokens", + "Number of active JWT tokens (estimated)", + ))?; + registry.register(Box::new(active_jwt_tokens.clone()))?; + + let revoked_tokens_cached = IntGauge::with_opts(Opts::new( + "api_revoked_tokens_cached", + "Number of revoked tokens in Redis cache", + ))?; + registry.register(Box::new(revoked_tokens_cached.clone()))?; + + let rbac_cache_size = IntGauge::with_opts(Opts::new( + "api_rbac_cache_size", + "RBAC permission cache entries", + ))?; + registry.register(Box::new(rbac_cache_size.clone()))?; + + let rate_limiter_entries = IntGauge::with_opts(Opts::new( + "api_rate_limiter_entries", + "Rate limiter tracked users", + ))?; + registry.register(Box::new(rate_limiter_entries.clone()))?; + + // === Per-User Metrics === + let requests_by_user = CounterVec::new( + Opts::new("api_requests_by_user", "Requests per user"), + &["user_id"], + )?; + registry.register(Box::new(requests_by_user.clone()))?; + + let auth_failures_by_user = CounterVec::new( + Opts::new( + "api_auth_failures_by_user", + "Auth failures per user", + ), + &["user_id", "reason"], + )?; + registry.register(Box::new(auth_failures_by_user.clone()))?; + + let rate_limits_by_user = CounterVec::new( + Opts::new( + "api_rate_limits_by_user", + "Rate limit hits per user", + ), + &["user_id"], + )?; + registry.register(Box::new(rate_limits_by_user.clone()))?; + + // === Cache Performance === + let jwt_cache_hits = Counter::with_opts(Opts::new( + "api_jwt_cache_hits", + "JWT decoding key cache hits", + ))?; + registry.register(Box::new(jwt_cache_hits.clone()))?; + + let jwt_cache_misses = Counter::with_opts(Opts::new( + "api_jwt_cache_misses", + "JWT decoding key cache misses", + ))?; + registry.register(Box::new(jwt_cache_misses.clone()))?; + + let rbac_cache_hits = Counter::with_opts(Opts::new( + "api_rbac_cache_hits", + "RBAC permission cache hits", + ))?; + registry.register(Box::new(rbac_cache_hits.clone()))?; + + let rbac_cache_misses = Counter::with_opts(Opts::new( + "api_rbac_cache_misses", + "RBAC permission cache misses", + ))?; + registry.register(Box::new(rbac_cache_misses.clone()))?; + + // === Performance SLA === + let auth_sla_met = Counter::with_opts(Opts::new( + "api_auth_sla_met", + "Auth requests meeting <10μs SLA", + ))?; + registry.register(Box::new(auth_sla_met.clone()))?; + + let auth_sla_exceeded = Counter::with_opts(Opts::new( + "api_auth_sla_exceeded", + "Auth requests exceeding <10μs SLA", + ))?; + registry.register(Box::new(auth_sla_exceeded.clone()))?; + + Ok(Self { + auth_requests_total, + auth_requests_success, + auth_requests_failure, + jwt_extraction_duration_us, + jwt_validation_duration_us, + revocation_check_duration_us, + rbac_check_duration_us, + rate_limit_check_duration_us, + mfa_verification_duration_us, + auth_total_duration_us, + auth_errors_missing_jwt, + auth_errors_invalid_jwt, + auth_errors_signature_failed, + auth_errors_expired_jwt, + auth_errors_revoked_jwt, + auth_errors_permission_denied, + auth_errors_rate_limited, + auth_errors_mfa_failed, + auth_errors_redis_failure, + active_jwt_tokens, + revoked_tokens_cached, + rbac_cache_size, + rate_limiter_entries, + requests_by_user, + auth_failures_by_user, + rate_limits_by_user, + jwt_cache_hits, + jwt_cache_misses, + rbac_cache_hits, + rbac_cache_misses, + auth_sla_met, + auth_sla_exceeded, + }) + } + + /// Record successful authentication with latency breakdown + pub fn record_success(&self, total_duration_us: f64) { + self.auth_requests_total.inc(); + self.auth_requests_success.inc(); + self.auth_total_duration_us.observe(total_duration_us); + + // Check SLA: <10μs target + if total_duration_us < 10.0 { + self.auth_sla_met.inc(); + } else { + self.auth_sla_exceeded.inc(); + } + } + + /// Record authentication failure with reason + pub fn record_failure(&self, reason: &str, user_id: Option<&str>) { + self.auth_requests_total.inc(); + self.auth_requests_failure.inc(); + + // Increment specific error counter + match reason { + "missing_jwt" => self.auth_errors_missing_jwt.inc(), + "invalid_jwt" => self.auth_errors_invalid_jwt.inc(), + "signature_failed" => self.auth_errors_signature_failed.inc(), + "expired_jwt" => self.auth_errors_expired_jwt.inc(), + "revoked_jwt" => self.auth_errors_revoked_jwt.inc(), + "permission_denied" => self.auth_errors_permission_denied.inc(), + "rate_limited" => self.auth_errors_rate_limited.inc(), + "mfa_failed" => self.auth_errors_mfa_failed.inc(), + "redis_failure" => self.auth_errors_redis_failure.inc(), + _ => {}, // Unknown error type + } + + // Track per-user failures + if let Some(uid) = user_id { + self.auth_failures_by_user + .with_label_values(&[uid, reason]) + .inc(); + } + } + + /// Record per-user request + pub fn record_user_request(&self, user_id: &str) { + self.requests_by_user.with_label_values(&[user_id]).inc(); + } + + /// Record rate limit hit for user + pub fn record_rate_limit(&self, user_id: &str) { + self.rate_limits_by_user.with_label_values(&[user_id]).inc(); + } +} diff --git a/services/api/src/metrics/config_metrics.rs b/services/api/src/metrics/config_metrics.rs new file mode 100644 index 000000000..ff68a87d1 --- /dev/null +++ b/services/api/src/metrics/config_metrics.rs @@ -0,0 +1,231 @@ +//! Configuration Management Metrics +//! +//! Tracks hot-reload events and configuration performance: +//! - PostgreSQL NOTIFY/LISTEN events +//! - Configuration cache hits/misses +//! - Hot-reload latency +//! - NOTIFY listener connection health + +use prometheus::{Counter, Histogram, HistogramOpts, IntGauge, Opts, Registry}; + +/// Configuration and hot-reload metrics +pub struct ConfigMetrics { + // === NOTIFY/LISTEN Events === + /// Total NOTIFY events received + pub notify_events_total: Counter, + /// NOTIFY events processed successfully + pub notify_events_success: Counter, + /// NOTIFY events failed to process + pub notify_events_failure: Counter, + + // === Configuration Cache === + /// Configuration cache hits + pub config_cache_hits: Counter, + /// Configuration cache misses + pub config_cache_misses: Counter, + /// Configuration cache entries + pub config_cache_size: IntGauge, + + // === Hot-Reload Latency === + /// Time from NOTIFY to config reload completion + pub config_reload_duration_ms: Histogram, + /// Time to fetch config from PostgreSQL + pub config_fetch_duration_ms: Histogram, + + // === NOTIFY Listener Health === + /// NOTIFY listener connection status (0=disconnected, 1=connected) + pub notify_listener_connected: IntGauge, + /// NOTIFY listener reconnections + pub notify_listener_reconnections: Counter, + /// NOTIFY listener errors + pub notify_listener_errors: Counter, + + // === Configuration Updates === + /// Auth configuration updates + pub config_updates_auth: Counter, + /// Routing configuration updates + pub config_updates_routing: Counter, + /// Rate limit configuration updates + pub config_updates_rate_limit: Counter, + /// Backend endpoint updates + pub config_updates_backend: Counter, + + // === Configuration Validation === + /// Configuration validation successes + pub config_validation_success: Counter, + /// Configuration validation failures + pub config_validation_failure: Counter, +} + +impl ConfigMetrics { + /// Create new config metrics and register with Prometheus + pub fn new(registry: &Registry) -> Result { + // === NOTIFY/LISTEN Events === + let notify_events_total = Counter::with_opts(Opts::new( + "api_notify_events_total", + "Total PostgreSQL NOTIFY events received", + ))?; + registry.register(Box::new(notify_events_total.clone()))?; + + let notify_events_success = Counter::with_opts(Opts::new( + "api_notify_events_success", + "Successfully processed NOTIFY events", + ))?; + registry.register(Box::new(notify_events_success.clone()))?; + + let notify_events_failure = Counter::with_opts(Opts::new( + "api_notify_events_failure", + "Failed to process NOTIFY events", + ))?; + registry.register(Box::new(notify_events_failure.clone()))?; + + // === Configuration Cache === + let config_cache_hits = Counter::with_opts(Opts::new( + "api_config_cache_hits", + "Configuration cache hits", + ))?; + registry.register(Box::new(config_cache_hits.clone()))?; + + let config_cache_misses = Counter::with_opts(Opts::new( + "api_config_cache_misses", + "Configuration cache misses", + ))?; + registry.register(Box::new(config_cache_misses.clone()))?; + + let config_cache_size = IntGauge::with_opts(Opts::new( + "api_config_cache_size", + "Number of cached configuration entries", + ))?; + registry.register(Box::new(config_cache_size.clone()))?; + + // === Hot-Reload Latency === + let config_reload_duration_ms = Histogram::with_opts( + HistogramOpts::new( + "api_config_reload_duration_milliseconds", + "Time from NOTIFY to config reload completion in milliseconds", + ) + .buckets(vec![1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0]), + )?; + registry.register(Box::new(config_reload_duration_ms.clone()))?; + + let config_fetch_duration_ms = Histogram::with_opts( + HistogramOpts::new( + "api_config_fetch_duration_milliseconds", + "Time to fetch configuration from PostgreSQL in milliseconds", + ) + .buckets(vec![1.0, 5.0, 10.0, 25.0, 50.0, 100.0]), + )?; + registry.register(Box::new(config_fetch_duration_ms.clone()))?; + + // === NOTIFY Listener Health === + let notify_listener_connected = IntGauge::with_opts(Opts::new( + "api_notify_listener_connected", + "NOTIFY listener connection status (0=disconnected, 1=connected)", + ))?; + registry.register(Box::new(notify_listener_connected.clone()))?; + + let notify_listener_reconnections = Counter::with_opts(Opts::new( + "api_notify_listener_reconnections", + "Number of NOTIFY listener reconnections", + ))?; + registry.register(Box::new(notify_listener_reconnections.clone()))?; + + let notify_listener_errors = Counter::with_opts(Opts::new( + "api_notify_listener_errors", + "NOTIFY listener errors", + ))?; + registry.register(Box::new(notify_listener_errors.clone()))?; + + // === Configuration Updates === + let config_updates_auth = Counter::with_opts(Opts::new( + "api_config_updates_auth", + "Auth configuration updates", + ))?; + registry.register(Box::new(config_updates_auth.clone()))?; + + let config_updates_routing = Counter::with_opts(Opts::new( + "api_config_updates_routing", + "Routing configuration updates", + ))?; + registry.register(Box::new(config_updates_routing.clone()))?; + + let config_updates_rate_limit = Counter::with_opts(Opts::new( + "api_config_updates_rate_limit", + "Rate limit configuration updates", + ))?; + registry.register(Box::new(config_updates_rate_limit.clone()))?; + + let config_updates_backend = Counter::with_opts(Opts::new( + "api_config_updates_backend", + "Backend endpoint configuration updates", + ))?; + registry.register(Box::new(config_updates_backend.clone()))?; + + // === Configuration Validation === + let config_validation_success = Counter::with_opts(Opts::new( + "api_config_validation_success", + "Configuration validation successes", + ))?; + registry.register(Box::new(config_validation_success.clone()))?; + + let config_validation_failure = Counter::with_opts(Opts::new( + "api_config_validation_failure", + "Configuration validation failures", + ))?; + registry.register(Box::new(config_validation_failure.clone()))?; + + Ok(Self { + notify_events_total, + notify_events_success, + notify_events_failure, + config_cache_hits, + config_cache_misses, + config_cache_size, + config_reload_duration_ms, + config_fetch_duration_ms, + notify_listener_connected, + notify_listener_reconnections, + notify_listener_errors, + config_updates_auth, + config_updates_routing, + config_updates_rate_limit, + config_updates_backend, + config_validation_success, + config_validation_failure, + }) + } + + /// Record NOTIFY event received + pub fn record_notify_event(&self, success: bool) { + self.notify_events_total.inc(); + if success { + self.notify_events_success.inc(); + } else { + self.notify_events_failure.inc(); + } + } + + /// Record configuration reload + pub fn record_config_reload(&self, config_type: &str, duration_ms: f64) { + self.config_reload_duration_ms.observe(duration_ms); + + match config_type { + "auth" => self.config_updates_auth.inc(), + "routing" => self.config_updates_routing.inc(), + "rate_limit" => self.config_updates_rate_limit.inc(), + "backend" => self.config_updates_backend.inc(), + _ => {}, + } + } + + /// Update NOTIFY listener connection status + pub fn update_listener_status(&self, connected: bool) { + self.notify_listener_connected + .set(if connected { 1 } else { 0 }); + } + + /// Record NOTIFY listener reconnection + pub fn record_reconnection(&self) { + self.notify_listener_reconnections.inc(); + } +} diff --git a/services/api/src/metrics/exporter.rs b/services/api/src/metrics/exporter.rs new file mode 100644 index 000000000..a4a179a0a --- /dev/null +++ b/services/api/src/metrics/exporter.rs @@ -0,0 +1,136 @@ +//! Prometheus Metrics Exporter +//! +//! Provides HTTP endpoint for Prometheus scraping at /metrics + +use prometheus::{Encoder, Registry, TextEncoder}; +use std::sync::Arc; +use tonic::{Response, Status}; + +/// Prometheus exporter for metrics collection +#[derive(Clone)] +pub struct PrometheusExporter { + registry: Arc, +} + +impl PrometheusExporter { + /// Create new exporter with registry + pub fn new(registry: Arc) -> Self { + Self { registry } + } + + /// Gather and encode metrics in Prometheus text format + pub fn gather_metrics(&self) -> Result { + let encoder = TextEncoder::new(); + let metric_families = self.registry.gather(); + let mut buffer = vec![]; + encoder.encode(&metric_families, &mut buffer)?; + String::from_utf8(buffer).map_err(|e| { + prometheus::Error::Msg(format!("Failed to convert metrics to UTF-8: {}", e)) + }) + } + + /// Export metrics as `HTTP` response (for Axum/Hyper integration) + pub fn export_http(&self) -> Result, prometheus::Error> { + let encoder = TextEncoder::new(); + let metric_families = self.registry.gather(); + let mut buffer = vec![]; + encoder.encode(&metric_families, &mut buffer)?; + Ok(buffer) + } +} + +/// Create Prometheus metrics endpoint (for standalone `HTTP` server) +/// +/// Usage with Axum: +/// ```rust,ignore +/// use axum::{Router, routing::get}; +/// use api::metrics::{GatewayMetrics, metrics_router}; +/// +/// let metrics = GatewayMetrics::new()?; +/// let router = metrics_router(metrics.registry()); +/// ``` +pub fn metrics_router(registry: Arc) -> axum::Router { + use axum::{routing::get, Router}; + + let exporter = PrometheusExporter::new(registry); + + Router::new().route( + "/metrics", + get(move || { + let exporter = exporter.clone(); + async move { + match exporter.gather_metrics() { + Ok(metrics) => Ok(metrics), + Err(e) => Err(format!("Failed to gather metrics: {}", e)), + } + } + }), + ) +} + +/// Create combined metrics and health router +/// +/// This combines Prometheus metrics with health/resilience endpoints +pub fn combined_router(registry: Arc) -> axum::Router { + use crate::health_router::{health_router, HealthState}; + use axum::Router; + + let metrics_routes = metrics_router(registry); + let health_state = HealthState::new(); + let health_routes = health_router(health_state); + + Router::new().merge(metrics_routes).merge(health_routes) +} + +/// Create Prometheus endpoint as gRPC health check extension +/// +/// This allows exposing metrics through the same gRPC server +pub async fn serve_metrics_grpc(registry: Arc) -> Result, Status> { + let exporter = PrometheusExporter::new(registry); + + exporter + .gather_metrics() + .map(Response::new) + .map_err(|e| Status::internal(format!("Failed to gather metrics: {}", e))) +} + +#[cfg(test)] +mod tests { + use super::*; + use prometheus::{Counter, Opts}; + + #[test] + fn test_prometheus_exporter() { + let registry = Arc::new(Registry::new()); + + // Register test counter + let counter = Counter::with_opts(Opts::new("test_counter", "Test counter")).unwrap(); + registry.register(Box::new(counter.clone())).unwrap(); + + counter.inc(); + counter.inc(); + + let exporter = PrometheusExporter::new(registry); + let metrics = exporter.gather_metrics().unwrap(); + + assert!(metrics.contains("test_counter")); + assert!(metrics.contains("# HELP test_counter Test counter")); + assert!(metrics.contains("test_counter 2")); + } + + #[test] + fn test_http_export() { + let registry = Arc::new(Registry::new()); + let counter = Counter::with_opts(Opts::new("http_test", "HTTP test")).unwrap(); + registry.register(Box::new(counter.clone())).unwrap(); + + counter.inc_by(42.0); + + let exporter = PrometheusExporter::new(registry); + let buffer = exporter.export_http().unwrap(); + let metrics = String::from_utf8(buffer).expect("INVARIANT: Valid UTF-8 bytes"); + + assert!(metrics.contains("http_test")); + assert!(metrics.contains("42")); + } +} diff --git a/services/api/src/metrics/mod.rs b/services/api/src/metrics/mod.rs new file mode 100644 index 000000000..3916293de --- /dev/null +++ b/services/api/src/metrics/mod.rs @@ -0,0 +1,74 @@ +//! Comprehensive Prometheus Metrics for API Gateway +//! +//! Provides detailed monitoring for: +//! - Authentication performance (6 layers) +//! - Backend proxy latencies +//! - Configuration hot-reload events +//! - Rate limiting effectiveness +//! - Circuit breaker states +//! +//! ## Metric Categories +//! +//! 1. **Auth Metrics**: JWT validation, revocation checks, RBAC, MFA +//! 2. **Proxy Metrics**: Backend latencies, connection pool, circuit breaker +//! 3. **Config Metrics**: NOTIFY events, cache hits/misses, hot-reload latency +//! 4. **Exporter**: Prometheus /metrics endpoint + +pub mod auth_metrics; +pub mod config_metrics; +pub mod exporter; +pub mod proxy_metrics; + +// Re-export core types +pub use auth_metrics::AuthMetrics; +pub use config_metrics::ConfigMetrics; +pub use exporter::{combined_router, metrics_router, PrometheusExporter}; +pub use proxy_metrics::ProxyMetrics; + +use prometheus::Registry; +use std::sync::Arc; + +/// Central metrics registry for API Gateway +#[derive(Clone)] +pub struct GatewayMetrics { + pub auth: Arc, + pub proxy: Arc, + pub config: Arc, + pub registry: Arc, +} + +impl GatewayMetrics { + /// Create new metrics registry with all subsystems + pub fn new() -> Result { + let registry = Arc::new(Registry::new()); + + let auth = Arc::new(AuthMetrics::new(®istry)?); + let proxy = Arc::new(ProxyMetrics::new(®istry)?); + let config = Arc::new(ConfigMetrics::new(®istry)?); + + Ok(Self { + auth, + proxy, + config, + registry, + }) + } + + /// Get reference to Prometheus registry for exporter + pub fn registry(&self) -> Arc { + self.registry.clone() + } +} + +impl Default for GatewayMetrics { + fn default() -> Self { + match Self::new() { + Ok(metrics) => metrics, + Err(_) => { + // Metrics initialization should not fail in practice; + // abort rather than panicking to satisfy deny(expect_used). + std::process::abort(); + } + } + } +} diff --git a/services/api/src/metrics/proxy_metrics.rs b/services/api/src/metrics/proxy_metrics.rs new file mode 100644 index 000000000..3988cd345 --- /dev/null +++ b/services/api/src/metrics/proxy_metrics.rs @@ -0,0 +1,346 @@ +//! Backend Proxy Metrics +//! +//! Tracks performance and health for backend gRPC services: +//! - Trading Service +//! - Backtesting Service +//! - ML Training Service + +use prometheus::{CounterVec, Histogram, HistogramOpts, HistogramVec, IntGaugeVec, Opts, Registry}; + +/// Backend proxy and routing metrics +pub struct ProxyMetrics { + // === Backend Request Metrics === + /// Total requests to each backend service + pub backend_requests_total: CounterVec, + /// Successful backend requests + pub backend_requests_success: CounterVec, + /// Failed backend requests + pub backend_requests_failure: CounterVec, + + // === Backend Latencies (milliseconds) === + /// Request latency to backend services + pub backend_request_duration_ms: HistogramVec, + /// gRPC connection establishment time + pub backend_connection_duration_ms: HistogramVec, + + // === Circuit Breaker State === + /// Circuit breaker state per service (0=closed, 1=half-open, 2=open) + pub circuit_breaker_state: IntGaugeVec, + /// Circuit breaker trips (transitions to open) + pub circuit_breaker_trips: CounterVec, + /// Circuit breaker recoveries (transitions to closed) + pub circuit_breaker_recoveries: CounterVec, + + // === Connection Pool === + /// Active connections per backend + pub connection_pool_active: IntGaugeVec, + /// Idle connections per backend + pub connection_pool_idle: IntGaugeVec, + /// Connection pool max size + pub connection_pool_max: IntGaugeVec, + /// Connection pool wait time (when exhausted) + pub connection_pool_wait_duration_ms: HistogramVec, + + // === Health Checks === + /// Health check successes per service + pub health_check_success: CounterVec, + /// Health check failures per service + pub health_check_failure: CounterVec, + /// Health check latency + pub health_check_duration_ms: HistogramVec, + /// Current health status (0=unhealthy, 1=healthy) + pub health_status: IntGaugeVec, + + // === gRPC Error Breakdown === + /// gRPC errors by status code and service + pub grpc_errors: CounterVec, + /// gRPC timeouts per service + pub grpc_timeouts: CounterVec, + + // === Request Routing === + /// Requests routed to each method + pub requests_by_method: CounterVec, + /// Routing latency (method resolution) + pub routing_duration_us: Histogram, +} + +impl ProxyMetrics { + /// Create new proxy metrics and register with Prometheus + pub fn new(registry: &Registry) -> Result { + // === Backend Request Metrics === + let backend_requests_total = CounterVec::new( + Opts::new( + "api_backend_requests_total", + "Total requests to backend services", + ), + &["service"], + )?; + registry.register(Box::new(backend_requests_total.clone()))?; + + let backend_requests_success = CounterVec::new( + Opts::new( + "api_backend_requests_success", + "Successful backend requests", + ), + &["service"], + )?; + registry.register(Box::new(backend_requests_success.clone()))?; + + let backend_requests_failure = CounterVec::new( + Opts::new( + "api_backend_requests_failure", + "Failed backend requests", + ), + &["service", "error_type"], + )?; + registry.register(Box::new(backend_requests_failure.clone()))?; + + // === Backend Latencies (milliseconds) === + let backend_request_duration_ms = HistogramVec::new( + HistogramOpts::new( + "api_backend_request_duration_milliseconds", + "Backend request latency in milliseconds", + ) + .buckets(vec![ + 1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, + ]), + &["service", "method"], + )?; + registry.register(Box::new(backend_request_duration_ms.clone()))?; + + let backend_connection_duration_ms = HistogramVec::new( + HistogramOpts::new( + "api_backend_connection_duration_milliseconds", + "gRPC connection establishment time in milliseconds", + ) + .buckets(vec![1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0]), + &["service"], + )?; + registry.register(Box::new(backend_connection_duration_ms.clone()))?; + + // === Circuit Breaker State === + let circuit_breaker_state = IntGaugeVec::new( + Opts::new( + "api_circuit_breaker_state", + "Circuit breaker state (0=closed, 1=half-open, 2=open)", + ), + &["service"], + )?; + registry.register(Box::new(circuit_breaker_state.clone()))?; + + let circuit_breaker_trips = CounterVec::new( + Opts::new( + "api_circuit_breaker_trips", + "Circuit breaker trips to open state", + ), + &["service"], + )?; + registry.register(Box::new(circuit_breaker_trips.clone()))?; + + let circuit_breaker_recoveries = CounterVec::new( + Opts::new( + "api_circuit_breaker_recoveries", + "Circuit breaker recoveries to closed state", + ), + &["service"], + )?; + registry.register(Box::new(circuit_breaker_recoveries.clone()))?; + + // === Connection Pool === + let connection_pool_active = IntGaugeVec::new( + Opts::new( + "api_connection_pool_active", + "Active connections per backend", + ), + &["service"], + )?; + registry.register(Box::new(connection_pool_active.clone()))?; + + let connection_pool_idle = IntGaugeVec::new( + Opts::new( + "api_connection_pool_idle", + "Idle connections per backend", + ), + &["service"], + )?; + registry.register(Box::new(connection_pool_idle.clone()))?; + + let connection_pool_max = IntGaugeVec::new( + Opts::new( + "api_connection_pool_max", + "Connection pool max size per backend", + ), + &["service"], + )?; + registry.register(Box::new(connection_pool_max.clone()))?; + + let connection_pool_wait_duration_ms = HistogramVec::new( + HistogramOpts::new( + "api_connection_pool_wait_duration_milliseconds", + "Connection pool wait time when exhausted", + ) + .buckets(vec![1.0, 5.0, 10.0, 25.0, 50.0, 100.0]), + &["service"], + )?; + registry.register(Box::new(connection_pool_wait_duration_ms.clone()))?; + + // === Health Checks === + let health_check_success = CounterVec::new( + Opts::new( + "api_health_check_success", + "Health check successes per service", + ), + &["service"], + )?; + registry.register(Box::new(health_check_success.clone()))?; + + let health_check_failure = CounterVec::new( + Opts::new( + "api_health_check_failure", + "Health check failures per service", + ), + &["service"], + )?; + registry.register(Box::new(health_check_failure.clone()))?; + + let health_check_duration_ms = HistogramVec::new( + HistogramOpts::new( + "api_health_check_duration_milliseconds", + "Health check latency in milliseconds", + ) + .buckets(vec![10.0, 50.0, 100.0, 250.0, 500.0, 1000.0]), + &["service"], + )?; + registry.register(Box::new(health_check_duration_ms.clone()))?; + + let health_status = IntGaugeVec::new( + Opts::new( + "api_health_status", + "Current health status (0=unhealthy, 1=healthy)", + ), + &["service"], + )?; + registry.register(Box::new(health_status.clone()))?; + + // === gRPC Error Breakdown === + let grpc_errors = CounterVec::new( + Opts::new("api_grpc_errors", "gRPC errors by status code"), + &["service", "status_code"], + )?; + registry.register(Box::new(grpc_errors.clone()))?; + + let grpc_timeouts = CounterVec::new( + Opts::new("api_grpc_timeouts", "gRPC timeouts per service"), + &["service"], + )?; + registry.register(Box::new(grpc_timeouts.clone()))?; + + // === Request Routing === + let requests_by_method = CounterVec::new( + Opts::new("api_requests_by_method", "Requests by gRPC method"), + &["service", "method"], + )?; + registry.register(Box::new(requests_by_method.clone()))?; + + let routing_duration_us = Histogram::with_opts( + HistogramOpts::new( + "api_routing_duration_microseconds", + "Routing decision latency in microseconds", + ) + .buckets(vec![0.1, 0.5, 1.0, 2.0, 5.0, 10.0]), + )?; + registry.register(Box::new(routing_duration_us.clone()))?; + + Ok(Self { + backend_requests_total, + backend_requests_success, + backend_requests_failure, + backend_request_duration_ms, + backend_connection_duration_ms, + circuit_breaker_state, + circuit_breaker_trips, + circuit_breaker_recoveries, + connection_pool_active, + connection_pool_idle, + connection_pool_max, + connection_pool_wait_duration_ms, + health_check_success, + health_check_failure, + health_check_duration_ms, + health_status, + grpc_errors, + grpc_timeouts, + requests_by_method, + routing_duration_us, + }) + } + + /// Record successful backend request + pub fn record_backend_success(&self, service: &str, method: &str, duration_ms: f64) { + self.backend_requests_total + .with_label_values(&[service]) + .inc(); + self.backend_requests_success + .with_label_values(&[service]) + .inc(); + self.backend_request_duration_ms + .with_label_values(&[service, method]) + .observe(duration_ms); + self.requests_by_method + .with_label_values(&[service, method]) + .inc(); + } + + /// Record failed backend request + pub fn record_backend_failure(&self, service: &str, error_type: &str) { + self.backend_requests_total + .with_label_values(&[service]) + .inc(); + self.backend_requests_failure + .with_label_values(&[service, error_type]) + .inc(); + } + + /// Update circuit breaker state (0=closed, 1=half-open, 2=open) + pub fn update_circuit_breaker_state(&self, service: &str, state: i64) { + self.circuit_breaker_state + .with_label_values(&[service]) + .set(state); + } + + /// Record circuit breaker trip + pub fn record_circuit_breaker_trip(&self, service: &str) { + self.circuit_breaker_trips + .with_label_values(&[service]) + .inc(); + self.update_circuit_breaker_state(service, 2); // 2 = open + } + + /// Record circuit breaker recovery + pub fn record_circuit_breaker_recovery(&self, service: &str) { + self.circuit_breaker_recoveries + .with_label_values(&[service]) + .inc(); + self.update_circuit_breaker_state(service, 0); // 0 = closed + } + + /// Update connection pool stats + pub fn update_connection_pool(&self, service: &str, active: i64, idle: i64, max: i64) { + self.connection_pool_active + .with_label_values(&[service]) + .set(active); + self.connection_pool_idle + .with_label_values(&[service]) + .set(idle); + self.connection_pool_max + .with_label_values(&[service]) + .set(max); + } + + /// Update health status (0=unhealthy, 1=healthy) + pub fn update_health_status(&self, service: &str, healthy: bool) { + self.health_status + .with_label_values(&[service]) + .set(if healthy { 1 } else { 0 }); + } +} diff --git a/services/api/src/routing/mod.rs b/services/api/src/routing/mod.rs new file mode 100644 index 000000000..787c5ccc5 --- /dev/null +++ b/services/api/src/routing/mod.rs @@ -0,0 +1,12 @@ +//! gRPC routing module for API Gateway +//! +//! Provides: +//! - Request routing to backend services +//! - Service discovery +//! - Load balancing +//! - Circuit breaking +//! - Token bucket rate limiting + +pub mod rate_limiter; + +pub use rate_limiter::{CacheStats, RateLimitConfig, RateLimiter}; diff --git a/services/api/src/routing/rate_limiter.rs b/services/api/src/routing/rate_limiter.rs new file mode 100644 index 000000000..f2f41eec3 --- /dev/null +++ b/services/api/src/routing/rate_limiter.rs @@ -0,0 +1,446 @@ +//! High-Performance Token Bucket Rate Limiter with Redis Backend +//! +//! Provides: +//! - Token bucket algorithm for smooth rate limiting +//! - Redis persistence for distributed rate limiting +//! - In-memory LRU cache for <8ns cache hits (DashMap lock-free) +//! - Per-endpoint rate limit configurations +//! - Atomic Lua script execution for consistency +//! +//! Performance targets: +//! - Cache hit: <8ns (DashMap lock-free lookup - 6x improvement) +//! - Redis hit: <500μs (local Redis, Lua script) +//! - Cache size: 10,000 entries with LRU eviction + +use anyhow::{Context, Result}; +use dashmap::DashMap; +use redis::aio::ConnectionManager; +use std::sync::Arc; +use tokio::time::{Duration, Instant}; +use tracing::debug; +use uuid::Uuid; + +/// Token bucket for rate limiting +#[derive(Debug, Clone)] +struct TokenBucket { + /// Current number of tokens available + tokens: f64, + /// Last time tokens were refilled + last_refill: Instant, + /// Maximum capacity of the bucket + capacity: f64, + /// Tokens added per second + refill_rate: f64, + /// Last access time (for LRU eviction) + last_access: Instant, +} + +impl TokenBucket { + /// Create new token bucket with full capacity + fn new(capacity: f64, refill_rate: f64) -> Self { + let now = Instant::now(); + Self { + tokens: capacity, + last_refill: now, + capacity, + refill_rate, + last_access: now, + } + } + + /// Refill tokens based on elapsed time + fn refill(&mut self) { + let now = Instant::now(); + let elapsed = now.duration_since(self.last_refill).as_secs_f64(); + + // Calculate new token count (f64 operations are safe for this use case) + let refilled = elapsed * self.refill_rate; + let new_tokens = self.tokens + refilled; + // Check for overflow/infinity and cap at capacity + self.tokens = if new_tokens.is_finite() && new_tokens <= self.capacity { + new_tokens + } else { + self.capacity + }; + self.last_refill = now; + self.last_access = now; + } + + /// Consume one token if available + fn consume(&mut self) -> bool { + self.refill(); + + if self.tokens >= 1.0 { + self.tokens -= 1.0; // f64 subtraction is safe for small values + true + } else { + false + } + } +} + +/// Rate limit configuration for a specific endpoint +#[derive(Debug, Clone)] +pub struct RateLimitConfig { + /// Endpoint pattern (e.g., "trading.submit_order") + pub endpoint: String, + /// Maximum tokens in bucket + pub capacity: f64, + /// Tokens added per second + pub refill_rate: f64, + /// Maximum concurrent requests (burst size) + pub burst_size: usize, +} + +impl RateLimitConfig { + /// Create default config for trading endpoints + pub fn trading_submit_order() -> Self { + Self { + endpoint: "trading.submit_order".to_string(), + capacity: 100.0, + refill_rate: 100.0, // 100 requests/second + burst_size: 10_usize, + } + } + + /// Create default config for configuration updates + pub fn config_update() -> Self { + Self { + endpoint: "config.update".to_string(), + capacity: 10.0, + refill_rate: 10.0, // 10 requests/second + burst_size: 2_usize, + } + } + + /// Create default config for backtesting + pub fn backtesting_run() -> Self { + Self { + endpoint: "backtesting.run".to_string(), + capacity: 5.0, + refill_rate: 5.0 / 60.0, // 5 requests/minute (f64 division is safe) + burst_size: 1_usize, + } + } + + /// Get default configuration for an endpoint + pub fn default_for_endpoint(endpoint: &str) -> Self { + match endpoint { + "trading.submit_order" => Self::trading_submit_order(), + "config.update" => Self::config_update(), + "backtesting.run" => Self::backtesting_run(), + _ => Self { + endpoint: endpoint.to_string(), + capacity: 50.0, + refill_rate: 50.0, // Default: 50 requests/second + burst_size: 5_usize, + }, + } + } +} + +/// LRU cache entry with access tracking +struct CacheEntry { + bucket: TokenBucket, + last_access: Instant, +} + +/// High-performance rate limiter with Redis backend and in-memory cache +#[derive(Clone)] +pub struct RateLimiter { + /// Redis connection for persistence + redis: Arc, + /// In-memory LRU cache for fast lookups (TARGET: <8ns with DashMap) + local_cache: Arc>, + /// Maximum cache size (10,000 entries) + max_cache_size: usize, + /// Cache TTL (1 second) + cache_ttl: Duration, + /// Per-endpoint configurations (lock-free concurrent access) + endpoint_configs: Arc>, +} + +impl RateLimiter { + /// Create new rate limiter with Redis backend + pub async fn new(redis_url: &str) -> Result { + let client = redis::Client::open(redis_url) + .context("Failed to create Redis client for rate limiter")?; + + let redis = ConnectionManager::new(client) + .await + .context("Failed to connect to Redis for rate limiter")?; + + let endpoint_configs = DashMap::new(); + + // Load default configurations + let default_configs = vec![ + RateLimitConfig::trading_submit_order(), + RateLimitConfig::config_update(), + RateLimitConfig::backtesting_run(), + ]; + + for config in default_configs { + endpoint_configs.insert(config.endpoint.clone(), config); + } + + Ok(Self { + redis: Arc::new(redis), + local_cache: Arc::new(DashMap::new()), + max_cache_size: 10_000, + cache_ttl: Duration::from_secs(1), + endpoint_configs: Arc::new(endpoint_configs), + }) + } + + /// Check rate limit for a user and endpoint + /// + /// Performance: + /// - Cache hit: <8ns (DashMap lock-free lookup - 6x improvement) + /// + /// - Cache miss: <500μs (Redis Lua script) + pub async fn check_limit(&self, user_id: &Uuid, endpoint: &str) -> Result { + let key = format!("ratelimit:{}:{}", user_id, endpoint); + + // 1. Check local cache first (TARGET: <8ns with DashMap) + if let Some(mut entry) = self.local_cache.get_mut(&key) { + // Check if cache entry is still valid + if entry.last_access.elapsed() < self.cache_ttl { + debug!("Rate limit cache hit for {}", key); + let allowed = entry.bucket.consume(); + entry.last_access = Instant::now(); + return Ok(allowed); + } else { + // Cache expired, drop the reference before removing + drop(entry); + self.local_cache.remove(&key); + } + } + + // 2. Cache miss - check Redis (fallback) + debug!("Rate limit cache miss for {}", key); + let allowed = self.check_redis_limit(&key, endpoint).await?; + + // 3. Update local cache + self.update_local_cache(&key, endpoint, allowed).await; + + Ok(allowed) + } + + /// Check rate limit against Redis using Lua script for atomic operations + async fn check_redis_limit(&self, key: &str, endpoint: &str) -> Result { + // Get endpoint configuration (lock-free DashMap read) + let config = self + .endpoint_configs + .get(endpoint) + .map(|entry| entry.value().clone()) + .unwrap_or_else(|| RateLimitConfig::default_for_endpoint(endpoint)); + + // Token bucket algorithm using Redis Lua script + let script = r#" + local key = KEYS[1] + local capacity = tonumber(ARGV[1]) + local refill_rate = tonumber(ARGV[2]) + local now = tonumber(ARGV[3]) + + -- Get current bucket state + local bucket = redis.call('HGETALL', key) + local tokens = capacity + local last_refill = now + + -- Parse existing state + if #bucket > 0 then + for i = 1, #bucket, 2 do + if bucket[i] == 'tokens' then + tokens = tonumber(bucket[i + 1]) + elseif bucket[i] == 'last_refill' then + last_refill = tonumber(bucket[i + 1]) + end + end + end + + -- Refill tokens based on elapsed time + local elapsed = now - last_refill + local new_tokens = tokens + (elapsed * refill_rate) + if new_tokens < 0 then + new_tokens = capacity + end + tokens = math.min(capacity, new_tokens) + + -- Check if request is allowed + if tokens >= 1 then + tokens = tokens - 1 + redis.call('HSET', key, 'tokens', tokens, 'last_refill', now) + redis.call('EXPIRE', key, 300) -- 5 minute TTL + return 1 + else + -- Update state even on failure to maintain accurate refill time + redis.call('HSET', key, 'tokens', tokens, 'last_refill', now) + redis.call('EXPIRE', key, 300) + return 0 + end + "#; + + // Get current timestamp in milliseconds + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .context("System time error")? + .as_millis() as f64 + / 1000.0; + + // Execute Lua script + let result: i32 = redis::Script::new(script) + .key(key) + .arg(config.capacity) + .arg(config.refill_rate) + .arg(now) + .invoke_async(&mut (*self.redis).clone()) + .await + .context("Failed to execute rate limit check in Redis")?; + + Ok(result == 1) + } + + /// Update local cache after Redis check + async fn update_local_cache(&self, key: &str, endpoint: &str, allowed: bool) { + // Evict oldest entries if cache is full (lock-free size check) + if self.local_cache.len() >= self.max_cache_size { + self.evict_lru_entries().await; + } + + // Get endpoint configuration (lock-free DashMap read) + let config = self + .endpoint_configs + .get(endpoint) + .map(|entry| entry.value().clone()) + .unwrap_or_else(|| RateLimitConfig::default_for_endpoint(endpoint)); + + // Create new bucket based on Redis response + let mut bucket = TokenBucket::new(config.capacity, config.refill_rate); + + // If request was allowed, we just consumed a token + if allowed { + bucket.tokens = (config.capacity - 1.0).max(0.0); // f64 subtraction, ensure non-negative + } else { + bucket.tokens = 0.0; // Bucket is empty + } + + let entry = CacheEntry { + bucket, + last_access: Instant::now(), + }; + + self.local_cache.insert(key.to_string(), entry); + } + + /// Evict least recently used entries from cache + async fn evict_lru_entries(&self) { + // Remove 10% of entries (1,000 entries) to make room + let num_to_evict = self + .max_cache_size + .checked_div(10) + .unwrap_or(1); + + // Collect entries with their last access time (lock-free iteration) + let mut entries: Vec<_> = self + .local_cache + .iter() + .map(|entry| (entry.key().clone(), entry.value().last_access)) + .collect(); + + // Sort by last access time (oldest first) + entries.sort_by_key(|(_, last_access)| *last_access); + + // Remove oldest entries (lock-free removal) + for (key, _) in entries.iter().take(num_to_evict) { + self.local_cache.remove(key); + } + + debug!("Evicted {} LRU entries from rate limit cache", num_to_evict); + } + + /// Add or update endpoint configuration (lock-free insertion) + pub async fn set_endpoint_config(&self, config: RateLimitConfig) { + self.endpoint_configs + .insert(config.endpoint.clone(), config); + } + + /// Get current cache statistics (lock-free reads) + pub async fn get_cache_stats(&self) -> CacheStats { + CacheStats { + size: self.local_cache.len(), + max_size: self.max_cache_size, + ttl_seconds: self.cache_ttl.as_secs(), + } + } + + /// Clear local cache (useful for testing or after configuration changes) + pub async fn clear_cache(&self) { + self.local_cache.clear(); + debug!("Rate limit cache cleared"); + } +} + +/// Cache statistics for monitoring +#[derive(Debug, Clone)] +pub struct CacheStats { + pub size: usize, + pub max_size: usize, + pub ttl_seconds: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_token_bucket_basic() { + let mut bucket = TokenBucket::new(10.0, 10.0); + + // Should have full capacity (tokens >= 1.0) + assert!(bucket.tokens >= 1.0); + + // Consume 10 tokens + for _ in 0..10 { + assert!(bucket.consume()); + } + + // Should be empty now + assert!(!bucket.consume()); + } + + #[test] + fn test_token_bucket_refill() { + let mut bucket = TokenBucket::new(10.0, 10.0); + + // Consume all tokens + for _ in 0..10 { + assert!(bucket.consume()); + } + + // Wait for refill (in real scenario) + // In test, we manually advance time by calling refill + std::thread::sleep(Duration::from_millis(500)); + bucket.refill(); + + // Should have refilled ~5 tokens (500ms * 10 tokens/sec) + assert!(bucket.tokens >= 4.0 && bucket.tokens <= 6.0); + } + + #[test] + fn test_rate_limit_configs() { + let trading_config = RateLimitConfig::trading_submit_order(); + assert_eq!(trading_config.capacity, 100.0); + assert_eq!(trading_config.refill_rate, 100.0); + assert_eq!(trading_config.burst_size, 10); + + let config_update = RateLimitConfig::config_update(); + assert_eq!(config_update.capacity, 10.0); + assert_eq!(config_update.refill_rate, 10.0); + assert_eq!(config_update.burst_size, 2); + + let backtesting = RateLimitConfig::backtesting_run(); + assert_eq!(backtesting.capacity, 5.0); + assert_eq!(backtesting.refill_rate, 5.0 / 60.0); + assert_eq!(backtesting.burst_size, 1); + } +} diff --git a/services/api/tests/auth_edge_cases.rs b/services/api/tests/auth_edge_cases.rs new file mode 100644 index 000000000..fb0548abf --- /dev/null +++ b/services/api/tests/auth_edge_cases.rs @@ -0,0 +1,1289 @@ +//! Comprehensive Authentication Edge Case Tests +//! +//! This test suite focuses on edge cases and security scenarios for the +//! API Gateway authentication system. Coverage areas: +//! +//! 1. JWT Token Edge Cases: +//! - Expired tokens +//! - Malformed tokens +//! - Invalid signatures +//! - Missing required claims +//! - Token format validation +//! - Token refresh scenarios +//! +//! 2. Session Management: +//! - Session timeout +//! - Concurrent sessions +//! - Session invalidation +//! - Session renewal +//! +//! 3. Rate Limiting Edge Cases: +//! - Per-user limits +//! - Per-IP limits +//! - Burst handling +//! - Rate limit reset +//! - Concurrent rate limit checks +//! +//! 4. Request Routing: +//! - Backend service failures +//! - Circuit breaker activation +//! - Timeout handling +//! - Service discovery +//! - Load balancing + +#[path = "common/mod.rs"] +mod common; + +use anyhow::Result; +use common::{ + cleanup_redis, generate_expired_token, generate_invalid_signature_token, generate_test_token, + wait_for_redis, TestJwtConfig, +}; +use jsonwebtoken::{encode, EncodingKey, Header}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use tonic::{metadata::MetadataValue, Request}; +use uuid::Uuid; + +use api::auth::{ + AuditLogger, AuthInterceptor, AuthzService, Jti, JwtClaims, JwtService, RateLimiter, + RevocationService, +}; + +const REDIS_URL: &str = "redis://localhost:6379"; + +/// Setup test authentication components +async fn setup_auth_components() -> Result { + wait_for_redis(REDIS_URL, 50).await?; + cleanup_redis(REDIS_URL).await?; + + let config = TestJwtConfig::default(); + + let jwt_service = JwtService::new(config.secret, config.issuer, config.audience); + + let revocation_service = RevocationService::new(REDIS_URL).await?; + let authz_service = AuthzService::new(); + let rate_limiter = RateLimiter::new(100).map_err(|e| anyhow::anyhow!(e))?; + let audit_logger = AuditLogger::new(true); + + Ok(AuthInterceptor::new( + jwt_service, + revocation_service, + authz_service, + rate_limiter, + audit_logger, + )) +} + +// ============================================================================ +// JWT Token Edge Cases +// ============================================================================ + +#[tokio::test] +async fn test_expired_jwt_rejected() -> Result<()> { + println!("\n=== Test: Expired JWT Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate expired token + let token = generate_expired_token("user_expired")?; + + // Create request with expired token + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "Expired JWT should be rejected"); + + if let Err(status) = result { + assert_eq!(status.code(), tonic::Code::Unauthenticated); + println!("✓ Expired token rejected: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_invalid_signature_jwt_rejected() -> Result<()> { + println!("\n=== Test: Invalid Signature JWT Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate token with invalid signature + let token = generate_invalid_signature_token("user_invalid_sig")?; + + // Create request + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!( + result.is_err(), + "JWT with invalid signature should be rejected" + ); + + if let Err(status) = result { + assert_eq!(status.code(), tonic::Code::Unauthenticated); + println!("✓ Invalid signature rejected: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_malformed_jwt_rejected() -> Result<()> { + println!("\n=== Test: Malformed JWT Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Create request with malformed token + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from("Bearer not.a.valid.jwt.token.format")?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "Malformed JWT should be rejected"); + + if let Err(status) = result { + assert_eq!(status.code(), tonic::Code::Unauthenticated); + println!("✓ Malformed token rejected: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_missing_jti_claim_rejected() -> Result<()> { + println!("\n=== Test: JWT Missing JTI Claim Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + let config = TestJwtConfig::default(); + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + // Create claims without JTI (empty string) + let claims = JwtClaims { + jti: "".to_string(), // Invalid: empty JTI + sub: "user_no_jti".to_string(), + iat: now, + exp: now + 3600, + nbf: Some(now), + iss: config.issuer.clone(), + aud: config.audience.clone(), + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(config.secret.as_bytes()), + )?; + + // Create request + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "JWT without JTI should be rejected"); + + if let Err(status) = result { + println!("✓ Token without JTI rejected: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_wrong_issuer_rejected() -> Result<()> { + println!("\n=== Test: JWT with Wrong Issuer Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + let config = TestJwtConfig::default(); + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + // Create claims with wrong issuer + let claims = JwtClaims { + jti: Jti::new().0, + sub: "user_wrong_issuer".to_string(), + iat: now, + exp: now + 3600, + nbf: Some(now), + iss: "wrong-issuer".to_string(), // Wrong issuer + aud: config.audience.clone(), + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(config.secret.as_bytes()), + )?; + + // Create request + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "JWT with wrong issuer should be rejected"); + + if let Err(status) = result { + println!("✓ Wrong issuer rejected: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_wrong_audience_rejected() -> Result<()> { + println!("\n=== Test: JWT with Wrong Audience Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + let config = TestJwtConfig::default(); + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + // Create claims with wrong audience + let claims = JwtClaims { + jti: Jti::new().0, + sub: "user_wrong_aud".to_string(), + iat: now, + exp: now + 3600, + nbf: Some(now), + iss: config.issuer.clone(), + aud: "wrong-audience".to_string(), // Wrong audience + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(config.secret.as_bytes()), + )?; + + // Create request + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!( + result.is_err(), + "JWT with wrong audience should be rejected" + ); + + if let Err(status) = result { + println!("✓ Wrong audience rejected: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_token_not_yet_valid_rejected() -> Result<()> { + println!("\n=== Test: JWT Not Yet Valid (nbf) Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + let config = TestJwtConfig::default(); + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + // Create claims with future nbf (not before) + let claims = JwtClaims { + jti: Jti::new().0, + sub: "user_future_nbf".to_string(), + iat: now, + exp: now + 7200, + nbf: Some(now + 3600), // Valid only in 1 hour + iss: config.issuer.clone(), + aud: config.audience.clone(), + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(config.secret.as_bytes()), + )?; + + // Create request + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "JWT not yet valid should be rejected"); + + if let Err(status) = result { + println!("✓ Not-yet-valid token rejected: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_missing_authorization_header() -> Result<()> { + println!("\n=== Test: Missing Authorization Header ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Create request without Authorization header + let request = Request::new(()); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!( + result.is_err(), + "Request without auth header should be rejected" + ); + + if let Err(status) = result { + assert_eq!(status.code(), tonic::Code::Unauthenticated); + println!("✓ Missing auth header rejected: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_invalid_authorization_format() -> Result<()> { + println!("\n=== Test: Invalid Authorization Format ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Create request with invalid format (missing "Bearer ") + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from("InvalidFormat token")?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "Invalid auth format should be rejected"); + + if let Err(status) = result { + println!("✓ Invalid format rejected: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_empty_bearer_token() -> Result<()> { + println!("\n=== Test: Empty Bearer Token ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Create request with empty token + let mut request = Request::new(()); + request + .metadata_mut() + .insert("authorization", MetadataValue::try_from("Bearer ")?); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "Empty bearer token should be rejected"); + + if let Err(status) = result { + println!("✓ Empty token rejected: {}", status.message()); + } + + Ok(()) +} + +// ============================================================================ +// Session Management Edge Cases +// ============================================================================ + +#[tokio::test] +async fn test_concurrent_sessions_same_user() -> Result<()> { + println!("\n=== Test: Concurrent Sessions for Same User ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate two tokens for same user with different session IDs + let (token1, _jti1) = generate_test_token( + "user_concurrent", + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + )?; + + let (token2, _jti2) = generate_test_token( + "user_concurrent", + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + )?; + + // First session request + let mut request1 = Request::new(()); + request1.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token1))?, + ); + + // Second session request + let mut request2 = Request::new(()); + request2.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token2))?, + ); + + // Both sessions should be valid + let result1 = auth_interceptor.clone().authenticate(request1).await; + let result2 = auth_interceptor.clone().authenticate(request2).await; + + assert!(result1.is_ok(), "First concurrent session should succeed"); + assert!(result2.is_ok(), "Second concurrent session should succeed"); + + println!("✓ Both concurrent sessions validated successfully"); + + Ok(()) +} + +#[tokio::test] +async fn test_session_invalidation_revokes_token() -> Result<()> { + println!("\n=== Test: Session Invalidation Revokes Token ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate token + let (token, jti) = generate_test_token( + "user_session_invalidate", + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + )?; + + // First request should succeed + let mut request1 = Request::new(()); + request1.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result1 = auth_interceptor.clone().authenticate(request1).await; + assert!(result1.is_ok(), "First request should succeed"); + println!("✓ Token validated successfully"); + + // Revoke the token (simulate session invalidation) + let revocation_service = RevocationService::new(REDIS_URL).await?; + revocation_service + .revoke_token(&Jti::from_string(jti), 3600) + .await?; + println!("✓ Token revoked (session invalidated)"); + + // Second request should fail + let mut request2 = Request::new(()); + request2.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result2 = auth_interceptor.clone().authenticate(request2).await; + assert!(result2.is_err(), "Revoked token should be rejected"); + + if let Err(status) = result2 { + println!("✓ Revoked token rejected: {}", status.message()); + } + + Ok(()) +} + +// ============================================================================ +// Rate Limiting Edge Cases +// ============================================================================ + +#[tokio::test] +async fn test_rate_limit_per_user_independence() -> Result<()> { + println!("\n=== Test: Rate Limit Per-User Independence ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Create rate limiter with very low limit (5 req/s) for testing + let rate_limiter = RateLimiter::new(5).map_err(|e| anyhow::anyhow!(e))?; + + // User 1: Make requests until rate limited + let mut user1_allowed = 0; + for _ in 0..10 { + if rate_limiter.check_rate_limit("user_1") { + user1_allowed += 1; + } + } + + // User 2: Should have independent rate limit + let mut user2_allowed = 0; + for _ in 0..10 { + if rate_limiter.check_rate_limit("user_2") { + user2_allowed += 1; + } + } + + println!(" User 1 allowed: {} requests", user1_allowed); + println!(" User 2 allowed: {} requests", user2_allowed); + + // Both users should be able to make requests despite the other being limited + assert!(user1_allowed > 0, "User 1 should be allowed some requests"); + assert!(user2_allowed > 0, "User 2 should be allowed some requests"); + assert!(user1_allowed <= 5, "User 1 should be rate limited"); + assert!(user2_allowed <= 5, "User 2 should be rate limited"); + + println!("✓ Per-user rate limits are independent"); + + Ok(()) +} + +#[tokio::test] +async fn test_rate_limit_burst_handling() -> Result<()> { + println!("\n=== Test: Rate Limit Burst Handling ==="); + + let rate_limiter = RateLimiter::new(10).map_err(|e| anyhow::anyhow!(e))?; // 10 req/s + + // Make burst of 20 requests + let mut allowed = 0; + let mut denied = 0; + + for _ in 0..20 { + if rate_limiter.check_rate_limit("user_burst") { + allowed += 1; + } else { + denied += 1; + } + } + + println!(" Burst: {} allowed, {} denied", allowed, denied); + + assert!(allowed <= 10, "Burst should be limited to capacity"); + assert!(denied > 0, "Some requests should be denied"); + + println!("✓ Burst correctly limited"); + + Ok(()) +} + +#[tokio::test] +async fn test_rate_limit_reset_after_time() -> Result<()> { + println!("\n=== Test: Rate Limit Reset After Time ==="); + + let rate_limiter = RateLimiter::new(5).map_err(|e| anyhow::anyhow!(e))?; // 5 req/s + + // Exhaust rate limit + let mut first_batch = 0; + for _ in 0..10 { + if rate_limiter.check_rate_limit("user_reset") { + first_batch += 1; + } + } + + println!(" First batch: {} allowed", first_batch); + assert!(first_batch <= 5, "Should be rate limited"); + + // Wait for rate limit to reset (1 second) + println!(" Waiting 1 second for rate limit reset..."); + tokio::time::sleep(Duration::from_millis(1100)).await; + + // Try again - should allow more requests + let mut second_batch = 0; + for _ in 0..10 { + if rate_limiter.check_rate_limit("user_reset") { + second_batch += 1; + } + } + + println!(" Second batch: {} allowed", second_batch); + assert!(second_batch > 0, "Rate limit should have reset"); + + println!("✓ Rate limit reset successfully"); + + Ok(()) +} + +#[tokio::test] +async fn test_concurrent_rate_limit_checks() -> Result<()> { + println!("\n=== Test: Concurrent Rate Limit Checks ==="); + + let rate_limiter = RateLimiter::new(50).map_err(|e| anyhow::anyhow!(e))?; // 50 req/s + + // Spawn 100 concurrent tasks + let mut handles = vec![]; + + for _ in 0..100 { + let limiter = rate_limiter.clone(); + let handle = tokio::spawn(async move { limiter.check_rate_limit("user_concurrent") }); + handles.push(handle); + } + + // Collect results + let mut allowed = 0; + for handle in handles { + if let Ok(true) = handle.await { + allowed += 1; + } + } + + println!(" Concurrent: {} / 100 allowed", allowed); + + // Should be approximately limited to 50 + assert!(allowed >= 45, "Should allow close to rate limit"); + assert!(allowed <= 55, "Should not significantly exceed rate limit"); + + println!("✓ Concurrent rate limiting working correctly"); + + Ok(()) +} + +// ============================================================================ +// Authorization Edge Cases +// ============================================================================ + +#[tokio::test] +async fn test_insufficient_permissions_rejected() -> Result<()> { + println!("\n=== Test: Insufficient Permissions Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate token with minimal permissions + let (token, _jti) = generate_test_token( + "user_no_perms", + vec!["viewer".to_string()], // Only viewer role + vec!["api.access".to_string()], // Only basic access + 3600, + )?; + + // Create request + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + // Token should be valid but have limited permissions + let result = auth_interceptor.clone().authenticate(request).await; + + // Authentication succeeds, but user context reflects limited permissions + if let Ok(authenticated_request) = result { + let extensions = authenticated_request.extensions(); + if let Some(user_ctx) = extensions.get::() { + assert_eq!(user_ctx.roles, vec!["viewer".to_string()]); + assert!(!user_ctx.permissions.contains(&"trading.submit".to_string())); + println!( + "✓ User authenticated with limited permissions: {:?}", + user_ctx.permissions + ); + } + } else { + panic!("Authentication should succeed even with limited permissions"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_empty_roles_accepted() -> Result<()> { + println!("\n=== Test: Empty Roles Still Authenticated ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate token with no roles + let (token, _jti) = generate_test_token( + "user_no_roles", + vec![], // No roles + vec!["api.access".to_string()], + 3600, + )?; + + // Create request + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!( + result.is_ok(), + "User with no roles should still authenticate" + ); + + if let Ok(authenticated_request) = result { + let extensions = authenticated_request.extensions(); + if let Some(user_ctx) = extensions.get::() { + assert!(user_ctx.roles.is_empty()); + println!("✓ User authenticated with no roles"); + } + } + + Ok(()) +} + +// ============================================================================ +// Additional JWT Edge Cases (Wave 1 Agent 8) +// ============================================================================ + +#[tokio::test] +async fn test_token_with_future_iat_rejected() -> Result<()> { + println!("\n=== Test: JWT with Future IAT (Issued At) Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + let config = TestJwtConfig::default(); + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + // Create claims with future iat (issued 1 hour in the future) + let claims = JwtClaims { + jti: Jti::new().0, + sub: "user_future_iat".to_string(), + iat: now + 3600, // Issued 1 hour in the future + exp: now + 7200, // Valid for 2 hours + nbf: Some(now), + iss: config.issuer.clone(), + aud: config.audience.clone(), + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(config.secret.as_bytes()), + )?; + + // Create request + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "JWT with future iat should be rejected"); + + if let Err(status) = result { + println!("✓ Token with future iat rejected: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_token_too_old_rejected() -> Result<()> { + println!("\n=== Test: JWT Too Old (>1 hour) Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + let config = TestJwtConfig::default(); + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + // Create claims with old iat (issued 2 hours ago, max age is 1 hour) + let claims = JwtClaims { + jti: Jti::new().0, + sub: "user_old_token".to_string(), + iat: now - 7200, // Issued 2 hours ago + exp: now + 3600, // Still valid for 1 hour + nbf: Some(now - 7200), + iss: config.issuer.clone(), + aud: config.audience.clone(), + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(config.secret.as_bytes()), + )?; + + // Create request + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "JWT older than 1 hour should be rejected"); + + if let Err(status) = result { + println!("✓ Token too old rejected: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_token_too_long_rejected() -> Result<()> { + println!("\n=== Test: JWT Token Too Long (>8192 chars) Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Create an excessively long token (potential DoS attack) + let long_token = "a".repeat(8200); + + // Create request + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", long_token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "Excessively long JWT should be rejected"); + + if let Err(status) = result { + println!("✓ Token too long rejected: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_token_with_empty_subject_rejected() -> Result<()> { + println!("\n=== Test: JWT with Empty Subject Claim Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + let config = TestJwtConfig::default(); + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + // Create claims with empty subject + let claims = JwtClaims { + jti: Jti::new().0, + sub: "".to_string(), // Empty subject + iat: now, + exp: now + 3600, + nbf: Some(now), + iss: config.issuer.clone(), + aud: config.audience.clone(), + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(config.secret.as_bytes()), + )?; + + // Create request + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "JWT with empty subject should be rejected"); + + if let Err(status) = result { + println!("✓ Token with empty subject rejected: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_token_with_corrupted_payload() -> Result<()> { + println!("\n=== Test: JWT with Corrupted Payload ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Create a token with corrupted base64 payload + let corrupted_token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.CORRUPTED!!!.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"; + + // Create request + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", corrupted_token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!( + result.is_err(), + "JWT with corrupted payload should be rejected" + ); + + if let Err(status) = result { + println!("✓ Corrupted payload rejected: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_token_with_missing_permissions_claim() -> Result<()> { + println!("\n=== Test: JWT with Empty Permissions List ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate token with no permissions + let (token, _jti) = generate_test_token( + "user_no_perms", + vec!["trader".to_string()], + vec![], // Empty permissions + 3600, + )?; + + // Create request + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + // Should fail because api.access permission is required + assert!( + result.is_err(), + "JWT without api.access permission should be rejected" + ); + + if let Err(status) = result { + assert_eq!(status.code(), tonic::Code::PermissionDenied); + println!( + "✓ Token without required permissions rejected: {}", + status.message() + ); + } + + Ok(()) +} + +#[tokio::test] +async fn test_multiple_failed_authentication_attempts() -> Result<()> { + println!("\n=== Test: Multiple Failed Authentication Attempts ==="); + + let auth_interceptor = setup_auth_components().await?; + + let mut failed_attempts = 0; + + // Attempt authentication 10 times with invalid tokens + for i in 0..10 { + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer invalid_token_{}", i))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + if result.is_err() { + failed_attempts += 1; + } + } + + println!(" Failed attempts: {} / 10", failed_attempts); + assert_eq!( + failed_attempts, 10, + "All invalid token attempts should fail" + ); + + println!("✓ Multiple failed authentication attempts handled correctly"); + + Ok(()) +} + +#[tokio::test] +async fn test_token_refresh_scenario() -> Result<()> { + println!("\n=== Test: Token Refresh Scenario ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate initial token + let (token1, jti1) = generate_test_token( + "user_refresh", + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + )?; + + // First request with token1 + let mut request1 = Request::new(()); + request1.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token1))?, + ); + + let result1 = auth_interceptor.clone().authenticate(request1).await; + assert!(result1.is_ok(), "First token should be valid"); + println!("✓ Initial token validated"); + + // Revoke first token (simulate user requested refresh) + let revocation_service = RevocationService::new(REDIS_URL).await?; + revocation_service + .revoke_token(&Jti::from_string(jti1), 3600) + .await?; + println!("✓ Old token revoked"); + + // Generate new token for same user (refresh) + let (token2, _jti2) = generate_test_token( + "user_refresh", + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + )?; + + // Request with revoked token should fail + let mut request2 = Request::new(()); + request2.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token1))?, + ); + + let result2 = auth_interceptor.clone().authenticate(request2).await; + assert!(result2.is_err(), "Revoked token should fail"); + println!("✓ Revoked token rejected"); + + // Request with new token should succeed + let mut request3 = Request::new(()); + request3.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token2))?, + ); + + let result3 = auth_interceptor.clone().authenticate(request3).await; + assert!(result3.is_ok(), "New token should be valid"); + println!("✓ New token validated"); + + Ok(()) +} + +#[tokio::test] +async fn test_bearer_token_case_sensitivity() -> Result<()> { + println!("\n=== Test: Bearer Token Case Sensitivity ==="); + + let auth_interceptor = setup_auth_components().await?; + + let (token, _jti) = generate_test_token( + "user_case_test", + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + )?; + + // Test with lowercase "bearer" (should fail) + let mut request1 = Request::new(()); + request1.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("bearer {}", token))?, + ); + + let result1 = auth_interceptor.clone().authenticate(request1).await; + assert!(result1.is_err(), "Lowercase 'bearer' should be rejected"); + println!("✓ Lowercase 'bearer' rejected"); + + // Test with correct "Bearer" (should succeed) + let mut request2 = Request::new(()); + request2.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result2 = auth_interceptor.clone().authenticate(request2).await; + assert!(result2.is_ok(), "Correct 'Bearer' should succeed"); + println!("✓ Correct 'Bearer' accepted"); + + Ok(()) +} + +#[tokio::test] +async fn test_token_with_whitespace_padding() -> Result<()> { + println!("\n=== Test: Token with Whitespace Padding ==="); + + let auth_interceptor = setup_auth_components().await?; + + let (token, _jti) = generate_test_token( + "user_whitespace", + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + )?; + + // Test with trailing whitespace + let mut request1 = Request::new(()); + request1.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {} ", token))?, + ); + + let result1 = auth_interceptor.clone().authenticate(request1).await; + // This might fail due to whitespace in token + println!( + " Trailing whitespace result: {}", + if result1.is_ok() { "OK" } else { "FAIL" } + ); + + // Test with leading whitespace after Bearer + let mut request2 = Request::new(()); + request2.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result2 = auth_interceptor.clone().authenticate(request2).await; + println!( + " Leading whitespace result: {}", + if result2.is_ok() { "OK" } else { "FAIL" } + ); + + println!("✓ Whitespace handling tested"); + + Ok(()) +} + +// ============================================================================ +// Performance and Stress Edge Cases +// ============================================================================ + +#[tokio::test] +async fn test_authentication_performance_under_load() -> Result<()> { + println!("\n=== Test: Authentication Performance Under Load ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate token + let (token, _jti) = generate_test_token( + "user_perf", + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + )?; + + // Measure 100 sequential authentications + let mut latencies = vec![]; + + for _ in 0..100 { + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let start = Instant::now(); + let _ = auth_interceptor.clone().authenticate(request).await?; + latencies.push(start.elapsed()); + } + + // Calculate percentiles + latencies.sort(); + let p50 = latencies[49]; + let p95 = latencies[94]; + let p99 = latencies[98]; + + println!("\n Authentication Latency:"); + println!(" ├─ P50: {:?}", p50); + println!(" ├─ P95: {:?}", p95); + println!(" └─ P99: {:?}", p99); + println!(" Target: <10μs"); + + if p99 > Duration::from_micros(10) { + println!(" ⚠ WARNING: P99 latency exceeds 10μs target"); + } else { + println!(" ✓ Performance target met"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_concurrent_authentication_requests() -> Result<()> { + println!("\n=== Test: Concurrent Authentication Requests ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate token + let (token, _jti) = generate_test_token( + "user_concurrent_auth", + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + )?; + + // Spawn 50 concurrent authentication requests + let mut handles = vec![]; + + println!(" Spawning 50 concurrent auth requests..."); + for _ in 0..50 { + let interceptor = auth_interceptor.clone(); + let token_clone = token.clone(); + + let handle = tokio::spawn(async move { + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token_clone)).unwrap(), + ); + + interceptor.authenticate(request).await + }); + + handles.push(handle); + } + + // Collect results + let mut success_count = 0; + for handle in handles { + if let Ok(Ok(_)) = handle.await { + success_count += 1; + } + } + + println!(" ✓ {} / 50 concurrent auths succeeded", success_count); + assert_eq!( + success_count, 50, + "All concurrent authentications should succeed" + ); + + Ok(()) +} diff --git a/services/api/tests/auth_flow_tests.rs b/services/api/tests/auth_flow_tests.rs new file mode 100644 index 000000000..bfb21860c --- /dev/null +++ b/services/api/tests/auth_flow_tests.rs @@ -0,0 +1,519 @@ +//! Authentication Flow Integration Tests +//! +//! Comprehensive tests for the 8-layer authentication pipeline: +//! 1. mTLS client certificate validation +//! 2. JWT extraction from Authorization header +//! 3. JWT revocation check (Redis) +//! 4. JWT signature and expiration validation +//! 5. RBAC permission check +//! 6. Rate limiting +//! 7. User context injection +//! 8. Async audit logging + +#[path = "common/mod.rs"] +mod common; + +use anyhow::Result; +use common::{ + cleanup_redis, generate_expired_token, generate_invalid_signature_token, generate_test_token, + wait_for_redis, TestJwtConfig, +}; +use std::time::{Duration, Instant}; +use tonic::{metadata::MetadataValue, Request}; + +use api::auth::{ + AuditLogger, AuthInterceptor, AuthzService, Jti, JwtService, RateLimiter, RevocationService, +}; + +const REDIS_URL: &str = "redis://localhost:6379"; + +/// Setup test authentication components +async fn setup_auth_components() -> Result { + wait_for_redis(REDIS_URL, 50).await?; + cleanup_redis(REDIS_URL).await?; + + let config = TestJwtConfig::default(); + + let jwt_service = JwtService::new(config.secret, config.issuer, config.audience); + + let revocation_service = RevocationService::new(REDIS_URL).await?; + let authz_service = AuthzService::new(); + let rate_limiter = RateLimiter::new(100).map_err(|e| anyhow::anyhow!(e))?; // 100 req/s + let audit_logger = AuditLogger::new(true); + + Ok(AuthInterceptor::new( + jwt_service, + revocation_service, + authz_service, + rate_limiter, + audit_logger, + )) +} + +#[tokio::test] +async fn test_successful_authentication() -> Result<()> { + println!("\n=== Test: Successful Authentication ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate valid token + let (token, _jti) = generate_test_token( + "user123", + vec!["trader".to_string()], + vec!["api.access".to_string(), "trading.submit".to_string()], + 3600, + )?; + + // Create request with Authorization header + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + // Measure authentication time + let start = Instant::now(); + let result = auth_interceptor.clone().authenticate(request).await; + let elapsed = start.elapsed(); + + println!("✓ Authentication succeeded in {:?}", elapsed); + println!(" Performance target: <10μs, Actual: {:?}", elapsed); + + assert!(result.is_ok(), "Authentication should succeed"); + + // Verify user context was injected + let authenticated_request = result.unwrap(); + let extensions = authenticated_request.extensions(); + + assert!( + extensions.get::().is_some(), + "User context should be injected" + ); + + if let Some(user_ctx) = extensions.get::() { + assert_eq!(user_ctx.user_id, "user123"); + assert!(user_ctx.roles.contains(&"trader".to_string())); + assert!(user_ctx.permissions.contains(&"api.access".to_string())); + println!("✓ User context verified: user_id={}", user_ctx.user_id); + } + + // Warn if latency exceeds target + if elapsed > Duration::from_micros(10) { + println!( + "⚠ WARNING: Authentication latency {:?} exceeds 10μs target", + elapsed + ); + } + + Ok(()) +} + +#[tokio::test] +async fn test_missing_jwt_rejected() -> Result<()> { + println!("\n=== Test: Missing JWT Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Create request without Authorization header + let request = Request::new(()); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "Request without JWT should be rejected"); + + if let Err(status) = result { + assert_eq!(status.code(), tonic::Code::Unauthenticated); + println!("✓ Request rejected with status: {}", status.code()); + println!(" Message: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_revoked_jwt_rejected() -> Result<()> { + println!("\n=== Test: Revoked JWT Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate valid token + let (token, jti) = generate_test_token( + "user456", + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + )?; + + // Add token to blacklist + let revocation_service = RevocationService::new(REDIS_URL).await?; + revocation_service + .revoke_token(&Jti::from_string(jti), 3600) + .await?; + + println!("✓ Token added to blacklist"); + + // Create request with revoked token + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "Revoked token should be rejected"); + + if let Err(status) = result { + assert_eq!(status.code(), tonic::Code::Unauthenticated); + println!("✓ Revoked token rejected with status: {}", status.code()); + assert!( + status.message().contains("revoked"), + "Error message should mention revocation" + ); + } + + Ok(()) +} + +#[tokio::test] +async fn test_expired_jwt_rejected() -> Result<()> { + println!("\n=== Test: Expired JWT Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate expired token + let token = generate_expired_token("user789")?; + + // Create request with expired token + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "Expired token should be rejected"); + + if let Err(status) = result { + assert_eq!(status.code(), tonic::Code::Unauthenticated); + println!("✓ Expired token rejected with status: {}", status.code()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_invalid_signature_rejected() -> Result<()> { + println!("\n=== Test: Invalid Signature Rejected ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate token with wrong signature + let token = generate_invalid_signature_token("attacker")?; + + // Create request with invalid token + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!( + result.is_err(), + "Token with invalid signature should be rejected" + ); + + if let Err(status) = result { + assert_eq!(status.code(), tonic::Code::Unauthenticated); + println!( + "✓ Invalid signature rejected with status: {}", + status.code() + ); + } + + Ok(()) +} + +#[tokio::test] +async fn test_rbac_permission_denied() -> Result<()> { + println!("\n=== Test: RBAC Permission Denied ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate token without api.access permission + let (token, _jti) = generate_test_token( + "restricted_user", + vec!["guest".to_string()], + vec!["limited.access".to_string()], // Missing api.access + 3600, + )?; + + // Create request with limited permissions + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!( + result.is_err(), + "Request without api.access should be denied" + ); + + if let Err(status) = result { + assert_eq!(status.code(), tonic::Code::PermissionDenied); + println!("✓ Permission denied with status: {}", status.code()); + println!(" Message: {}", status.message()); + } + + Ok(()) +} + +#[tokio::test] +async fn test_rate_limit_exceeded() -> Result<()> { + println!("\n=== Test: Rate Limit Exceeded ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate valid token + let (token, _jti) = generate_test_token( + "rate_limited_user", + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + )?; + + let mut success_count = 0; + let mut rate_limited_count = 0; + + // Make 110 rapid requests (limit is 100/s) + for i in 1..=110 { + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + if result.is_ok() { + success_count += 1; + } else if let Err(status) = result { + if status.code() == tonic::Code::ResourceExhausted { + rate_limited_count += 1; + if rate_limited_count == 1 { + println!("✓ First rate limit hit at request #{}", i); + } + } + } + } + + println!(" Successful requests: {}", success_count); + println!(" Rate limited requests: {}", rate_limited_count); + + assert!( + rate_limited_count > 0, + "Some requests should be rate limited" + ); + // Allow small tolerance (3-5 extra) due to token bucket timing granularity + assert!( + success_count <= 105, + "Success count should not significantly exceed limit (got {}, limit 100, tolerance 105)", + success_count + ); + + Ok(()) +} + +#[tokio::test] +async fn test_8_layer_auth_performance() -> Result<()> { + println!("\n=== Test: 8-Layer Authentication Performance ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate valid token + let (token, _jti) = generate_test_token( + "perf_user", + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + )?; + + let mut latencies = Vec::new(); + + // Perform 100 authentication requests + println!(" Running 100 authentication requests..."); + for _ in 0..100 { + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let start = Instant::now(); + let result = auth_interceptor.clone().authenticate(request).await; + let elapsed = start.elapsed(); + + assert!(result.is_ok(), "Authentication should succeed"); + latencies.push(elapsed); + } + + // Calculate percentiles + latencies.sort(); + let p50 = latencies[49]; + let p95 = latencies[94]; + let p99 = latencies[98]; + let p999 = latencies[99]; + + println!("\n Performance Metrics:"); + println!(" ├─ P50: {:?}", p50); + println!(" ├─ P95: {:?}", p95); + println!(" ├─ P99: {:?}", p99); + println!(" └─ P99.9: {:?}", p999); + + println!("\n Target: <10μs per request"); + + // Performance assertions (may fail in CI/CD, so we just warn) + if p99 > Duration::from_micros(10) { + println!(" ⚠ WARNING: P99 latency {:?} exceeds 10μs target", p99); + } else { + println!(" ✓ P99 latency within 10μs target"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_concurrent_authentication() -> Result<()> { + println!("\n=== Test: Concurrent Authentication ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate tokens for 10 different users + let mut tokens = Vec::new(); + for i in 1..=10 { + let (token, _jti) = generate_test_token( + &format!("concurrent_user_{}", i), + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + )?; + tokens.push(token); + } + + // Spawn 100 concurrent authentication requests + let mut handles = Vec::new(); + + println!(" Spawning 100 concurrent authentication requests..."); + for i in 0..100 { + let token = tokens[i % 10].clone(); + let auth = auth_interceptor.clone(); + + let handle = tokio::spawn(async move { + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token)).unwrap(), + ); + + auth.authenticate(request).await + }); + + handles.push(handle); + } + + // Wait for all requests to complete + let mut success_count = 0; + for handle in handles { + if let Ok(result) = handle.await { + if result.is_ok() { + success_count += 1; + } + } + } + + println!( + " ✓ {}/100 concurrent authentications succeeded", + success_count + ); + assert_eq!(success_count, 100, "All concurrent requests should succeed"); + + Ok(()) +} + +#[tokio::test] +async fn test_user_context_injection() -> Result<()> { + println!("\n=== Test: User Context Injection (Layer 7) ==="); + + let auth_interceptor = setup_auth_components().await?; + + let (token, _jti) = generate_test_token( + "context_user", + vec!["admin".to_string(), "trader".to_string()], + vec!["api.access".to_string(), "admin.manage".to_string()], + 3600, + )?; + + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await?; + + // Verify user context + let user_ctx = result + .extensions() + .get::() + .expect("UserContext should be present"); + + println!(" ✓ User context injected:"); + println!(" ├─ User ID: {}", user_ctx.user_id); + println!(" ├─ Roles: {:?}", user_ctx.roles); + println!(" ├─ Permissions: {:?}", user_ctx.permissions); + println!(" └─ Session ID: {}", user_ctx.session_id); + + assert_eq!(user_ctx.user_id, "context_user"); + assert_eq!(user_ctx.roles.len(), 2); + assert_eq!(user_ctx.permissions.len(), 2); + assert!(user_ctx.roles.contains(&"admin".to_string())); + assert!(user_ctx.permissions.contains(&"admin.manage".to_string())); + + Ok(()) +} + +#[tokio::test] +async fn test_malformed_authorization_header() -> Result<()> { + println!("\n=== Test: Malformed Authorization Header ==="); + + let auth_interceptor = setup_auth_components().await?; + + let test_cases = vec![ + ("Basic dXNlcjpwYXNzd29yZA==", "Basic auth instead of Bearer"), + ("Bearer", "Bearer without token"), + ("Bearer ", "Bearer with empty token"), + ("", "Empty header"), + ("InvalidFormat token123", "Invalid format"), + ]; + + for (header_value, description) in test_cases { + let mut request = Request::new(()); + if !header_value.is_empty() { + request + .metadata_mut() + .insert("authorization", MetadataValue::try_from(header_value)?); + } + + let result = auth_interceptor.clone().authenticate(request).await; + assert!(result.is_err(), "{} should be rejected", description); + println!(" ✓ Rejected: {}", description); + } + + Ok(()) +} diff --git a/services/api/tests/common/mod.rs b/services/api/tests/common/mod.rs new file mode 100644 index 000000000..02e76e3ab --- /dev/null +++ b/services/api/tests/common/mod.rs @@ -0,0 +1,211 @@ +//! Common test utilities for API Gateway integration tests + +use anyhow::{Context, Result}; +use jsonwebtoken::{encode, EncodingKey, Header}; +use std::time::{SystemTime, UNIX_EPOCH}; +use uuid::Uuid; + +pub use api::auth::{Jti, JwtClaims}; + +/// Test JWT configuration +pub struct TestJwtConfig { + pub secret: String, + pub issuer: String, + pub audience: String, +} + +impl Default for TestJwtConfig { + fn default() -> Self { + Self { + secret: "test-secret-must-be-at-least-64-characters-long-for-security-validation-ok-1234567890".to_string(), + issuer: "foxhunt-api-gateway".to_string(), + audience: "foxhunt-services".to_string(), + } + } +} + +/// Generate a valid JWT token for testing +pub fn generate_test_token( + user_id: &str, + roles: Vec, + permissions: Vec, + ttl_seconds: u64, +) -> Result<(String, String)> { + let config = TestJwtConfig::default(); + let jti = Jti::new(); + + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + let claims = JwtClaims { + jti: jti.0.clone(), + sub: user_id.to_string(), + iat: now, + exp: now + ttl_seconds, + nbf: Some(now), // Not before: valid from now + iss: config.issuer, + aud: config.audience, + roles, + permissions, + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(config.secret.as_bytes()), + )?; + + Ok((token, jti.0)) +} + +/// Generate an expired JWT token for testing +pub fn generate_expired_token(user_id: &str) -> Result { + let config = TestJwtConfig::default(); + + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + let claims = JwtClaims { + jti: Jti::new().0, + sub: user_id.to_string(), + iat: now - 7200, + exp: now - 3600, // Expired 1 hour ago + nbf: Some(now - 7200), // Not before: from 2 hours ago + iss: config.issuer, + aud: config.audience, + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(config.secret.as_bytes()), + )?; + + Ok(token) +} + +/// Generate a token with invalid signature +pub fn generate_invalid_signature_token(user_id: &str) -> Result { + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + let claims = JwtClaims { + jti: Jti::new().0, + sub: user_id.to_string(), + iat: now, + exp: now + 3600, + nbf: Some(now), // Not before: valid from now + iss: "foxhunt-api-gateway".to_string(), + aud: "foxhunt-services".to_string(), + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + // Use wrong secret to create invalid signature + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(b"wrong-secret-key-that-will-fail-validation-checks-completely"), + )?; + + Ok(token) +} + +/// Wait for Redis to be ready with proper timeout handling +/// +/// CRITICAL FIX: The redis crate v0.27.6 does NOT support connection_timeout +/// or response_timeout as URL parameters. These are silently ignored, causing +/// 60+ second OS-level TCP timeouts when Redis is unavailable. +/// +/// Solution: Use tokio::time::timeout() to wrap async operations. +pub async fn wait_for_redis(redis_url: &str, max_attempts: usize) -> Result<()> { + use tokio::time::{timeout, Duration}; + + for attempt in 1..=max_attempts { + match redis::Client::open(redis_url) { + Ok(client) => { + // CRITICAL: Wrap connection with tokio timeout (2s) + match timeout( + Duration::from_secs(2), + client.get_multiplexed_async_connection() + ).await { + Ok(Ok(mut conn)) => { + // CRITICAL: Wrap PING command with tokio timeout (1s) + match timeout( + Duration::from_secs(1), + redis::cmd("PING").query_async::(&mut conn) + ).await { + Ok(Ok(_)) => { + println!("✓ Redis ready after {} attempts", attempt); + return Ok(()); + }, + Ok(Err(e)) => { + if attempt == max_attempts { + return Err(anyhow::anyhow!("Redis PING failed: {}", e)); + } + }, + Err(_) => { + if attempt == max_attempts { + return Err(anyhow::anyhow!("Redis PING timeout after 1s")); + } + }, + } + }, + Ok(Err(e)) => { + if attempt == max_attempts { + return Err(anyhow::anyhow!("Redis connection failed: {}", e)); + } + }, + Err(_) => { + if attempt == max_attempts { + return Err(anyhow::anyhow!("Redis connection timeout after 2s")); + } + }, + } + }, + Err(e) => { + if attempt == max_attempts { + return Err(anyhow::anyhow!("Failed to create Redis client: {}", e)); + } + }, + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + + Err(anyhow::anyhow!( + "Redis not ready after {} attempts", + max_attempts + )) +} + +/// Clean up Redis test data with proper timeout handling +pub async fn cleanup_redis(redis_url: &str) -> Result<()> { + use tokio::time::{timeout, Duration}; + + let client = redis::Client::open(redis_url)?; + + // CRITICAL: Wrap connection with tokio timeout (2s) + let mut conn = timeout( + Duration::from_secs(2), + client.get_multiplexed_async_connection() + ) + .await + .context("Redis connection timed out after 2s")? + .context("Failed to connect to Redis")?; + + // CRITICAL: Wrap FLUSHDB with tokio timeout (1s) + timeout( + Duration::from_secs(1), + redis::cmd("FLUSHDB").query_async::<()>(&mut conn) + ) + .await + .context("FLUSHDB timeout after 1s")? + .context("FLUSHDB failed")?; + + Ok(()) +} diff --git a/services/api/tests/docker-compose.yml b/services/api/tests/docker-compose.yml new file mode 100644 index 000000000..4ec4ea37f --- /dev/null +++ b/services/api/tests/docker-compose.yml @@ -0,0 +1,36 @@ +version: '3.8' + +services: + # Redis for JWT revocation and rate limiting + redis: + image: redis:7-alpine + container_name: api_test_redis + ports: + - "6380:6379" + command: redis-server --save "" --appendonly no --maxmemory 256mb --maxmemory-policy allkeys-lru + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 1s + timeout: 3s + retries: 5 + + # PostgreSQL for configuration (if needed) + postgres: + image: postgres:16-alpine + container_name: api_test_postgres + ports: + - "5433:5432" + environment: + POSTGRES_DB: foxhunt_test + POSTGRES_USER: foxhunt_test + POSTGRES_PASSWORD: test_password + POSTGRES_INITDB_ARGS: "-E UTF8" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U foxhunt_test"] + interval: 1s + timeout: 3s + retries: 5 + +networks: + default: + name: api_test_network diff --git a/services/api/tests/e2e_tests.rs b/services/api/tests/e2e_tests.rs new file mode 100644 index 000000000..4a6c875ec --- /dev/null +++ b/services/api/tests/e2e_tests.rs @@ -0,0 +1,879 @@ +//! End-to-End Tests for API Gateway Service +//! +//! Comprehensive E2E testing covering: +//! 1. Authentication flow (JWT generation, validation, expiration) +//! 2. MFA flow (enrollment, TOTP verification with pgcrypto encryption) +//! 3. Rate limiting (request throttling, burst protection) +//! 4. Request routing (proxy to backend services) +//! 5. Session management (session creation, validation, expiration) +//! 6. Audit logging (all operations logged to PostgreSQL) +//! +//! Test Count: 35+ tests +//! Coverage: Complete authentication and authorization pipeline + +#[path = "common/mod.rs"] +mod common; + +use anyhow::Result; +use chrono::Utc; +use common::{ + cleanup_redis, generate_expired_token, generate_invalid_signature_token, generate_test_token, + wait_for_redis, TestJwtConfig, +}; +use sqlx::Row; +use std::time::{Duration as StdDuration, Instant}; +use tonic::{metadata::MetadataValue, Request}; +use uuid::Uuid; + +use api::auth::{ + AuditLogger, AuthInterceptor, AuthzService, JwtService, RateLimiter, RevocationService, +}; +#[cfg(feature = "mfa")] +use api::auth::mfa::{MfaManager, TotpGenerator}; + +const REDIS_URL: &str = "redis://localhost:6379"; +const DATABASE_URL: &str = "postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt"; + +// ============================================================================ +// SECTION 1: AUTHENTICATION FLOW E2E TESTS (12 tests) +// ============================================================================ + +/// Setup test authentication components +async fn setup_auth_components() -> Result { + wait_for_redis(REDIS_URL, 50).await?; + cleanup_redis(REDIS_URL).await?; + + let config = TestJwtConfig::default(); + + let jwt_service = JwtService::new(config.secret, config.issuer, config.audience); + + let revocation_service = RevocationService::new(REDIS_URL).await?; + let authz_service = AuthzService::new(); + let rate_limiter = RateLimiter::new(100).map_err(|e| anyhow::anyhow!(e))?; // 100 req/s + let audit_logger = AuditLogger::new(true); + + Ok(AuthInterceptor::new( + jwt_service, + revocation_service, + authz_service, + rate_limiter, + audit_logger, + )) +} + +#[tokio::test] +async fn test_e2e_successful_authentication_flow() -> Result<()> { + println!("\n=== E2E Test: Successful Authentication Flow ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate valid token + let (token, _jti) = generate_test_token( + "user123", + vec!["trader".to_string()], + vec!["api.access".to_string(), "trading.submit".to_string()], + 3600, + )?; + + // Create request with Authorization header + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + // Measure authentication time (E2E performance) + let start = Instant::now(); + let result = auth_interceptor.clone().authenticate(request).await; + let elapsed = start.elapsed(); + + println!("✓ E2E authentication succeeded in {:?}", elapsed); + println!(" Performance target: <10μs, Actual: {:?}", elapsed); + + assert!(result.is_ok(), "E2E authentication should succeed"); + + // Verify user context was injected + let authenticated_request = result.unwrap(); + let extensions = authenticated_request.extensions(); + + assert!( + extensions.get::().is_some(), + "User context should be injected" + ); + + if let Some(user_ctx) = extensions.get::() { + assert_eq!(user_ctx.user_id, "user123"); + assert!(user_ctx.roles.contains(&"trader".to_string())); + assert!(user_ctx.permissions.contains(&"api.access".to_string())); + println!( + "✓ User context verified: user_id={}, roles={:?}", + user_ctx.user_id, user_ctx.roles + ); + } + + Ok(()) +} + +#[tokio::test] +async fn test_e2e_authentication_with_expired_token() -> Result<()> { + println!("\n=== E2E Test: Authentication with Expired Token ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate expired token + let token = generate_expired_token("user123")?; + + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "Expired token should be rejected"); + println!("✓ Expired token correctly rejected"); + + Ok(()) +} + +#[tokio::test] +async fn test_e2e_authentication_with_invalid_signature() -> Result<()> { + println!("\n=== E2E Test: Authentication with Invalid Signature ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate token with invalid signature + let token = generate_invalid_signature_token("user123")?; + + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!( + result.is_err(), + "Token with invalid signature should be rejected" + ); + println!("✓ Invalid signature correctly rejected"); + + Ok(()) +} + +#[tokio::test] +async fn test_e2e_authentication_missing_authorization_header() -> Result<()> { + println!("\n=== E2E Test: Authentication without Authorization Header ==="); + + let auth_interceptor = setup_auth_components().await?; + + let request = Request::new(()); // No Authorization header + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!( + result.is_err(), + "Request without Authorization header should be rejected" + ); + println!("✓ Missing Authorization header correctly rejected"); + + Ok(()) +} + +#[tokio::test] +async fn test_e2e_authentication_malformed_bearer_token() -> Result<()> { + println!("\n=== E2E Test: Authentication with Malformed Bearer Token ==="); + + let auth_interceptor = setup_auth_components().await?; + + let mut request = Request::new(()); + request + .metadata_mut() + .insert("authorization", MetadataValue::from_static("InvalidFormat")); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!( + result.is_err(), + "Malformed Authorization header should be rejected" + ); + println!("✓ Malformed bearer token correctly rejected"); + + Ok(()) +} + +#[tokio::test] +async fn test_e2e_jwt_revocation_check() -> Result<()> { + println!("\n=== E2E Test: JWT Revocation Check ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Generate valid token + let (token, jti) = generate_test_token( + "user123", + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + )?; + + // Revoke the token + let revocation_service = RevocationService::new(REDIS_URL).await?; + use api::auth::Jti; + let jti_obj = Jti(jti); + revocation_service.revoke_token(&jti_obj, 3600).await?; + println!("✓ Token revoked: {}", jti_obj.0); + + // Try to authenticate with revoked token + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + + let result = auth_interceptor.clone().authenticate(request).await; + + assert!(result.is_err(), "Revoked token should be rejected"); + println!("✓ Revoked token correctly rejected"); + + Ok(()) +} + +#[tokio::test] +async fn test_e2e_multiple_concurrent_authentications() -> Result<()> { + println!("\n=== E2E Test: Multiple Concurrent Authentications ==="); + + let auth_interceptor = setup_auth_components().await?; + + // Create multiple concurrent authentication requests + let mut handles = vec![]; + + for i in 0..10 { + let interceptor = auth_interceptor.clone(); + let handle = tokio::spawn(async move { + let (token, _) = generate_test_token( + &format!("user{}", i), + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + ) + .unwrap(); + + let mut request = Request::new(()); + request + .metadata_mut() + .insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token)).unwrap(), + ) + .unwrap(); + + interceptor.authenticate(request).await + }); + handles.push(handle); + } + + // Wait for all authentications to complete + let results = futures::future::join_all(handles).await; + + let successes = results.iter().filter(|r| r.is_ok()).count(); + assert_eq!( + successes, 10, + "All concurrent authentications should succeed" + ); + + println!("✓ {} concurrent authentications succeeded", successes); + + Ok(()) +} + +// ============================================================================ +// SECTION 2: MFA E2E TESTS (10 tests) +// ============================================================================ + +#[cfg(feature = "mfa")] +async fn setup_mfa_manager() -> Result { + let db_pool = sqlx::PgPool::connect(DATABASE_URL).await?; + + // Create test encryption key (in production, this comes from secure vault) + let encryption_key = "test-encryption-key-32-bytes-long-12345678".to_string(); + + MfaManager::new(db_pool, encryption_key) +} + +#[cfg(feature = "mfa")] +async fn create_test_user(db_pool: &sqlx::PgPool) -> Result { + let user_id = Uuid::new_v4(); + sqlx::query( + r#" + INSERT INTO users (id, username, email, password_hash, created_at) + VALUES ($1, $2, $3, $4, NOW()) + ON CONFLICT (id) DO NOTHING + "#, + ) + .bind(user_id) + .bind(format!("testuser_{}", user_id)) + .bind(format!("test_{}@example.com", user_id)) + .bind("hashed_password_placeholder") + .execute(db_pool) + .await?; + + Ok(user_id) +} + +#[cfg(feature = "mfa")] +#[tokio::test] +async fn test_e2e_mfa_enrollment_flow() -> Result<()> { + println!("\n=== E2E Test: MFA Enrollment Flow ==="); + + let mfa_manager = setup_mfa_manager().await?; + let db_pool = sqlx::PgPool::connect(DATABASE_URL).await?; + let user_id = create_test_user(&db_pool).await?; + + // Step 1: Start enrollment + let enrollment_session = mfa_manager + .start_enrollment(user_id, "Foxhunt", &format!("user_{}", user_id)) + .await?; + println!("✓ Enrollment started for user: {}", user_id); + + assert!( + enrollment_session + .qr_code_uri + .starts_with("otpauth://totp/"), + "QR code should be valid TOTP URI" + ); + println!("✓ QR code generated: {}", enrollment_session.qr_code_uri); + + // Step 2: Generate TOTP code from manual entry key + let totp_generator = TotpGenerator::new(); + let totp_code = totp_generator.generate_code(&enrollment_session.manual_entry_key)?; + println!("✓ Generated TOTP code: {}", totp_code); + + // Step 3: Complete enrollment with TOTP code + let backup_codes = mfa_manager + .complete_enrollment(enrollment_session.session_id, user_id, &totp_code) + .await?; + + assert_eq!(backup_codes.len(), 10, "Should generate 10 backup codes"); + println!( + "✓ Enrollment completed successfully with {} backup codes", + backup_codes.len() + ); + + // Clean up test user + sqlx::query("DELETE FROM users WHERE id = $1") + .bind(user_id) + .execute(&db_pool) + .await?; + + Ok(()) +} + +#[cfg(feature = "mfa")] +#[tokio::test] +async fn test_e2e_mfa_totp_verification() -> Result<()> { + println!("\n=== E2E Test: MFA TOTP Verification ==="); + + let mfa_manager = setup_mfa_manager().await?; + let db_pool = sqlx::PgPool::connect(DATABASE_URL).await?; + let user_id = create_test_user(&db_pool).await?; + + // Enroll MFA first + let enrollment = mfa_manager + .start_enrollment(user_id, "Foxhunt", &format!("user_{}", user_id)) + .await?; + let totp_generator = TotpGenerator::new(); + let enrollment_code = totp_generator.generate_code(&enrollment.manual_entry_key)?; + let _backup_codes = mfa_manager + .complete_enrollment(enrollment.session_id, user_id, &enrollment_code) + .await?; + println!("✓ MFA enrolled for user: {}", user_id); + + // Generate new TOTP code for verification + let verification_code = totp_generator.generate_code(&enrollment.manual_entry_key)?; + + // Verify TOTP code (verify_totp returns Result) + let verified = mfa_manager + .verify_totp( + user_id, + &verification_code, + Some("192.168.1.100".to_string()), + ) + .await?; + + assert!(verified, "TOTP verification should succeed"); + println!("✓ TOTP verification succeeded"); + + // Clean up + sqlx::query("DELETE FROM users WHERE id = $1") + .bind(user_id) + .execute(&db_pool) + .await?; + + Ok(()) +} + +#[cfg(feature = "mfa")] +#[tokio::test] +async fn test_e2e_mfa_backup_code_generation_and_usage() -> Result<()> { + println!("\n=== E2E Test: MFA Backup Code Generation and Usage ==="); + + let mfa_manager = setup_mfa_manager().await?; + let db_pool = sqlx::PgPool::connect(DATABASE_URL).await?; + let user_id = create_test_user(&db_pool).await?; + + // Enroll MFA (backup codes are generated during enrollment) + let enrollment = mfa_manager + .start_enrollment(user_id, "Foxhunt", &format!("user_{}", user_id)) + .await?; + let totp_generator = TotpGenerator::new(); + let enrollment_code = totp_generator.generate_code(&enrollment.manual_entry_key)?; + let backup_codes = mfa_manager + .complete_enrollment(enrollment.session_id, user_id, &enrollment_code) + .await?; + + assert_eq!(backup_codes.len(), 10, "Should generate 10 backup codes"); + println!("✓ Generated {} backup codes", backup_codes.len()); + + // Use first backup code (get the actual code string from Secret) + use secrecy::ExposeSecret; + let first_code = backup_codes[0].code.expose_secret(); + let verified = mfa_manager + .verify_backup_code(user_id, first_code, Some("192.168.1.100".to_string())) + .await?; + + assert!(verified, "Backup code verification should succeed"); + println!("✓ Backup code verified successfully"); + + // Try to reuse the same backup code (should fail) + let reuse_result = mfa_manager + .verify_backup_code(user_id, first_code, Some("192.168.1.100".to_string())) + .await; + assert!( + reuse_result.is_err() || !reuse_result.unwrap(), + "Reused backup code should fail" + ); + println!("✓ Backup code reuse correctly prevented"); + + // Clean up + sqlx::query("DELETE FROM users WHERE id = $1") + .bind(user_id) + .execute(&db_pool) + .await?; + + Ok(()) +} + +#[cfg(feature = "mfa")] +#[tokio::test] +async fn test_e2e_mfa_encryption_verification() -> Result<()> { + println!("\n=== E2E Test: MFA Secret Encryption Verification ==="); + + let db_pool = sqlx::PgPool::connect(DATABASE_URL).await?; + + // Test the encryption functions directly + let test_secret = "JBSWY3DPEHPK3PXP"; + + // Encrypt secret using PostgreSQL function + let encrypted: Vec = sqlx::query_scalar("SELECT encrypt_mfa_secret($1)") + .bind(test_secret) + .fetch_one(&db_pool) + .await?; + + println!("✓ Secret encrypted successfully"); + + // Decrypt secret + let decrypted: String = sqlx::query_scalar("SELECT decrypt_mfa_secret($1)") + .bind(&encrypted) + .fetch_one(&db_pool) + .await?; + + assert_eq!( + decrypted, test_secret, + "Decrypted secret should match original" + ); + println!("✓ Secret decrypted correctly: {}", test_secret); + + // Verify encrypted data is different from plaintext + assert_ne!( + encrypted, + test_secret.as_bytes(), + "Encrypted data should differ from plaintext" + ); + println!("✓ Encryption verification PASSED"); + + Ok(()) +} + +#[cfg(feature = "mfa")] +#[tokio::test] +async fn test_e2e_mfa_account_lockout_after_failed_attempts() -> Result<()> { + println!("\n=== E2E Test: MFA Account Lockout After Failed Attempts ==="); + + let mfa_manager = setup_mfa_manager().await?; + let db_pool = sqlx::PgPool::connect(DATABASE_URL).await?; + let user_id = create_test_user(&db_pool).await?; + + // Enroll MFA + let enrollment = mfa_manager + .start_enrollment(user_id, "Foxhunt", &format!("user_{}", user_id)) + .await?; + let totp_generator = TotpGenerator::new(); + let enrollment_code = totp_generator.generate_code(&enrollment.manual_entry_key)?; + let _backup_codes = mfa_manager + .complete_enrollment(enrollment.session_id, user_id, &enrollment_code) + .await?; + + // Attempt verification with wrong code multiple times + for _i in 1..=5 { + let _result = mfa_manager + .verify_totp(user_id, "000000", Some("192.168.1.100".to_string())) + .await; + // Ignore errors - we expect these to fail + } + + // Check if account is locked + let row = sqlx::query( + r#" + SELECT locked_until, failed_verification_attempts + FROM mfa_config + WHERE user_id = $1 + "#, + ) + .bind(user_id) + .fetch_one(&db_pool) + .await?; + + let locked_until: Option> = row.try_get("locked_until")?; + let failed_attempts: i32 = row.try_get("failed_verification_attempts")?; + + assert!( + locked_until.is_some(), + "Account should be locked after 5 failed attempts" + ); + assert_eq!(failed_attempts, 5, "Should record 5 failed attempts"); + println!("✓ Account locked until: {:?}", locked_until.unwrap()); + + // Clean up + sqlx::query("DELETE FROM users WHERE id = $1") + .bind(user_id) + .execute(&db_pool) + .await?; + + Ok(()) +} + +// ============================================================================ +// SECTION 3: RATE LIMITING E2E TESTS (5 tests) +// ============================================================================ + +#[tokio::test] +async fn test_e2e_rate_limiting_enforcement() -> Result<()> { + println!("\n=== E2E Test: Rate Limiting Enforcement ==="); + + let rate_limiter = RateLimiter::new(5).map_err(|e| anyhow::anyhow!(e))?; // 5 req/s limit + + let user_id = "user123"; + + // Send 5 requests (should succeed) + for i in 1..=5 { + let allowed = rate_limiter.check_rate_limit(user_id); + assert!(allowed, "Request {} should be allowed", i); + } + println!("✓ 5 requests allowed within rate limit"); + + // 6th request should be rate limited + let allowed = rate_limiter.check_rate_limit(user_id); + assert!(!allowed, "6th request should be rate limited"); + println!("✓ Request #6 correctly rate limited"); + + Ok(()) +} + +#[tokio::test] +async fn test_e2e_rate_limiting_reset_after_window() -> Result<()> { + println!("\n=== E2E Test: Rate Limit Reset After Window ==="); + + let rate_limiter = RateLimiter::new(3).map_err(|e| anyhow::anyhow!(e))?; // 3 req/s + + let user_id = "user456"; + + // Use up the rate limit + for _ in 0..3 { + rate_limiter.check_rate_limit(user_id); + } + + // Should be rate limited now + assert!( + !rate_limiter.check_rate_limit(user_id), + "Should be rate limited" + ); + + // Wait for rate limit window to reset (1 second) + tokio::time::sleep(StdDuration::from_secs(1)).await; + + // Should be allowed again + let allowed = rate_limiter.check_rate_limit(user_id); + assert!(allowed, "Should be allowed after rate limit reset"); + println!("✓ Rate limit reset after 1 second window"); + + Ok(()) +} + +#[tokio::test] +async fn test_e2e_rate_limiting_per_user_isolation() -> Result<()> { + println!("\n=== E2E Test: Rate Limiting Per-User Isolation ==="); + + let rate_limiter = RateLimiter::new(2).map_err(|e| anyhow::anyhow!(e))?; // 2 req/s + + let user1 = "user_alpha"; + let user2 = "user_beta"; + + // User1 uses up their limit + assert!(rate_limiter.check_rate_limit(user1)); + assert!(rate_limiter.check_rate_limit(user1)); + assert!(!rate_limiter.check_rate_limit(user1)); // Rate limited + + // User2 should still have their full quota + assert!( + rate_limiter.check_rate_limit(user2), + "User2 should not be affected by User1's rate limit" + ); + assert!(rate_limiter.check_rate_limit(user2)); + println!("✓ Per-user rate limit isolation verified"); + + Ok(()) +} + +// ============================================================================ +// SECTION 4: SESSION MANAGEMENT E2E TESTS (5 tests) +// ============================================================================ + +#[tokio::test] +async fn test_e2e_session_creation_and_validation() -> Result<()> { + println!("\n=== E2E Test: Session Creation and Validation ==="); + + let config = TestJwtConfig::default(); + let jwt_service = JwtService::new(config.secret, config.issuer, config.audience); + + let user_id = "session_user_123"; + + // Generate token with session ID + let (token, _jti) = generate_test_token( + user_id, + vec!["trader".to_string()], + vec!["api.access".to_string()], + 3600, + )?; + + // Validate token + let claims = jwt_service.validate_token(&token)?; + + assert_eq!(claims.sub, user_id, "User ID should match"); + assert!( + claims.session_id.is_some(), + "Session ID should be present in token" + ); + println!( + "✓ Session created and validated: {}", + claims.session_id.unwrap() + ); + + Ok(()) +} + +#[tokio::test] +async fn test_e2e_session_expiration() -> Result<()> { + println!("\n=== E2E Test: Session Expiration ==="); + + let config = TestJwtConfig::default(); + let jwt_service = JwtService::new(config.secret, config.issuer, config.audience); + + // Create token with 1 second TTL (simulating expired session) + let expired_token = generate_expired_token("session_user_789")?; + + // Validate should fail + let result = jwt_service.validate_token(&expired_token); + assert!(result.is_err(), "Expired session token should be rejected"); + println!("✓ Expired session correctly rejected"); + + Ok(()) +} + +// ============================================================================ +// SECTION 5: AUDIT LOGGING E2E TESTS (3 tests) +// ============================================================================ + +#[tokio::test] +async fn test_e2e_audit_logging_authentication_events() -> Result<()> { + println!("\n=== E2E Test: Audit Logging for Authentication Events ==="); + + let audit_logger = AuditLogger::new(true); // Enable audit logging + + // Simulate successful authentication + audit_logger.log_auth_success("user123", Some("192.168.1.100")); + println!("✓ Successful authentication logged"); + + // Simulate failed authentication + audit_logger.log_auth_failure("invalid_credentials", Some("192.168.1.200")); + println!("✓ Failed authentication logged"); + + println!("✓ Authentication events logged to audit system"); + + // Note: In a real E2E test, we would query the database to verify logs + // For this test, we're verifying the logging API works + + Ok(()) +} + +#[cfg(feature = "mfa")] +#[tokio::test] +async fn test_e2e_audit_logging_mfa_events() -> Result<()> { + println!("\n=== E2E Test: Audit Logging for MFA Events ==="); + + let audit_logger = AuditLogger::new(true); + + // Simulate MFA enrollment success + audit_logger.log_auth_success("user789_mfa_enrolled", Some("192.168.1.150")); + println!("✓ MFA enrollment logged"); + + // Simulate MFA verification success + audit_logger.log_auth_success("user789_mfa_verified", Some("192.168.1.150")); + println!("✓ MFA verification success logged"); + + // Simulate failed MFA verification + audit_logger.log_auth_failure("mfa_code_invalid", Some("192.168.1.151")); + println!("✓ MFA verification failure logged"); + + println!("✓ MFA events logged to audit system"); + + Ok(()) +} + +#[tokio::test] +async fn test_e2e_complete_authentication_pipeline() -> Result<()> { + println!("\n=== E2E Test: Complete Authentication Pipeline ==="); + + // This test simulates the complete flow: + // 1. User authenticates with JWT + // 2. JWT is validated + // 3. User context is created + // 4. Rate limiting is checked + // 5. Authorization is verified + // 6. Audit log is created + + let auth_interceptor = setup_auth_components().await?; + + // Step 1: Generate valid JWT + let (token, _jti) = generate_test_token( + "pipeline_user", + vec!["admin".to_string()], + vec![ + "api.access".to_string(), + "trading.submit".to_string(), + "system.admin".to_string(), + ], + 3600, + )?; + println!("✓ Step 1: JWT generated"); + + // Step 2: Create authenticated request + let mut request = Request::new(()); + request.metadata_mut().insert( + "authorization", + MetadataValue::try_from(format!("Bearer {}", token))?, + ); + println!("✓ Step 2: Request prepared with Authorization header"); + + // Step 3: Authenticate (runs through full pipeline) + let start = Instant::now(); + let result = auth_interceptor.authenticate(request).await; + let elapsed = start.elapsed(); + + assert!(result.is_ok(), "Complete pipeline should succeed"); + println!( + "✓ Step 3: Authentication pipeline completed in {:?}", + elapsed + ); + + // Step 4: Verify all components worked + let authenticated_request = result.unwrap(); + let extensions = authenticated_request.extensions(); + let user_ctx = extensions + .get::() + .expect("User context should exist"); + + assert_eq!(user_ctx.user_id, "pipeline_user"); + assert!(user_ctx.roles.contains(&"admin".to_string())); + assert!(user_ctx.permissions.contains(&"system.admin".to_string())); + + println!("✓ Step 4: User context verified"); + println!(" User ID: {}", user_ctx.user_id); + println!(" Roles: {:?}", user_ctx.roles); + println!(" Permissions: {:?}", user_ctx.permissions); + + println!("\n✓ Complete E2E authentication pipeline PASSED"); + + Ok(()) +} + +// ============================================================================ +// SECTION 6: AUTHORIZATION E2E TESTS (5 tests) +// ============================================================================ + +#[tokio::test] +async fn test_e2e_authorization_permission_check() -> Result<()> { + println!("\n=== E2E Test: Authorization Permission Check ==="); + + let authz_service = AuthzService::new(); + + // Cache user permissions first + authz_service.cache_permissions( + "user123".to_string(), + vec!["trading.submit".to_string(), "api.access".to_string()], + ); + + // User with trading permissions + let has_permission = authz_service.has_permission("user123", "trading.submit"); + assert!(has_permission, "User should have trading.submit permission"); + println!("✓ Permission check succeeded: trading.submit"); + + // User without admin permissions + let has_permission = authz_service.has_permission("user123", "system.admin"); + assert!( + !has_permission, + "User should NOT have system.admin permission" + ); + println!("✓ Permission check denied: system.admin"); + + Ok(()) +} + +#[tokio::test] +async fn test_e2e_authorization_cache_management() -> Result<()> { + println!("\n=== E2E Test: Authorization Cache Management ==="); + + let authz_service = AuthzService::new(); + + // Cache permissions + authz_service.cache_permissions( + "cache_user".to_string(), + vec!["read".to_string(), "write".to_string()], + ); + + // Verify cached permissions + assert!(authz_service.has_permission("cache_user", "read")); + assert!(authz_service.has_permission("cache_user", "write")); + println!("✓ Permissions cached and verified"); + + // Clear cache + authz_service.clear_cache("cache_user"); + + // Verify permissions are cleared + assert!(!authz_service.has_permission("cache_user", "read")); + assert!(!authz_service.has_permission("cache_user", "write")); + println!("✓ Permission cache cleared successfully"); + + Ok(()) +} diff --git a/services/api/tests/grpc_error_handling.rs b/services/api/tests/grpc_error_handling.rs new file mode 100644 index 000000000..9b4e79c10 --- /dev/null +++ b/services/api/tests/grpc_error_handling.rs @@ -0,0 +1,627 @@ +//! Comprehensive gRPC Error Handling Tests for API Gateway +//! +//! This test suite validates all gRPC error codes and edge cases for the API Gateway, +//! focusing on authentication, rate limiting, routing, and proxy error handling. +//! +//! Coverage areas: +//! - InvalidArgument: Malformed requests, missing required fields +//! - Unauthenticated: JWT validation failures, expired tokens, missing auth +//! - PermissionDenied: Insufficient roles/permissions, MFA requirements +//! - ResourceExhausted: Rate limiting, quota exceeded +//! - Unavailable: Backend service down, timeout +//! - Internal: Server errors, configuration issues +//! - DeadlineExceeded: Request timeout +//! - FailedPrecondition: MFA not configured, service prerequisites +//! +//! Total: 15 comprehensive error scenario tests + +use anyhow::Result; +use std::time::Duration; +use tonic::{Code, Request}; + +// Import TLI proto definitions (API Gateway interface) +use fxt::proto::trading::{ + trading_service_client::TradingServiceClient, CancelOrderRequest, GetOrderStatusRequest, + SubmitOrderRequest, +}; + +// ============================================================================ +// HELPER FUNCTIONS +// ============================================================================ + +/// Create authenticated API Gateway client +async fn create_authenticated_client() -> Result< + TradingServiceClient< + tonic::service::interceptor::InterceptedService< + tonic::transport::Channel, + impl tonic::service::Interceptor + Clone, + >, + >, +> { + let channel = tonic::transport::Channel::from_static("http://localhost:50051") + .connect() + .await?; + + // Create valid JWT token for authentication + let token = create_valid_jwt_token()?; + + // Create interceptor closure that adds JWT auth header + let interceptor = move |mut req: Request<()>| { + req.metadata_mut().insert( + "authorization", + format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"), + ); + Ok(req) + }; + + // Create client with interceptor + let client = TradingServiceClient::with_interceptor(channel, interceptor); + + Ok(client) +} + +/// Create unauthenticated client (no JWT token) +async fn create_unauthenticated_client() -> Result> +{ + let channel = tonic::transport::Channel::from_static("http://localhost:50051") + .connect() + .await?; + Ok(TradingServiceClient::new(channel)) +} + +/// Generate valid JWT token for testing +fn create_valid_jwt_token() -> Result { + use chrono::{Duration, Utc}; + use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize)] + struct Claims { + sub: String, + exp: usize, + iat: usize, + iss: String, + aud: String, + roles: Vec, + permissions: Vec, + jti: String, + } + + let jwt_secret = std::env::var("JWT_SECRET") + .unwrap_or_else(|_| "OvFLDUbIDak3CSCi5t6zKfsAp65cjTOJ85q9YE+TFY8b361DGg1gSTra2rW6mps3cWrRGQ/NXRA5uftUpMldvOaEHMMgfBs4JjVODDElREdvUFm0EttD1A==".to_string()); + + let claims = Claims { + sub: "test_user_001".to_string(), + exp: (Utc::now() + Duration::hours(1)).timestamp() as usize, + iat: Utc::now().timestamp() as usize, + iss: "foxhunt-api-gateway".to_string(), + aud: "foxhunt-trading".to_string(), + roles: vec!["trader".to_string()], + permissions: vec!["trade:submit".to_string(), "trade:cancel".to_string()], + jti: uuid::Uuid::new_v4().to_string(), + }; + + let token = encode( + &Header::new(Algorithm::HS256), + &claims, + &EncodingKey::from_secret(jwt_secret.as_bytes()), + )?; + + Ok(token) +} + +/// Generate expired JWT token +fn create_expired_jwt_token() -> Result { + use chrono::{Duration, Utc}; + use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize)] + struct Claims { + sub: String, + exp: usize, + iat: usize, + iss: String, + aud: String, + roles: Vec, + permissions: Vec, + jti: String, + } + + let jwt_secret = std::env::var("JWT_SECRET") + .unwrap_or_else(|_| "OvFLDUbIDak3CSCi5t6zKfsAp65cjTOJ85q9YE+TFY8b361DGg1gSTra2rW6mps3cWrRGQ/NXRA5uftUpMldvOaEHMMgfBs4JjVODDElREdvUFm0EttD1A==".to_string()); + + let claims = Claims { + sub: "test_user_002".to_string(), + exp: (Utc::now() - Duration::hours(1)).timestamp() as usize, // Expired 1 hour ago + iat: (Utc::now() - Duration::hours(2)).timestamp() as usize, + iss: "foxhunt-api-gateway".to_string(), + aud: "foxhunt-trading".to_string(), + roles: vec!["trader".to_string()], + permissions: vec!["trade:submit".to_string()], + jti: uuid::Uuid::new_v4().to_string(), + }; + + let token = encode( + &Header::new(Algorithm::HS256), + &claims, + &EncodingKey::from_secret(jwt_secret.as_bytes()), + )?; + + Ok(token) +} + +// ============================================================================ +// UNAUTHENTICATED TESTS (Authentication Failures) +// ============================================================================ + +#[tokio::test] +async fn test_submit_order_without_token_returns_unauthenticated() -> Result<()> { + println!("\n=== Test: Submit Order - No Token (Unauthenticated) ==="); + + let mut client = create_unauthenticated_client().await?; + + let request = Request::new(SubmitOrderRequest { + symbol: "BTC/USD".to_string(), + side: 1, + order_type: 1, + quantity: 1.0, + price: None, + stop_price: None, + time_in_force: "GTC".to_string(), + client_order_id: "test_001".to_string(), + }); + + let result = client.submit_order(request).await; + + assert!(result.is_err(), "Expected error for missing authentication"); + let status = result.unwrap_err(); + assert_eq!( + status.code(), + Code::Unauthenticated, + "Expected Unauthenticated error code" + ); + assert!( + status.message().contains("authentication") + || status.message().contains("token") + || status.message().contains("unauthorized"), + "Error message should mention authentication" + ); + + println!(" ✓ Request without token correctly rejected"); + Ok(()) +} + +#[tokio::test] +async fn test_submit_order_with_expired_token_returns_unauthenticated() -> Result<()> { + println!("\n=== Test: Submit Order - Expired Token (Unauthenticated) ==="); + + let channel = tonic::transport::Channel::from_static("http://localhost:50051") + .connect() + .await?; + + let expired_token = create_expired_jwt_token()?; + + let mut client = TradingServiceClient::with_interceptor(channel, move |mut req: Request<()>| { + req.metadata_mut().insert( + "authorization", + format!("Bearer {}", expired_token).parse().expect("INVARIANT: Valid parse input"), + ); + Ok(req) + }); + + let request = Request::new(SubmitOrderRequest { + symbol: "ETH/USD".to_string(), + side: 1, + order_type: 1, + quantity: 1.0, + price: None, + stop_price: None, + time_in_force: "GTC".to_string(), + client_order_id: "test_002".to_string(), + }); + + let result = client.submit_order(request).await; + + assert!(result.is_err(), "Expected error for expired token"); + let status = result.unwrap_err(); + assert_eq!(status.code(), Code::Unauthenticated); + assert!( + status.message().contains("expired") || status.message().contains("invalid"), + "Error message should mention token expiration" + ); + + println!(" ✓ Expired token correctly rejected"); + Ok(()) +} + +#[tokio::test] +async fn test_submit_order_with_malformed_token_returns_unauthenticated() -> Result<()> { + println!("\n=== Test: Submit Order - Malformed Token (Unauthenticated) ==="); + + let channel = tonic::transport::Channel::from_static("http://localhost:50051") + .connect() + .await?; + + let malformed_token = "this_is_not_a_valid_jwt_token"; + + let mut client = TradingServiceClient::with_interceptor(channel, move |mut req: Request<()>| { + req.metadata_mut().insert( + "authorization", + format!("Bearer {}", malformed_token).parse().expect("INVARIANT: Valid parse input"), + ); + Ok(req) + }); + + let request = Request::new(SubmitOrderRequest { + symbol: "BTC/USD".to_string(), + side: 1, + order_type: 1, + quantity: 1.0, + price: None, + stop_price: None, + time_in_force: "GTC".to_string(), + client_order_id: "test_003".to_string(), + }); + + let result = client.submit_order(request).await; + + assert!(result.is_err(), "Expected error for malformed token"); + let status = result.unwrap_err(); + assert_eq!(status.code(), Code::Unauthenticated); + + println!(" ✓ Malformed token correctly rejected"); + Ok(()) +} + +// ============================================================================ +// INVALID ARGUMENT TESTS (Request Validation) +// ============================================================================ + +#[tokio::test] +async fn test_submit_order_empty_symbol_returns_invalid_argument() -> Result<()> { + println!("\n=== Test: Submit Order - Empty Symbol (InvalidArgument) ==="); + + let mut client = create_authenticated_client().await?; + + let request = Request::new(SubmitOrderRequest { + symbol: "".to_string(), // Invalid: empty symbol + side: 1, + order_type: 1, + quantity: 1.0, + price: None, + stop_price: None, + time_in_force: "GTC".to_string(), + client_order_id: "test_004".to_string(), + }); + + let result = client.submit_order(request).await; + + assert!(result.is_err(), "Expected error for empty symbol"); + let status = result.unwrap_err(); + assert_eq!(status.code(), Code::InvalidArgument); + + println!(" ✓ Empty symbol correctly rejected"); + Ok(()) +} + +#[tokio::test] +async fn test_get_order_status_empty_order_id_returns_invalid_argument() -> Result<()> { + println!("\n=== Test: Get Order Status - Empty Order ID (InvalidArgument) ==="); + + let mut client = create_authenticated_client().await?; + + let request = Request::new(GetOrderStatusRequest { + order_id: "".to_string(), // Invalid: empty order ID + }); + + let result = client.get_order_status(request).await; + + assert!(result.is_err(), "Expected error for empty order ID"); + let status = result.unwrap_err(); + assert_eq!(status.code(), Code::InvalidArgument); + + println!(" ✓ Empty order ID correctly rejected"); + Ok(()) +} + +// ============================================================================ +// RESOURCE EXHAUSTED TESTS (Rate Limiting) +// ============================================================================ + +#[tokio::test] +#[ignore = "Slow test - requires many requests to trigger rate limiting"] +async fn test_submit_order_rate_limit_returns_resource_exhausted() -> Result<()> { + println!("\n=== Test: Submit Order - Rate Limit (ResourceExhausted) ==="); + + let mut client = create_authenticated_client().await?; + + // Send many requests rapidly to trigger rate limiting + let mut rate_limited = false; + for i in 0..1000 { + let request = Request::new(SubmitOrderRequest { + symbol: "BTC/USD".to_string(), + side: 1, + order_type: 1, + quantity: 0.001, + price: None, + stop_price: None, + time_in_force: "IOC".to_string(), + client_order_id: format!("rate_limit_test_{}", i), + }); + + if let Err(status) = client.submit_order(request).await { + if status.code() == Code::ResourceExhausted { + println!(" ✓ Rate limit triggered after {} requests", i); + assert!( + status.message().contains("rate") + || status.message().contains("limit") + || status.message().contains("quota"), + "Error message should mention rate limiting" + ); + rate_limited = true; + break; + } + } + + // Small delay to avoid overwhelming the system + tokio::time::sleep(Duration::from_micros(100)).await; + } + + if !rate_limited { + println!(" ℹ Rate limit not triggered (may require higher request volume)"); + } + + Ok(()) +} + +// ============================================================================ +// PERMISSION DENIED TESTS (Authorization Failures) +// ============================================================================ + +#[tokio::test] +#[ignore = "Requires role-based access control configuration"] +async fn test_submit_order_insufficient_role_returns_permission_denied() -> Result<()> { + println!("\n=== Test: Submit Order - Insufficient Role (PermissionDenied) ==="); + + // Create token with viewer role (no trading permissions) + use chrono::{Duration as ChronoDuration, Utc}; + use jsonwebtoken::{encode, Algorithm, EncodingKey, Header}; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize)] + struct Claims { + sub: String, + exp: usize, + iat: usize, + iss: String, + aud: String, + roles: Vec, + permissions: Vec, + jti: String, + } + + let jwt_secret = std::env::var("JWT_SECRET") + .unwrap_or_else(|_| "OvFLDUbIDak3CSCi5t6zKfsAp65cjTOJ85q9YE+TFY8b361DGg1gSTra2rW6mps3cWrRGQ/NXRA5uftUpMldvOaEHMMgfBs4JjVODDElREdvUFm0EttD1A==".to_string()); + + let claims = Claims { + sub: "readonly_user".to_string(), + exp: (Utc::now() + ChronoDuration::hours(1)).timestamp() as usize, + iat: Utc::now().timestamp() as usize, + iss: "foxhunt-api-gateway".to_string(), + aud: "foxhunt-trading".to_string(), + roles: vec!["viewer".to_string()], // Read-only role + permissions: vec!["read:orders".to_string()], + jti: uuid::Uuid::new_v4().to_string(), + }; + + let token = encode( + &Header::new(Algorithm::HS256), + &claims, + &EncodingKey::from_secret(jwt_secret.as_bytes()), + )?; + + let channel = tonic::transport::Channel::from_static("http://localhost:50051") + .connect() + .await?; + + let mut client = TradingServiceClient::with_interceptor(channel, move |mut req: Request<()>| { + req.metadata_mut().insert( + "authorization", + format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"), + ); + Ok(req) + }); + + let request = Request::new(SubmitOrderRequest { + symbol: "BTC/USD".to_string(), + side: 1, + order_type: 1, + quantity: 1.0, + price: None, + stop_price: None, + time_in_force: "GTC".to_string(), + client_order_id: "test_005".to_string(), + }); + + let result = client.submit_order(request).await; + + if result.is_err() { + let status = result.unwrap_err(); + assert_eq!(status.code(), Code::PermissionDenied); + println!(" ✓ Insufficient role correctly rejected"); + } else { + println!(" ℹ Test requires RBAC configuration"); + } + + Ok(()) +} + +// ============================================================================ +// UNAVAILABLE TESTS (Backend Service Down) +// ============================================================================ + +#[tokio::test] +#[ignore = "Requires backend service to be stopped"] +async fn test_submit_order_backend_down_returns_unavailable() -> Result<()> { + println!("\n=== Test: Submit Order - Backend Unavailable (Unavailable) ==="); + + // This test requires the Trading Service to be stopped + // API Gateway should return Unavailable when backend is down + + let mut client = create_authenticated_client().await?; + + let request = Request::new(SubmitOrderRequest { + symbol: "BTC/USD".to_string(), + side: 1, + order_type: 1, + quantity: 1.0, + price: None, + stop_price: None, + time_in_force: "GTC".to_string(), + client_order_id: "test_006".to_string(), + }); + + let result = client.submit_order(request).await; + + if result.is_err() { + let status = result.unwrap_err(); + assert_eq!(status.code(), Code::Unavailable); + println!(" ✓ Backend unavailable correctly handled"); + } else { + println!(" ℹ Test requires backend service to be stopped"); + } + + Ok(()) +} + +// ============================================================================ +// DEADLINE EXCEEDED TESTS (Timeouts) +// ============================================================================ + +#[tokio::test] +async fn test_submit_order_with_short_timeout_may_fail() -> Result<()> { + println!("\n=== Test: Submit Order - Short Timeout (DeadlineExceeded) ==="); + + let channel = tonic::transport::Channel::from_static("http://localhost:50051") + .timeout(Duration::from_micros(1)) // Very short timeout + .connect() + .await?; + + let token = create_valid_jwt_token()?; + + let mut client = TradingServiceClient::with_interceptor(channel, move |mut req: Request<()>| { + req.metadata_mut().insert( + "authorization", + format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"), + ); + Ok(req) + }); + + let request = Request::new(SubmitOrderRequest { + symbol: "BTC/USD".to_string(), + side: 1, + order_type: 1, + quantity: 1.0, + price: None, + stop_price: None, + time_in_force: "GTC".to_string(), + client_order_id: format!("timeout_test_{}", uuid::Uuid::new_v4()), + }); + + let result = client.submit_order(request).await; + + if result.is_err() { + let status = result.unwrap_err(); + if status.code() == Code::DeadlineExceeded { + println!(" ✓ Request timed out as expected"); + } else { + println!(" ℹ Request failed with: {:?}", status.code()); + } + } else { + println!(" ℹ Request completed within deadline"); + } + + Ok(()) +} + +// ============================================================================ +// INTERNAL ERROR TESTS (Server Errors) +// ============================================================================ + +#[tokio::test] +#[ignore = "Requires configuration corruption simulation"] +async fn test_submit_order_internal_error_handling() -> Result<()> { + println!("\n=== Test: Submit Order - Internal Error (Internal) ==="); + + // This test would require simulating internal server errors + // such as database connection failures or corrupted configuration + + println!(" ℹ Test requires internal error simulation"); + Ok(()) +} + +// ============================================================================ +// NOT FOUND TESTS (Resource Not Found) +// ============================================================================ + +#[tokio::test] +async fn test_get_order_status_nonexistent_order_returns_not_found() -> Result<()> { + println!("\n=== Test: Get Order Status - Non-existent Order (NotFound) ==="); + + let mut client = create_authenticated_client().await?; + + let request = Request::new(GetOrderStatusRequest { + order_id: "nonexistent_order_999999".to_string(), + }); + + let result = client.get_order_status(request).await; + + assert!(result.is_err(), "Expected error for non-existent order"); + let status = result.unwrap_err(); + assert_eq!( + status.code(), + Code::NotFound, + "Expected NotFound error code" + ); + + println!(" ✓ Non-existent order correctly returns NotFound"); + Ok(()) +} + +#[tokio::test] +async fn test_cancel_order_nonexistent_order_returns_not_found() -> Result<()> { + println!("\n=== Test: Cancel Order - Non-existent Order (NotFound) ==="); + + let mut client = create_authenticated_client().await?; + + let request = Request::new(CancelOrderRequest { + order_id: "nonexistent_order_888888".to_string(), + symbol: "BTC/USD".to_string(), + }); + + let result = client.cancel_order(request).await; + + assert!(result.is_err(), "Expected error for non-existent order"); + let status = result.unwrap_err(); + assert_eq!(status.code(), Code::NotFound); + + println!(" ✓ Cancel non-existent order correctly returns NotFound"); + Ok(()) +} + +// ============================================================================ +// FAILED PRECONDITION TESTS (Service Prerequisites) +// ============================================================================ + +#[tokio::test] +#[ignore = "Requires MFA configuration"] +async fn test_submit_order_mfa_not_configured_returns_failed_precondition() -> Result<()> { + println!("\n=== Test: Submit Order - MFA Not Configured (FailedPrecondition) ==="); + + // Create token for user that requires MFA but hasn't configured it + // API Gateway should reject requests from such users + + println!(" ℹ Test requires MFA configuration logic"); + Ok(()) +} diff --git a/services/api/tests/grpc_error_handling_tests.rs b/services/api/tests/grpc_error_handling_tests.rs new file mode 100644 index 000000000..ddcfc701b --- /dev/null +++ b/services/api/tests/grpc_error_handling_tests.rs @@ -0,0 +1,24 @@ +//! gRPC Error Handling Integration Tests +//! +//! NOTE: These tests are currently disabled because they reference non-existent proxy types. +//! The api exports TradingServiceProxy, BacktestingServiceProxy, etc., but not a +//! generic ServiceProxy type as these tests assume. +//! +//! To enable these tests, either: +//! 1. Create the missing proxy abstractions +//! 2. Rewrite tests to use actual service-specific proxies +//! 3. Test error handling via deployed services + +#![cfg(test)] +#![allow(dead_code, unused_imports)] + +use anyhow::Result; +use tonic::{Code, Request, Status}; + +#[tokio::test] +#[ignore = "Missing proxy types - requires refactoring to use actual service proxies"] +async fn test_placeholder() -> Result<()> { + // Placeholder to keep test file valid + // Actual error handling tests should use real service proxies + Ok(()) +} diff --git a/services/api/tests/health_check_tests.rs b/services/api/tests/health_check_tests.rs new file mode 100644 index 000000000..a65b5a56f --- /dev/null +++ b/services/api/tests/health_check_tests.rs @@ -0,0 +1,672 @@ +//! Comprehensive health check tests for API Gateway +//! +//! Tests cover: +//! - HTTP /health/liveness, /health/readiness, /health/startup endpoints +//! - Backend service health checks +//! - Rate limiter status +//! - Circuit breaker status +//! - Service startup transitions +//! - Backend service cascade failures +//! - Health check latency +//! - Concurrent health checks +//! - Health during high load +//! - Resilience endpoint tests + +use axum::{ + body::Body, + http::{Request, StatusCode}, +}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::RwLock; +use tower::ServiceExt; + +/// Mock health state for API Gateway +#[derive(Clone)] +struct MockGatewayHealthState { + startup_complete: Arc>, + service_healthy: Arc>, + trading_service_up: Arc>, + backtesting_service_up: Arc>, + ml_service_up: Arc>, + rate_limiter_healthy: Arc>, + circuit_breaker_open: Arc>, +} + +impl MockGatewayHealthState { + fn new() -> Self { + Self { + startup_complete: Arc::new(RwLock::new(true)), + service_healthy: Arc::new(RwLock::new(true)), + trading_service_up: Arc::new(RwLock::new(true)), + backtesting_service_up: Arc::new(RwLock::new(true)), + ml_service_up: Arc::new(RwLock::new(true)), + rate_limiter_healthy: Arc::new(RwLock::new(true)), + circuit_breaker_open: Arc::new(RwLock::new(false)), + } + } + + async fn mark_startup_complete(&self) { + *self.startup_complete.write().await = true; + } + + async fn set_healthy(&self, healthy: bool) { + *self.service_healthy.write().await = healthy; + } + + async fn set_trading_service_up(&self, up: bool) { + *self.trading_service_up.write().await = up; + } + + async fn set_backtesting_service_up(&self, up: bool) { + *self.backtesting_service_up.write().await = up; + } + + async fn set_ml_service_up(&self, up: bool) { + *self.ml_service_up.write().await = up; + } + + async fn set_rate_limiter_healthy(&self, healthy: bool) { + *self.rate_limiter_healthy.write().await = healthy; + } + + async fn set_circuit_breaker_open(&self, open: bool) { + *self.circuit_breaker_open.write().await = open; + } + + async fn is_startup_complete(&self) -> bool { + *self.startup_complete.read().await + } + + async fn is_healthy(&self) -> bool { + *self.service_healthy.read().await + } + + async fn is_trading_service_up(&self) -> bool { + *self.trading_service_up.read().await + } + + async fn is_backtesting_service_up(&self) -> bool { + *self.backtesting_service_up.read().await + } + + async fn is_ml_service_up(&self) -> bool { + *self.ml_service_up.read().await + } + + async fn is_rate_limiter_healthy(&self) -> bool { + *self.rate_limiter_healthy.read().await + } + + async fn is_circuit_breaker_open(&self) -> bool { + *self.circuit_breaker_open.read().await + } +} + +/// Create health router for API Gateway +fn create_gateway_health_router(state: MockGatewayHealthState) -> axum::Router { + use axum::{extract::State, routing::get, Json, Router}; + use serde_json::json; + + async fn liveness_handler() -> &'static str { + "OK" + } + + async fn readiness_handler( + State(state): State, + ) -> Result { + if state.is_healthy().await { + Ok("READY".to_string()) + } else { + Err(StatusCode::SERVICE_UNAVAILABLE) + } + } + + async fn startup_handler( + State(state): State, + ) -> Result { + if state.is_startup_complete().await { + Ok("READY".to_string()) + } else { + Err(StatusCode::SERVICE_UNAVAILABLE) + } + } + + async fn circuit_breaker_status_handler( + State(state): State, + ) -> Json { + let is_open = state.is_circuit_breaker_open().await; + Json(json!({ + "state": if is_open { "open" } else { "closed" }, + "failure_count": if is_open { 5 } else { 0 }, + "success_count": 100, + "threshold": 5, + "timeout_seconds": 60 + })) + } + + async fn rate_limit_status_handler( + State(state): State, + ) -> Json { + let healthy = state.is_rate_limiter_healthy().await; + Json(json!({ + "enabled": true, + "requests_per_minute": 1000, + "current_usage": if healthy { 0 } else { 1000 }, + "burst_size": 100 + })) + } + + async fn timeout_config_handler() -> Json { + Json(json!({ + "request_timeout_ms": 5000, + "connection_timeout_ms": 3000, + "idle_timeout_ms": 60000 + })) + } + + async fn retry_config_handler() -> Json { + Json(json!({ + "max_retries": 3, + "retry_delay_ms": 100, + "backoff_multiplier": 2.0, + "max_backoff_ms": 10000 + })) + } + + async fn backend_services_handler( + State(state): State, + ) -> Json { + let trading_up = state.is_trading_service_up().await; + let backtesting_up = state.is_backtesting_service_up().await; + let ml_up = state.is_ml_service_up().await; + + Json(json!({ + "trading_service": if trading_up { "up" } else { "down" }, + "backtesting_service": if backtesting_up { "up" } else { "down" }, + "ml_training_service": if ml_up { "up" } else { "down" } + })) + } + + Router::new() + .route("/health/liveness", get(liveness_handler)) + .route("/health/readiness", get(readiness_handler)) + .route("/health/startup", get(startup_handler)) + .route( + "/resilience/circuit-breaker/status", + get(circuit_breaker_status_handler), + ) + .route( + "/resilience/rate-limit/status", + get(rate_limit_status_handler), + ) + .route("/resilience/timeout/config", get(timeout_config_handler)) + .route("/resilience/retry/config", get(retry_config_handler)) + .route( + "/health", + get(|| async { axum::Json(serde_json::json!({"status": "healthy"})) }), + ) + .route("/health/backends", get(backend_services_handler)) + .with_state(state) +} + +#[tokio::test] +async fn test_gateway_liveness_probe() { + let state = MockGatewayHealthState::new(); + let app = create_gateway_health_router(state.clone()); + + let response = app + .oneshot( + Request::builder() + .uri("/health/liveness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test] +async fn test_gateway_readiness_probe_healthy() { + let state = MockGatewayHealthState::new(); + state.set_healthy(true).await; + let app = create_gateway_health_router(state.clone()); + + let response = app + .oneshot( + Request::builder() + .uri("/health/readiness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test] +async fn test_gateway_readiness_probe_unhealthy() { + let state = MockGatewayHealthState::new(); + state.set_healthy(false).await; + let app = create_gateway_health_router(state.clone()); + + let response = app + .oneshot( + Request::builder() + .uri("/health/readiness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); +} + +#[tokio::test] +async fn test_gateway_startup_probe() { + let state = MockGatewayHealthState::new(); + state.mark_startup_complete().await; + let app = create_gateway_health_router(state.clone()); + + let response = app + .oneshot( + Request::builder() + .uri("/health/startup") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test] +async fn test_simple_health_endpoint() { + let state = MockGatewayHealthState::new(); + let app = create_gateway_health_router(state.clone()); + + let response = app + .oneshot( + Request::builder() + .uri("/health") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + + // Verify JSON response structure + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let json: serde_json::Value = serde_json::from_slice(&body).expect("INVARIANT: Deserialization should succeed for valid JSON"); + assert_eq!(json["status"], "healthy"); + assert_eq!(json.get("status").expect("INVARIANT: Key should exist in map"), "healthy"); +} + +#[tokio::test] +async fn test_gateway_service_startup_transition() { + let state = MockGatewayHealthState::new(); + *state.startup_complete.write().await = false; + + // Not ready initially + assert!(!state.is_startup_complete().await); + + // Simulate startup + tokio::time::sleep(Duration::from_millis(50)).await; + state.mark_startup_complete().await; + + assert!(state.is_startup_complete().await); +} + +#[tokio::test] +async fn test_gateway_circuit_breaker_status() { + let state = MockGatewayHealthState::new(); + let app = create_gateway_health_router(state.clone()); + + let response = app + .oneshot( + Request::builder() + .uri("/resilience/circuit-breaker/status") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test] +async fn test_gateway_circuit_breaker_open() { + let state = MockGatewayHealthState::new(); + state.set_circuit_breaker_open(true).await; + let app = create_gateway_health_router(state.clone()); + + let response = app + .oneshot( + Request::builder() + .uri("/resilience/circuit-breaker/status") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + // Circuit breaker open should still return 200 with state info +} + +#[tokio::test] +async fn test_gateway_rate_limit_status() { + let state = MockGatewayHealthState::new(); + let app = create_gateway_health_router(state.clone()); + + let response = app + .oneshot( + Request::builder() + .uri("/resilience/rate-limit/status") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test] +async fn test_gateway_timeout_config() { + let state = MockGatewayHealthState::new(); + let app = create_gateway_health_router(state.clone()); + + let response = app + .oneshot( + Request::builder() + .uri("/resilience/timeout/config") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test] +async fn test_gateway_retry_config() { + let state = MockGatewayHealthState::new(); + let app = create_gateway_health_router(state.clone()); + + let response = app + .oneshot( + Request::builder() + .uri("/resilience/retry/config") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test] +async fn test_gateway_backend_services_all_up() { + let state = MockGatewayHealthState::new(); + let app = create_gateway_health_router(state.clone()); + + let response = app + .oneshot( + Request::builder() + .uri("/health/backends") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test] +async fn test_gateway_trading_service_down() { + let state = MockGatewayHealthState::new(); + state.set_trading_service_up(false).await; + let app = create_gateway_health_router(state.clone()); + + let response = app + .oneshot( + Request::builder() + .uri("/health/backends") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + // Returns 200 with status info even if backend is down +} + +#[tokio::test] +async fn test_gateway_all_backends_down() { + let state = MockGatewayHealthState::new(); + state.set_trading_service_up(false).await; + state.set_backtesting_service_up(false).await; + state.set_ml_service_up(false).await; + + let app = create_gateway_health_router(state.clone()); + + let response = app + .oneshot( + Request::builder() + .uri("/health/backends") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + // Still returns 200 to report status +} + +#[tokio::test] +async fn test_gateway_health_check_latency() { + let state = MockGatewayHealthState::new(); + let app = create_gateway_health_router(state.clone()); + + let start = Instant::now(); + let response = app + .oneshot( + Request::builder() + .uri("/health/liveness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let latency = start.elapsed(); + + assert_eq!(response.status(), StatusCode::OK); + assert!( + latency < Duration::from_millis(100), + "Health check latency: {:?}", + latency + ); +} + +#[tokio::test] +async fn test_gateway_concurrent_health_checks() { + let state = MockGatewayHealthState::new(); + + let mut handles = vec![]; + for _ in 0..100 { + let state_clone = state.clone(); + let handle = tokio::spawn(async move { + let app = create_gateway_health_router(state_clone); + let response = app + .oneshot( + Request::builder() + .uri("/health/liveness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + }); + handles.push(handle); + } + + for handle in handles { + handle.await.unwrap(); + } +} + +#[tokio::test] +async fn test_gateway_health_during_shutdown() { + let state = MockGatewayHealthState::new(); + + // Graceful shutdown + state.set_healthy(false).await; + let app = create_gateway_health_router(state.clone()); + + // Readiness check fails + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/health/readiness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + + // Liveness check still passes + let response = app + .oneshot( + Request::builder() + .uri("/health/liveness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test] +async fn test_gateway_rapid_health_checks() { + let state = MockGatewayHealthState::new(); + + let start = Instant::now(); + for _ in 0..1000 { + let app = create_gateway_health_router(state.clone()); + let response = app + .oneshot( + Request::builder() + .uri("/health/liveness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + } + let duration = start.elapsed(); + + assert!( + duration < Duration::from_secs(1), + "1000 health checks took: {:?}", + duration + ); +} + +#[tokio::test] +async fn test_gateway_partial_backend_failure() { + let state = MockGatewayHealthState::new(); + + // Only trading service down + state.set_trading_service_up(false).await; + state.set_backtesting_service_up(true).await; + state.set_ml_service_up(true).await; + + let app = create_gateway_health_router(state.clone()); + + let response = app + .oneshot( + Request::builder() + .uri("/health/backends") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test] +async fn test_gateway_recovery_after_failure() { + let state = MockGatewayHealthState::new(); + + // Service fails + state.set_healthy(false).await; + let app = create_gateway_health_router(state.clone()); + + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/health/readiness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + + // Service recovers + state.set_healthy(true).await; + let response = app + .oneshot( + Request::builder() + .uri("/health/readiness") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); +} + +#[tokio::test] +async fn test_gateway_rate_limiter_unhealthy() { + let state = MockGatewayHealthState::new(); + state.set_rate_limiter_healthy(false).await; + let app = create_gateway_health_router(state.clone()); + + let response = app + .oneshot( + Request::builder() + .uri("/resilience/rate-limit/status") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + // Still returns 200 with status info + assert_eq!(response.status(), StatusCode::OK); +} diff --git a/services/api/tests/integration_tests.rs b/services/api/tests/integration_tests.rs new file mode 100644 index 000000000..98948aeb3 --- /dev/null +++ b/services/api/tests/integration_tests.rs @@ -0,0 +1,13 @@ +//! Integration Tests Main Harness +//! +//! This file serves as the entry point for all integration tests. +//! Run with: cargo test --test integration_tests + +mod common; + +// Re-export test modules +mod auth_flow_tests; +mod rate_limiting_tests; +mod service_proxy_tests; + +// Additional integration test utilities can be added here diff --git a/services/api/tests/jwt_service_edge_cases.rs b/services/api/tests/jwt_service_edge_cases.rs new file mode 100644 index 000000000..f76b971e0 --- /dev/null +++ b/services/api/tests/jwt_service_edge_cases.rs @@ -0,0 +1,662 @@ +//! JWT Service Edge Case Tests - Wave 17 Agent 17.10 +//! +//! Comprehensive tests for JWT token validation, secret validation, +//! and revocation checking with 100% edge case coverage. +//! +//! Coverage targets: +//! - JWT secret validation (entropy, length, patterns) +//! - Token validation edge cases (empty, too long, corrupted) +//! - Revocation service integration +//! - Configuration loading and error handling + +use anyhow::Result; +use std::time::{SystemTime, UNIX_EPOCH}; +use uuid::Uuid; + +use api::auth::jwt::{JwtClaims, JwtConfig, JwtService}; +use api::auth::{Jti, RevocationService}; + +// ============================================================================ +// JWT Secret Validation Tests (10 tests) +// ============================================================================ + +#[tokio::test] +async fn test_jwt_secret_too_short() { + std::env::remove_var("JWT_SECRET"); + std::env::remove_var("JWT_SECRET_FILE"); + + // Set a short secret (less than 64 chars) + std::env::set_var("JWT_SECRET", "short_secret_32chars_only!!!!!"); + + let result = JwtConfig::new().await; + + // Should fail due to insufficient length + // Note: Current implementation relaxes validation for dev secrets + // This test documents expected behavior + println!("Short secret result: {:?}", result.is_ok()); + + std::env::remove_var("JWT_SECRET"); +} + +#[tokio::test] +async fn test_jwt_secret_no_uppercase() { + std::env::remove_var("JWT_SECRET"); + std::env::remove_var("JWT_SECRET_FILE"); + + // 64 char secret with no uppercase + let weak_secret = "abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+-=[]{}|;:',.<>"; + assert!(weak_secret.len() >= 64, "Secret must be 64+ chars for test"); + + std::env::set_var("JWT_SECRET", weak_secret); + + let result = JwtConfig::new().await; + + // Should pass with warning (relaxed validation for dev) + println!("No uppercase secret result: {:?}", result.is_ok()); + + std::env::remove_var("JWT_SECRET"); +} + +#[tokio::test] +async fn test_jwt_secret_no_lowercase() { + std::env::remove_var("JWT_SECRET"); + std::env::remove_var("JWT_SECRET_FILE"); + + // 64 char secret with no lowercase + let weak_secret = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*()_+-=[]{}|;:',.<>"; + assert!(weak_secret.len() >= 64); + + std::env::set_var("JWT_SECRET", weak_secret); + + let result = JwtConfig::new().await; + println!("No lowercase secret result: {:?}", result.is_ok()); + + std::env::remove_var("JWT_SECRET"); +} + +#[tokio::test] +async fn test_jwt_secret_no_digits() { + std::env::remove_var("JWT_SECRET"); + std::env::remove_var("JWT_SECRET_FILE"); + + // 64 char secret with no digits + let weak_secret = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!@#$%^&*()_"; + assert!(weak_secret.len() >= 64); + + std::env::set_var("JWT_SECRET", weak_secret); + + let result = JwtConfig::new().await; + println!("No digits secret result: {:?}", result.is_ok()); + + std::env::remove_var("JWT_SECRET"); +} + +#[tokio::test] +async fn test_jwt_secret_no_symbols() { + std::env::remove_var("JWT_SECRET"); + std::env::remove_var("JWT_SECRET_FILE"); + + // 64 char secret with no symbols + let weak_secret = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + assert!(weak_secret.len() >= 64); + + std::env::set_var("JWT_SECRET", weak_secret); + + let result = JwtConfig::new().await; + println!("No symbols secret result: {:?}", result.is_ok()); + + std::env::remove_var("JWT_SECRET"); +} + +#[tokio::test] +async fn test_jwt_secret_repeated_characters() { + std::env::remove_var("JWT_SECRET"); + std::env::remove_var("JWT_SECRET_FILE"); + + // 64 char secret with 4+ repeated characters + let weak_secret = "AAAABCDefgh1234!@#$AAAABCDefgh1234!@#$AAAABCDefgh1234!@#$AAAA"; + assert!(weak_secret.len() >= 64); + + std::env::set_var("JWT_SECRET", weak_secret); + + let result = JwtConfig::new().await; + println!("Repeated characters secret result: {:?}", result.is_ok()); + + std::env::remove_var("JWT_SECRET"); +} + +#[tokio::test] +async fn test_jwt_secret_sequential_pattern() { + std::env::remove_var("JWT_SECRET"); + std::env::remove_var("JWT_SECRET_FILE"); + + // 64 char secret with sequential pattern + let weak_secret = "1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!@"; + assert!(weak_secret.len() >= 64); + + std::env::set_var("JWT_SECRET", weak_secret); + + let result = JwtConfig::new().await; + println!("Sequential pattern secret result: {:?}", result.is_ok()); + + std::env::remove_var("JWT_SECRET"); +} + +#[tokio::test] +async fn test_jwt_secret_common_weak_patterns() { + std::env::remove_var("JWT_SECRET"); + std::env::remove_var("JWT_SECRET_FILE"); + + // Test each weak pattern + let weak_patterns = vec![ + ( + "password", + "PASSWORDabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+-=[]{}", + ), + ( + "admin", + "ADMINabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+-=[]{}|;:", + ), + ( + "1234", + "1234ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!@#$%^&", + ), + ]; + + for (pattern_name, secret) in weak_patterns { + assert!(secret.len() >= 64, "Secret must be 64+ chars"); + std::env::set_var("JWT_SECRET", secret); + + let result = JwtConfig::new().await; + println!( + "Weak pattern '{}' result: {:?}", + pattern_name, + result.is_ok() + ); + + std::env::remove_var("JWT_SECRET"); + } +} + +#[tokio::test] +async fn test_jwt_secret_excessively_long() { + std::env::remove_var("JWT_SECRET"); + std::env::remove_var("JWT_SECRET_FILE"); + + // 2000 char secret (exceeds 1024 max) + let long_secret = "A".repeat(2000); + std::env::set_var("JWT_SECRET", &long_secret); + + let result = JwtConfig::new().await; + + // Should pass with relaxed validation (truncated or accepted) + println!("Excessively long secret result: {:?}", result.is_ok()); + + std::env::remove_var("JWT_SECRET"); +} + +#[tokio::test] +async fn test_jwt_secret_whitespace_handling() { + std::env::remove_var("JWT_SECRET"); + std::env::remove_var("JWT_SECRET_FILE"); + + // Secret with leading/trailing whitespace + let secret_with_whitespace = + " Kx7mP@9nR!2sW#5vY$8bC&3fG*6jH^1kL%4pQ+7tZ-0uN~9dM=5eV(8xS)2wT!6yA#4zB "; + + std::env::set_var("JWT_SECRET", secret_with_whitespace); + + let config = JwtConfig::new().await.expect("Should trim whitespace"); + + // Whitespace should be trimmed + assert_eq!(config.jwt_secret.trim(), config.jwt_secret); + + std::env::remove_var("JWT_SECRET"); +} + +// ============================================================================ +// Token Validation Edge Cases (10 tests) +// ============================================================================ + +#[tokio::test] +async fn test_validate_empty_token() { + let secret = + "Kx7mP@9nR!2sW#5vY$8bC&3fG*6jH^1kL%4pQ+7tZ-0uN~9dM=5eV(8xS)2wT!6yA#4zB".to_string(); + let config = JwtConfig { + jwt_secret: secret, + jwt_issuer: "test-issuer".to_string(), + jwt_audience: "test-audience".to_string(), + }; + let jwt_service = JwtService::new(config); + + let result = jwt_service.validate_token("").await; + + assert!(result.is_err(), "Empty token should be rejected"); + assert!(result.unwrap_err().to_string().contains("empty")); +} + +#[tokio::test] +async fn test_validate_token_exceeds_max_length() { + let secret = + "Kx7mP@9nR!2sW#5vY$8bC&3fG*6jH^1kL%4pQ+7tZ-0uN~9dM=5eV(8xS)2wT!6yA#4zB".to_string(); + let config = JwtConfig { + jwt_secret: secret, + jwt_issuer: "test-issuer".to_string(), + jwt_audience: "test-audience".to_string(), + }; + let jwt_service = JwtService::new(config); + + // Create an 8200 char token (exceeds 8192 max) + let long_token = "a".repeat(8200); + + let result = jwt_service.validate_token(&long_token).await; + + assert!(result.is_err(), "Token >8192 chars should be rejected"); + let error_msg = result.unwrap_err().to_string(); + assert!( + error_msg.contains("too long") + || error_msg.contains("attack") + ); +} + +#[tokio::test] +async fn test_validate_token_with_invalid_base64() { + let secret = + "Kx7mP@9nR!2sW#5vY$8bC&3fG*6jH^1kL%4pQ+7tZ-0uN~9dM=5eV(8xS)2wT!6yA#4zB".to_string(); + let config = JwtConfig { + jwt_secret: secret, + jwt_issuer: "test-issuer".to_string(), + jwt_audience: "test-audience".to_string(), + }; + let jwt_service = JwtService::new(config); + + // JWT with invalid base64 in payload + let invalid_token = "eyJhbGciOiJIUzI1NiJ9.!!!INVALID!!!.signature"; + + let result = jwt_service.validate_token(invalid_token).await; + + assert!( + result.is_err(), + "Token with invalid base64 should be rejected" + ); +} + +#[tokio::test] +async fn test_validate_token_with_empty_jti() -> Result<()> { + let secret = + "Kx7mP@9nR!2sW#5vY$8bC&3fG*6jH^1kL%4pQ+7tZ-0uN~9dM=5eV(8xS)2wT!6yA#4zB".to_string(); + let config = JwtConfig { + jwt_secret: secret.clone(), + jwt_issuer: "test-issuer".to_string(), + jwt_audience: "test-audience".to_string(), + }; + let jwt_service = JwtService::new(config); + + use jsonwebtoken::{encode, EncodingKey, Header}; + + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + let claims = JwtClaims { + jti: "".to_string(), // Empty JTI + sub: "test_user".to_string(), + iat: now, + exp: now + 3600, + iss: "test-issuer".to_string(), + aud: "test-audience".to_string(), + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(secret.as_bytes()), + )?; + + let result = jwt_service.validate_token(&token).await; + + assert!(result.is_err(), "Token with empty JTI should be rejected"); + assert!(result.unwrap_err().to_string().contains("jti")); + + Ok(()) +} + +#[tokio::test] +async fn test_validate_token_with_empty_subject() -> Result<()> { + let secret = + "Kx7mP@9nR!2sW#5vY$8bC&3fG*6jH^1kL%4pQ+7tZ-0uN~9dM=5eV(8xS)2wT!6yA#4zB".to_string(); + let config = JwtConfig { + jwt_secret: secret.clone(), + jwt_issuer: "test-issuer".to_string(), + jwt_audience: "test-audience".to_string(), + }; + let jwt_service = JwtService::new(config); + + use jsonwebtoken::{encode, EncodingKey, Header}; + + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + let claims = JwtClaims { + jti: Uuid::new_v4().to_string(), + sub: "".to_string(), // Empty subject + iat: now, + exp: now + 3600, + iss: "test-issuer".to_string(), + aud: "test-audience".to_string(), + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(secret.as_bytes()), + )?; + + let result = jwt_service.validate_token(&token).await; + + assert!( + result.is_err(), + "Token with empty subject should be rejected" + ); + assert!(result.unwrap_err().to_string().contains("subject")); + + Ok(()) +} + +#[tokio::test] +async fn test_validate_token_with_empty_roles() -> Result<()> { + let secret = + "Kx7mP@9nR!2sW#5vY$8bC&3fG*6jH^1kL%4pQ+7tZ-0uN~9dM=5eV(8xS)2wT!6yA#4zB".to_string(); + let config = JwtConfig { + jwt_secret: secret.clone(), + jwt_issuer: "test-issuer".to_string(), + jwt_audience: "test-audience".to_string(), + }; + let jwt_service = JwtService::new(config); + + use jsonwebtoken::{encode, EncodingKey, Header}; + + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + let claims = JwtClaims { + jti: Uuid::new_v4().to_string(), + sub: "test_user".to_string(), + iat: now, + exp: now + 3600, + iss: "test-issuer".to_string(), + aud: "test-audience".to_string(), + roles: vec![], // Empty roles + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(secret.as_bytes()), + )?; + + let result = jwt_service.validate_token(&token).await; + + assert!(result.is_err(), "Token with empty roles should be rejected"); + assert!(result.unwrap_err().to_string().contains("role")); + + Ok(()) +} + +#[tokio::test] +async fn test_validate_token_with_future_iat() -> Result<()> { + let secret = + "Kx7mP@9nR!2sW#5vY$8bC&3fG*6jH^1kL%4pQ+7tZ-0uN~9dM=5eV(8xS)2wT!6yA#4zB".to_string(); + let config = JwtConfig { + jwt_secret: secret.clone(), + jwt_issuer: "test-issuer".to_string(), + jwt_audience: "test-audience".to_string(), + }; + let jwt_service = JwtService::new(config); + + use jsonwebtoken::{encode, EncodingKey, Header}; + + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + let claims = JwtClaims { + jti: Uuid::new_v4().to_string(), + sub: "test_user".to_string(), + iat: now + 7200, // Issued 2 hours in the future + exp: now + 10800, + iss: "test-issuer".to_string(), + aud: "test-audience".to_string(), + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(secret.as_bytes()), + )?; + + let result = jwt_service.validate_token(&token).await; + + assert!(result.is_err(), "Token with future iat should be rejected"); + assert!(result.unwrap_err().to_string().contains("future")); + + Ok(()) +} + +#[tokio::test] +async fn test_validate_token_too_old() -> Result<()> { + let secret = + "Kx7mP@9nR!2sW#5vY$8bC&3fG*6jH^1kL%4pQ+7tZ-0uN~9dM=5eV(8xS)2wT!6yA#4zB".to_string(); + let config = JwtConfig { + jwt_secret: secret.clone(), + jwt_issuer: "test-issuer".to_string(), + jwt_audience: "test-audience".to_string(), + }; + let jwt_service = JwtService::new(config); + + use jsonwebtoken::{encode, EncodingKey, Header}; + + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + let claims = JwtClaims { + jti: Uuid::new_v4().to_string(), + sub: "test_user".to_string(), + iat: now - 7200, // Issued 2 hours ago (max age is 1 hour) + exp: now + 3600, // Still valid + iss: "test-issuer".to_string(), + aud: "test-audience".to_string(), + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(secret.as_bytes()), + )?; + + let result = jwt_service.validate_token(&token).await; + + assert!(result.is_err(), "Token >1 hour old should be rejected"); + let error_msg = result.unwrap_err().to_string(); + assert!( + error_msg.contains("too old") + || error_msg.contains("Token age") + ); + + Ok(()) +} + +#[tokio::test] +async fn test_validate_token_already_expired() -> Result<()> { + let secret = + "Kx7mP@9nR!2sW#5vY$8bC&3fG*6jH^1kL%4pQ+7tZ-0uN~9dM=5eV(8xS)2wT!6yA#4zB".to_string(); + let config = JwtConfig { + jwt_secret: secret.clone(), + jwt_issuer: "test-issuer".to_string(), + jwt_audience: "test-audience".to_string(), + }; + let jwt_service = JwtService::new(config); + + use jsonwebtoken::{encode, EncodingKey, Header}; + + let now = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + + let claims = JwtClaims { + jti: Uuid::new_v4().to_string(), + sub: "test_user".to_string(), + iat: now - 7200, + exp: now - 3600, // Expired 1 hour ago + iss: "test-issuer".to_string(), + aud: "test-audience".to_string(), + roles: vec!["trader".to_string()], + permissions: vec!["api.access".to_string()], + token_type: "access".to_string(), + session_id: Some(Uuid::new_v4().to_string()), + }; + + let token = encode( + &Header::default(), + &claims, + &EncodingKey::from_secret(secret.as_bytes()), + )?; + + let result = jwt_service.validate_token(&token).await; + + assert!(result.is_err(), "Expired token should be rejected"); + assert!(result.unwrap_err().to_string().contains("expired")); + + Ok(()) +} + +#[tokio::test] +async fn test_validate_token_wrong_algorithm() { + let secret = + "Kx7mP@9nR!2sW#5vY$8bC&3fG*6jH^1kL%4pQ+7tZ-0uN~9dM=5eV(8xS)2wT!6yA#4zB".to_string(); + let config = JwtConfig { + jwt_secret: secret.clone(), + jwt_issuer: "test-issuer".to_string(), + jwt_audience: "test-audience".to_string(), + }; + let jwt_service = JwtService::new(config); + + // Token signed with RS256 instead of HS256 + let token_rs256 = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMn0.NHVaYe26MbtOYhSKkoKYdFVomg4i8ZJd8_-RU8VNbftc4TSMb4bXP3l3YlNWACwyXPGffz5aXHc6lty1Y2t4SWRqGteragsVdZufDn5BlnJl9pdR_kdVFUsra2rWKEofkZeIC4yWytE58sMIihvo9H1ScmmVwBcQP6XETqYd0aSHp1gOa9RdUPDvoXQ5oqygTqVtxaDr6wUFKrKItgBMzWIdNZ6y7O9E0DhEPTbE9rfBo6KTFsHAZnMg4k68CDp2woYIaXbmYTWcvbzIuHO7_37GT79XdIwkm95QJ7hYC9RiwrV7mesbY4PAahERJawntho0my942XheVLmGwLMBkQ"; + + let result = jwt_service.validate_token(token_rs256).await; + + assert!( + result.is_err(), + "Token with wrong algorithm should be rejected" + ); +} + +// ============================================================================ +// Revocation Service Tests (5 tests) +// ============================================================================ + +#[tokio::test] +async fn test_revoke_already_expired_token() -> Result<()> { + let revocation_service = RevocationService::new("redis://localhost:6379").await?; + + let jti = Jti::new(); + + // Revoke with 0 TTL (already expired) + let result = revocation_service.revoke_token(&jti, 0).await; + + // Should succeed (no-op for expired token) + assert!(result.is_ok()); + + Ok(()) +} + +#[tokio::test] +async fn test_check_revocation_nonexistent_token() -> Result<()> { + let revocation_service = RevocationService::new("redis://localhost:6379").await?; + + let jti = Jti::new(); + + // Check revocation for token that was never revoked + let is_revoked = revocation_service.is_revoked(&jti).await?; + + assert!(!is_revoked, "Nonexistent token should not be revoked"); + + Ok(()) +} + +#[tokio::test] +async fn test_revoke_and_check_token() -> Result<()> { + let revocation_service = RevocationService::new("redis://localhost:6379").await?; + + let jti = Jti::new(); + + // Revoke token + revocation_service.revoke_token(&jti, 300).await?; + + // Check revocation + let is_revoked = revocation_service.is_revoked(&jti).await?; + + assert!(is_revoked, "Revoked token should be marked as revoked"); + + Ok(()) +} + +#[tokio::test] +async fn test_cache_stats_after_operations() -> Result<()> { + let revocation_service = RevocationService::new("redis://localhost:6379").await?; + + // Reset stats + revocation_service.reset_cache_stats(); + + let jti1 = Jti::new(); + let jti2 = Jti::new(); + + // First check (cache miss) + let _ = revocation_service.is_revoked(&jti1).await?; + + // Second check (cache hit) + let _ = revocation_service.is_revoked(&jti1).await?; + + // Third check different token (cache miss) + let _ = revocation_service.is_revoked(&jti2).await?; + + let stats = revocation_service.cache_stats(); + + println!("Cache stats: {:?}", stats); + assert!(stats.total >= 3, "Should have at least 3 checks"); + assert!(stats.hits >= 1, "Should have at least 1 cache hit"); + assert!(stats.misses >= 2, "Should have at least 2 cache misses"); + + Ok(()) +} + +#[tokio::test] +async fn test_clear_cache() -> Result<()> { + let revocation_service = RevocationService::new("redis://localhost:6379").await?; + + let jti = Jti::new(); + + // Make a check to populate cache + let _ = revocation_service.is_revoked(&jti).await?; + + // Clear cache + revocation_service.clear_cache(); + + // Next check should be a cache miss + let _ = revocation_service.is_revoked(&jti).await?; + + println!("Cache cleared successfully"); + + Ok(()) +} diff --git a/services/api/tests/metrics_integration_test.rs b/services/api/tests/metrics_integration_test.rs new file mode 100644 index 000000000..7f58b3a6a --- /dev/null +++ b/services/api/tests/metrics_integration_test.rs @@ -0,0 +1,298 @@ +//! Integration tests for API Gateway metrics system +//! +//! Verifies that all metrics are correctly registered and recorded + +use api::metrics::{AuthMetrics, ConfigMetrics, GatewayMetrics, ProxyMetrics}; +use prometheus::Registry; +use std::sync::Arc; + +#[test] +fn test_gateway_metrics_initialization() { + let metrics = GatewayMetrics::new().expect("Failed to create metrics"); + + // Verify all subsystems initialized + assert!(Arc::strong_count(&metrics.auth) >= 1); + assert!(Arc::strong_count(&metrics.proxy) >= 1); + assert!(Arc::strong_count(&metrics.config) >= 1); + assert!(Arc::strong_count(&metrics.registry) >= 1); +} + +#[test] +fn test_auth_metrics_registration() { + let registry = Registry::new(); + let auth_metrics = AuthMetrics::new(®istry).expect("Failed to create auth metrics"); + + // Record some events + auth_metrics.auth_requests_total.inc(); + auth_metrics.record_success(5.5); + auth_metrics.record_failure("expired_jwt", Some("test_user")); + auth_metrics.record_user_request("test_user"); + auth_metrics.record_rate_limit("rate_limited_user"); + + // Verify metrics can be gathered + let metrics = registry.gather(); + assert!(!metrics.is_empty(), "No metrics gathered"); + + // Check specific metrics exist + let metric_names: Vec = metrics.iter().map(|m| m.name().to_string()).collect(); + + assert!( + metric_names.contains(&"api_auth_requests_total".to_string()), + "Missing auth_requests_total metric" + ); + assert!( + metric_names.contains(&"api_auth_requests_success".to_string()), + "Missing auth_requests_success metric" + ); + assert!( + metric_names.contains(&"api_auth_sla_met".to_string()), + "Missing auth_sla_met metric" + ); +} + +#[test] +fn test_proxy_metrics_registration() { + let registry = Registry::new(); + let proxy_metrics = ProxyMetrics::new(®istry).expect("Failed to create proxy metrics"); + + // Record backend events + proxy_metrics.record_backend_success("trading", "ExecuteTrade", 15.5); + proxy_metrics.record_backend_failure("backtesting", "timeout"); + proxy_metrics.update_circuit_breaker_state("trading", 0); // closed + proxy_metrics.update_health_status("trading", true); + proxy_metrics.update_connection_pool("trading", 10, 5, 50); + + // Verify metrics can be gathered + let metrics = registry.gather(); + assert!(!metrics.is_empty(), "No metrics gathered"); + + let metric_names: Vec = metrics.iter().map(|m| m.name().to_string()).collect(); + + assert!( + metric_names.contains(&"api_backend_requests_total".to_string()), + "Missing backend_requests_total metric" + ); + assert!( + metric_names.contains(&"api_circuit_breaker_state".to_string()), + "Missing circuit_breaker_state metric" + ); + assert!( + metric_names.contains(&"api_health_status".to_string()), + "Missing health_status metric" + ); +} + +#[test] +fn test_config_metrics_registration() { + let registry = Registry::new(); + let config_metrics = ConfigMetrics::new(®istry).expect("Failed to create config metrics"); + + // Record config events + config_metrics.record_notify_event(true); + config_metrics.record_config_reload("auth", 25.0); + config_metrics.update_listener_status(true); + config_metrics.record_reconnection(); + + // Verify metrics can be gathered + let metrics = registry.gather(); + assert!(!metrics.is_empty(), "No metrics gathered"); + + let metric_names: Vec = metrics.iter().map(|m| m.name().to_string()).collect(); + + assert!( + metric_names.contains(&"api_notify_events_total".to_string()), + "Missing notify_events_total metric" + ); + assert!( + metric_names.contains(&"api_config_reload_duration_milliseconds".to_string()), + "Missing config_reload_duration metric" + ); + assert!( + metric_names.contains(&"api_notify_listener_connected".to_string()), + "Missing notify_listener_connected metric" + ); +} + +#[test] +fn test_auth_sla_tracking() { + let registry = Registry::new(); + let auth_metrics = AuthMetrics::new(®istry).expect("Failed to create auth metrics"); + + // Record requests meeting SLA (<10μs) + auth_metrics.record_success(5.0); + auth_metrics.record_success(8.5); + auth_metrics.record_success(9.9); + + // Record requests exceeding SLA (>10μs) + auth_metrics.record_success(15.0); + auth_metrics.record_success(25.5); + + let metrics = registry.gather(); + let sla_met = metrics + .iter() + .find(|m| m.name() == "api_auth_sla_met") + .expect("SLA met metric not found"); + + let sla_exceeded = metrics + .iter() + .find(|m| m.name() == "api_auth_sla_exceeded") + .expect("SLA exceeded metric not found"); + + // Verify counts (3 met, 2 exceeded) + assert_eq!(sla_met.metric[0].counter.value, Some(3.0)); + assert_eq!(sla_exceeded.metric[0].counter.value, Some(2.0)); +} + +#[test] +fn test_cache_hit_rate_tracking() { + let registry = Registry::new(); + let auth_metrics = AuthMetrics::new(®istry).expect("Failed to create auth metrics"); + + // Simulate 95% cache hit rate + auth_metrics.jwt_cache_hits.inc_by(95.0); + auth_metrics.jwt_cache_misses.inc_by(5.0); + auth_metrics.rbac_cache_hits.inc_by(98.0); + auth_metrics.rbac_cache_misses.inc_by(2.0); + + let metrics = registry.gather(); + + let jwt_hits = metrics + .iter() + .find(|m| m.name() == "api_jwt_cache_hits") + .expect("JWT cache hits not found"); + + let jwt_misses = metrics + .iter() + .find(|m| m.name() == "api_jwt_cache_misses") + .expect("JWT cache misses not found"); + + assert_eq!(jwt_hits.metric[0].counter.value, Some(95.0)); + assert_eq!(jwt_misses.metric[0].counter.value, Some(5.0)); + + // Verify hit rate calculation + let hit_rate = 100.0 * 95.0 / (95.0 + 5.0); + assert_eq!(hit_rate, 95.0, "JWT cache hit rate should be 95%"); +} + +#[test] +fn test_circuit_breaker_state_transitions() { + let registry = Registry::new(); + let proxy_metrics = ProxyMetrics::new(®istry).expect("Failed to create proxy metrics"); + + // Initial state: closed (0) + proxy_metrics.update_circuit_breaker_state("trading", 0); + + // Trip circuit breaker (open = 2) + proxy_metrics.record_circuit_breaker_trip("trading"); + + let metrics = registry.gather(); + let cb_state = metrics + .iter() + .find(|m| m.name() == "api_circuit_breaker_state") + .expect("Circuit breaker state not found"); + + // Verify state is open (2) + let trading_state = cb_state + .metric + .iter() + .find(|m| { + m.label + .iter() + .any(|l| l.name() == "service" && l.value() == "trading") + }) + .expect("Trading service state not found"); + + assert_eq!(trading_state.gauge.value, Some(2.0)); + + // Recover circuit breaker (closed = 0) + proxy_metrics.record_circuit_breaker_recovery("trading"); + + let metrics = registry.gather(); + let cb_state = metrics + .iter() + .find(|m| m.name() == "api_circuit_breaker_state") + .expect("Circuit breaker state not found"); + + let trading_state = cb_state + .metric + .iter() + .find(|m| { + m.label + .iter() + .any(|l| l.name() == "service" && l.value() == "trading") + }) + .expect("Trading service state not found"); + + assert_eq!(trading_state.gauge.value, Some(0.0)); +} + +#[test] +fn test_per_user_metrics() { + let registry = Registry::new(); + let auth_metrics = AuthMetrics::new(®istry).expect("Failed to create auth metrics"); + + // Record requests for different users + auth_metrics.record_user_request("user_1"); + auth_metrics.record_user_request("user_1"); + auth_metrics.record_user_request("user_2"); + + // Record failures per user + auth_metrics.record_failure("expired_jwt", Some("user_1")); + auth_metrics.record_failure("permission_denied", Some("user_2")); + + // Record rate limits + auth_metrics.record_rate_limit("user_3"); + auth_metrics.record_rate_limit("user_3"); + + let metrics = registry.gather(); + + // Verify per-user request counts + let requests_by_user = metrics + .iter() + .find(|m| m.name() == "api_requests_by_user") + .expect("Requests by user not found"); + + assert_eq!(requests_by_user.metric.len(), 2); // user_1 and user_2 + + // Verify per-user rate limits + let rate_limits_by_user = metrics + .iter() + .find(|m| m.name() == "api_rate_limits_by_user") + .expect("Rate limits by user not found"); + + let user_3_limits = rate_limits_by_user + .metric + .iter() + .find(|m| { + m.label + .iter() + .any(|l| l.name() == "user_id" && l.value() == "user_3") + }) + .expect("User 3 rate limits not found"); + + assert_eq!(user_3_limits.counter.value, Some(2.0)); +} + +#[test] +fn test_metrics_exporter_format() { + use api::metrics::PrometheusExporter; + + let metrics = GatewayMetrics::new().expect("Failed to create metrics"); + + // Record some events + metrics.auth.record_success(5.0); + metrics + .proxy + .record_backend_success("trading", "ExecuteTrade", 15.0); + metrics.config.record_notify_event(true); + + let exporter = PrometheusExporter::new(metrics.registry()); + let output = exporter.gather_metrics().expect("Failed to gather metrics"); + + // Verify Prometheus text format + assert!(output.contains("# HELP")); + assert!(output.contains("# TYPE")); + assert!(output.contains("api_auth_requests_success")); + assert!(output.contains("api_backend_requests_success")); + assert!(output.contains("api_notify_events_success")); +} diff --git a/services/api/tests/mfa_comprehensive.rs b/services/api/tests/mfa_comprehensive.rs new file mode 100644 index 000000000..5f1b40c4e --- /dev/null +++ b/services/api/tests/mfa_comprehensive.rs @@ -0,0 +1,1300 @@ +#![cfg(feature = "mfa")] +//! Comprehensive MFA (Multi-Factor Authentication) Tests +//! +//! Coverage: TOTP (RFC 6238), Backup Codes, Enrollment, Verification, Security +//! Test Count: 55 tests targeting 95%+ coverage +//! +//! Test Categories: +//! 1. TOTP Advanced (15 tests) - Replay attacks, drift tolerance, time boundaries +//! 2. Backup Code Security (12 tests) - One-time use, exhaustion, format validation +//! 3. Enrollment Flow (10 tests) - State machine, session expiration, max attempts +//! 4. Verification Flow (10 tests) - Account lockout, metadata, mixed methods +//! 5. Integration & Security (8 tests) - End-to-end, lifecycle, attack scenarios + +use chrono::{Duration, Utc}; +use uuid::Uuid; + +// Import MFA modules from api +use api::auth::mfa::{ + backup_codes::{hash_backup_code, BackupCodeGenerator}, + enrollment::{EnrollmentSession, EnrollmentStatus, MfaEnrollment}, + totp::{TotpGenerator, TotpVerifier}, + verification::{MfaVerification, VerificationMetadata, VerificationMethod, VerificationResult}, +}; +use secrecy::{ExposeSecret, SecretString}; + +// ============================================================================ +// SECTION 1: TOTP ADVANCED TESTS (15 tests) +// ============================================================================ + +#[test] +fn test_totp_replay_attack_prevention() { + let generator = TotpGenerator::new(); + let verifier = TotpVerifier::new(); + let secret = "JBSWY3DPEHPK3PXP"; + let time = 1234567890u64; + + // Generate code + let code = generator.generate_code_at_time(secret, time).unwrap(); + + // First verification succeeds + assert!(verifier.verify_at_time(secret, &code, time, 1).unwrap()); + + // In a real system, we'd track used codes to prevent replay + // This test documents that the current implementation allows replay + // within the same time window (expected behavior per RFC 6238) + assert!(verifier.verify_at_time(secret, &code, time, 1).unwrap()); + + // However, code should fail outside drift tolerance + let future_time = time + 90; // 3 periods later + assert!(!verifier + .verify_at_time(secret, &code, future_time, 1) + .unwrap()); +} + +#[test] +fn test_totp_time_boundary_conditions() { + let generator = TotpGenerator::new(); + let verifier = TotpVerifier::new(); + let secret = "JBSWY3DPEHPK3PXP"; + + // Test at exact period boundary (time % 30 == 0) + let boundary_time = 1234567890u64; // Divisible by 30 + assert_eq!(boundary_time % 30, 0); + + let code = generator + .generate_code_at_time(secret, boundary_time) + .unwrap(); + assert!(verifier + .verify_at_time(secret, &code, boundary_time, 1) + .unwrap()); + + // Code should work 1 second before period ends + let near_end = boundary_time + 29; + assert!(verifier.verify_at_time(secret, &code, near_end, 1).unwrap()); + + // Code should work at next period boundary with drift=1 + let next_boundary = boundary_time + 30; + assert!(verifier + .verify_at_time(secret, &code, next_boundary, 1) + .unwrap()); +} + +#[test] +fn test_totp_invalid_base32_secret() { + let generator = TotpGenerator::new(); + + // Invalid Base32 characters (0, 1, 8, 9 are valid in some alphabets but not RFC 4648) + let invalid_secret = "INVALID!@#$%"; + let result = generator.generate_code(invalid_secret); + assert!(result.is_err(), "Should reject invalid Base32 secret"); + + // Empty secret + let empty_secret = ""; + let result = generator.generate_code(empty_secret); + assert!(result.is_err(), "Should reject empty secret"); +} + +#[test] +fn test_totp_excessive_drift_tolerance() { + let generator = TotpGenerator::new(); + let verifier = TotpVerifier::new(); + let secret = "JBSWY3DPEHPK3PXP"; + let base_time = 1234567890u64; + + let code = generator.generate_code_at_time(secret, base_time).unwrap(); + + // With drift=2, should accept ±60 seconds + assert!(verifier + .verify_at_time(secret, &code, base_time + 60, 2) + .unwrap()); + assert!(verifier + .verify_at_time(secret, &code, base_time - 60, 2) + .unwrap()); + + // Should reject beyond drift=2 (±90 seconds) + assert!(!verifier + .verify_at_time(secret, &code, base_time + 90, 2) + .unwrap()); + assert!(!verifier + .verify_at_time(secret, &code, base_time - 90, 2) + .unwrap()); +} + +#[test] +fn test_totp_zero_drift_strict_validation() { + let generator = TotpGenerator::new(); + let verifier = TotpVerifier::new(); + let secret = "JBSWY3DPEHPK3PXP"; + let time = 1234567890u64; + + let code = generator.generate_code_at_time(secret, time).unwrap(); + + // With drift=0, only exact time window works + assert!(verifier.verify_at_time(secret, &code, time, 0).unwrap()); + assert!(verifier + .verify_at_time(secret, &code, time + 15, 0) + .unwrap()); // Same period + + // Even 1 period off fails with drift=0 + assert!(!verifier + .verify_at_time(secret, &code, time + 30, 0) + .unwrap()); + assert!(!verifier + .verify_at_time(secret, &code, time - 30, 0) + .unwrap()); +} + +#[test] +fn test_totp_future_code_rejection() { + let generator = TotpGenerator::new(); + let verifier = TotpVerifier::new(); + let secret = "JBSWY3DPEHPK3PXP"; + let current_time = 1234567890u64; + + // Generate code for future time + let future_time = current_time + 120; // 4 periods ahead + let future_code = generator + .generate_code_at_time(secret, future_time) + .unwrap(); + + // Should reject future code even with drift=1 + assert!(!verifier + .verify_at_time(secret, &future_code, current_time, 1) + .unwrap()); + + // Should reject even with drift=2 + assert!(!verifier + .verify_at_time(secret, &future_code, current_time, 2) + .unwrap()); +} + +#[test] +fn test_totp_past_code_expiration() { + let generator = TotpGenerator::new(); + let verifier = TotpVerifier::new(); + let secret = "JBSWY3DPEHPK3PXP"; + let current_time = 1234567890u64; + + // Generate code for past time + let past_time = current_time - 120; // 4 periods ago + let past_code = generator.generate_code_at_time(secret, past_time).unwrap(); + + // Should reject past code outside drift tolerance + assert!(!verifier + .verify_at_time(secret, &past_code, current_time, 1) + .unwrap()); + + // Should still reject with drift=2 (only covers ±60 seconds) + assert!(!verifier + .verify_at_time(secret, &past_code, current_time, 2) + .unwrap()); +} + +#[test] +fn test_totp_qr_uri_special_characters() { + let generator = TotpGenerator::new(); + let secret = SecretString::new("JBSWY3DPEHPK3PXP".to_string().into()); + + // Test with special characters in issuer and account + let uri = generator + .generate_qr_uri(&secret, "Foxhunt HFT (Test)", "user+test@example.com") + .unwrap(); + + // Should URL-encode special characters + assert!(uri.contains("otpauth://totp/")); + assert!(uri.contains("secret=JBSWY3DPEHPK3PXP")); + assert!(uri.contains("%28")); // '(' encoded + assert!(uri.contains("%29")); // ')' encoded + assert!(uri.contains("%2B")); // '+' encoded +} + +#[test] +fn test_totp_6_vs_8_digit_codes() { + let generator = TotpGenerator::new(); + let secret = "JBSWY3DPEHPK3PXP"; + let time = 1234567890u64; + + // Default is 6 digits + let code_6 = generator.generate_code_at_time(secret, time).unwrap(); + assert_eq!(code_6.len(), 6); + assert!(code_6.chars().all(|c| c.is_ascii_digit())); + + // 8 digit configuration (would require modifying TotpConfig) + // This test documents current behavior - only 6 digits supported + // In future, could test with TotpConfig { digits: 8, .. } +} + +#[test] +fn test_totp_hmac_determinism() { + let generator = TotpGenerator::new(); + let secret = "JBSWY3DPEHPK3PXP"; + let time = 1234567890u64; + + // Same secret and time should always produce same code + let code1 = generator.generate_code_at_time(secret, time).unwrap(); + let code2 = generator.generate_code_at_time(secret, time).unwrap(); + let code3 = generator.generate_code_at_time(secret, time).unwrap(); + + assert_eq!(code1, code2); + assert_eq!(code2, code3); +} + +#[test] +fn test_totp_different_secrets_different_codes() { + let generator = TotpGenerator::new(); + let time = 1234567890u64; + + let secret1 = "JBSWY3DPEHPK3PXP"; + let secret2 = "HXDMVJECJJWSRB3H"; + + let code1 = generator.generate_code_at_time(secret1, time).unwrap(); + let code2 = generator.generate_code_at_time(secret2, time).unwrap(); + + assert_ne!( + code1, code2, + "Different secrets should produce different codes" + ); +} + +#[test] +fn test_totp_secret_generation_uniqueness() { + let generator = TotpGenerator::new(); + + let secret1 = generator.generate_secret().unwrap(); + let secret2 = generator.generate_secret().unwrap(); + let secret3 = generator.generate_secret().unwrap(); + + // Should generate unique secrets + assert_ne!(secret1.expose_secret(), secret2.expose_secret()); + assert_ne!(secret2.expose_secret(), secret3.expose_secret()); + assert_ne!(secret1.expose_secret(), secret3.expose_secret()); + + // All should be valid Base32 + assert!(base32::decode( + base32::Alphabet::Rfc4648 { padding: false }, + secret1.expose_secret() + ) + .is_some()); +} + +#[test] +fn test_totp_code_at_period_transition() { + let generator = TotpGenerator::new(); + let verifier = TotpVerifier::new(); + let secret = "JBSWY3DPEHPK3PXP"; + + // Test exactly at period transition + let period_start = 1234567890u64; + let period_end = period_start + 29; // Last second of period + let next_period = period_start + 30; + + let code = generator + .generate_code_at_time(secret, period_start) + .unwrap(); + + // Should work at start and end of same period + assert!(verifier + .verify_at_time(secret, &code, period_start, 0) + .unwrap()); + assert!(verifier + .verify_at_time(secret, &code, period_end, 0) + .unwrap()); + + // Should fail at next period with drift=0 + assert!(!verifier + .verify_at_time(secret, &code, next_period, 0) + .unwrap()); + + // Should work at next period with drift=1 + assert!(verifier + .verify_at_time(secret, &code, next_period, 1) + .unwrap()); +} + +#[test] +fn test_totp_verifier_time_remaining() { + let verifier = TotpVerifier::new(); + + let remaining = verifier.time_remaining(); + + // Should be between 1 and 30 seconds (never 0 due to rounding) + assert!(remaining > 0, "Time remaining should be > 0"); + assert!(remaining <= 30, "Time remaining should be <= 30"); +} + +#[test] +fn test_totp_constant_time_comparison() { + let verifier = TotpVerifier::new(); + let secret = "JBSWY3DPEHPK3PXP"; + let time = 1234567890u64; + + // This test verifies constant-time comparison indirectly + // by ensuring timing doesn't leak information + + // Valid code + let generator = TotpGenerator::new(); + let valid_code = generator.generate_code_at_time(secret, time).unwrap(); + + // Nearly-valid code (differs by 1 digit at different positions) + let almost_code1 = format!("{}23456", &valid_code[0..0]); // Wrong from start + let almost_code2 = format!("{}3456", &valid_code[0..1]); // Wrong from position 1 + + // All should take similar time (constant-time comparison) + assert!(verifier + .verify_at_time(secret, &valid_code, time, 1) + .unwrap()); + assert!(!verifier + .verify_at_time(secret, &almost_code1, time, 1) + .unwrap()); + assert!(!verifier + .verify_at_time(secret, &almost_code2, time, 1) + .unwrap()); +} + +// ============================================================================ +// SECTION 2: BACKUP CODE SECURITY TESTS (12 tests) +// ============================================================================ + +#[test] +fn test_backup_code_generation_count() { + let generator = BackupCodeGenerator::new(); + + // Valid counts (1-20) + assert!(generator.generate_codes(1).is_ok()); + assert!(generator.generate_codes(10).is_ok()); + assert!(generator.generate_codes(20).is_ok()); + + // Invalid counts + assert!(generator.generate_codes(0).is_err()); + assert!(generator.generate_codes(21).is_err()); + assert!(generator.generate_codes(100).is_err()); +} + +#[test] +fn test_backup_code_format_normalization() { + // Test the normalization function directly + use api::auth::mfa::backup_codes::BackupCodeGenerator; + + let generator = BackupCodeGenerator::new(); + let codes = generator.generate_codes(1).unwrap(); + let code = &codes[0]; + + // Display format should have hyphens + assert!(code.display.contains('-')); + assert_eq!(code.display.split('-').count(), 4); // 4 groups + + // Hint should be first 4 characters + assert_eq!(code.hint.len(), 4); + assert_eq!(&code.hint, &code.expose()[0..4]); + + // Code itself should be 16 characters + assert_eq!(code.expose().len(), 16); +} + +#[test] +fn test_backup_code_charset_validation() { + let generator = BackupCodeGenerator::new(); + let codes = generator.generate_codes(10).unwrap(); + + for code in &codes { + let code_str = code.expose(); + + // Should only contain allowed characters (no 0, O, 1, I, l) + for c in code_str.chars() { + assert!( + c.is_ascii_uppercase() || c.is_ascii_digit(), + "Invalid character: {}", + c + ); + assert_ne!(c, '0', "Should not contain 0"); + assert_ne!(c, 'O', "Should not contain O"); + assert_ne!(c, '1', "Should not contain 1"); + assert_ne!(c, 'I', "Should not contain I"); + } + } +} + +#[test] +fn test_backup_code_uniqueness() { + let generator = BackupCodeGenerator::new(); + let codes = generator.generate_codes(20).unwrap(); + + // All codes should be unique + let mut seen = std::collections::HashSet::new(); + for code in &codes { + assert!( + seen.insert(code.expose().to_string()), + "Duplicate code generated: {}", + code.expose() + ); + } +} + +#[test] +fn test_backup_code_hash_uniqueness() { + let codes = vec!["ABCD2345EFGH6789", "ABCD2345EFGH678A", "DIFFERENT23456789"]; + + let hash1 = hash_backup_code(codes[0]); + let hash2 = hash_backup_code(codes[1]); + let hash3 = hash_backup_code(codes[2]); + + // Different codes should have different hashes + assert_ne!(hash1, hash2); + assert_ne!(hash2, hash3); + assert_ne!(hash1, hash3); + + // Same code should have same hash (determinism) + assert_eq!(hash_backup_code(codes[0]), hash1); +} + +#[test] +fn test_backup_code_hash_length() { + let code = "ABCD2345EFGH6789"; + let hash = hash_backup_code(code); + + // SHA-256 produces 64 hex characters + assert_eq!(hash.len(), 64); + assert!(hash.chars().all(|c| c.is_ascii_hexdigit())); +} + +#[test] +fn test_backup_code_single_generation() { + let generator = BackupCodeGenerator::new(); + let code = generator.generate_single_code().unwrap(); + + assert_eq!(code.expose().len(), 16); + assert_eq!(code.hint.len(), 4); + assert!(code.display.contains('-')); +} + +#[test] +fn test_backup_code_display_formatting() { + let generator = BackupCodeGenerator::new(); + let codes = generator.generate_codes(5).unwrap(); + + for code in &codes { + // Display should be in format XXXX-XXXX-XXXX-XXXX + let parts: Vec<&str> = code.display.split('-').collect(); + assert_eq!(parts.len(), 4); + + for part in parts { + assert_eq!(part.len(), 4); + } + } +} + +#[test] +fn test_backup_code_entropy() { + let generator = BackupCodeGenerator::new(); + let codes = generator.generate_codes(20).unwrap(); + + // Test character distribution (should be relatively uniform) + let mut char_counts = std::collections::HashMap::new(); + + for code in &codes { + for c in code.expose().chars() { + *char_counts.entry(c).or_insert(0) += 1; + } + } + + // Should have good distribution (at least 10 different characters used) + assert!( + char_counts.len() >= 10, + "Should use at least 10 different characters, got {}", + char_counts.len() + ); + + // No character should dominate (> 30% of all characters) + let total_chars: usize = char_counts.values().sum(); + for (_c, count) in &char_counts { + let percentage = (*count as f64) / (total_chars as f64); + assert!( + percentage < 0.3, + "Character distribution too skewed: {:.2}%", + percentage * 100.0 + ); + } +} + +#[test] +fn test_backup_code_case_insensitivity() { + // This test documents expected behavior for case handling + let code_upper = "ABCD2345EFGH6789"; + let code_lower = "abcd2345efgh6789"; + + // Hashes should be different (case-sensitive hashing) + let hash_upper = hash_backup_code(code_upper); + let hash_lower = hash_backup_code(code_lower); + assert_ne!(hash_upper, hash_lower); + + // Note: In real validation, codes are normalized to uppercase + // before hashing, so this test documents the raw hash behavior +} + +#[test] +fn test_backup_code_length_validation() { + // Test various invalid lengths + let too_short = "ABCD2345"; + let too_long = "ABCD2345EFGH67890123"; + let valid = "ABCD2345EFGH6789"; + + // All generate 64-char hashes, but validation checks length separately + assert_eq!(hash_backup_code(too_short).len(), 64); + assert_eq!(hash_backup_code(too_long).len(), 64); + assert_eq!(hash_backup_code(valid).len(), 64); +} + +#[test] +fn test_backup_code_hint_accuracy() { + let generator = BackupCodeGenerator::new(); + let codes = generator.generate_codes(10).unwrap(); + + for code in &codes { + // Hint should match first 4 characters of actual code + let actual_first_4: String = code.expose().chars().take(4).collect(); + assert_eq!(code.hint, actual_first_4); + } +} + +// ============================================================================ +// SECTION 3: ENROLLMENT FLOW TESTS (10 tests) +// ============================================================================ + +#[test] +fn test_enrollment_initial_state() { + let user_id = Uuid::new_v4(); + let enrollment = MfaEnrollment::new(user_id); + + assert_eq!(enrollment.user_id, user_id); + assert_eq!(enrollment.status, EnrollmentStatus::NotStarted); + assert!(enrollment.session.is_none()); + assert_eq!(enrollment.verification_attempts, 0); + assert_eq!(enrollment.max_attempts, 3); +} + +#[test] +fn test_enrollment_start_session() { + let user_id = Uuid::new_v4(); + let mut enrollment = MfaEnrollment::new(user_id); + + let session = EnrollmentSession { + session_id: Uuid::new_v4(), + user_id, + qr_code_uri: "otpauth://totp/test".to_string(), + qr_code_png: vec![1, 2, 3], + manual_entry_key: "TEST123".to_string(), + expires_at: Utc::now() + Duration::minutes(15), + }; + + enrollment.start(session); + + assert_eq!(enrollment.status, EnrollmentStatus::InProgress); + assert!(enrollment.session.is_some()); + assert_eq!(enrollment.verification_attempts, 0); +} + +#[test] +fn test_enrollment_complete() { + let user_id = Uuid::new_v4(); + let mut enrollment = MfaEnrollment::new(user_id); + + // Start enrollment + let session = EnrollmentSession { + session_id: Uuid::new_v4(), + user_id, + qr_code_uri: "otpauth://totp/test".to_string(), + qr_code_png: vec![], + manual_entry_key: "TEST123".to_string(), + expires_at: Utc::now() + Duration::minutes(15), + }; + enrollment.start(session); + + // Complete enrollment + enrollment.complete(); + + assert_eq!(enrollment.status, EnrollmentStatus::Completed); + assert!(enrollment.session.is_none()); +} + +#[test] +fn test_enrollment_fail() { + let user_id = Uuid::new_v4(); + let mut enrollment = MfaEnrollment::new(user_id); + + enrollment.fail("Invalid verification code".to_string()); + + match &enrollment.status { + EnrollmentStatus::Failed(reason) => { + assert_eq!(reason, "Invalid verification code"); + }, + _ => panic!("Expected Failed status"), + } + assert!(enrollment.session.is_none()); +} + +#[test] +fn test_enrollment_verification_attempts() { + let user_id = Uuid::new_v4(); + let mut enrollment = MfaEnrollment::new(user_id); + + assert_eq!(enrollment.verification_attempts, 0); + assert!(!enrollment.is_max_attempts_reached()); + + enrollment.increment_attempts(); + assert_eq!(enrollment.verification_attempts, 1); + assert!(!enrollment.is_max_attempts_reached()); + + enrollment.increment_attempts(); + enrollment.increment_attempts(); + assert_eq!(enrollment.verification_attempts, 3); + assert!(enrollment.is_max_attempts_reached()); + + // Can still increment beyond max + enrollment.increment_attempts(); + assert_eq!(enrollment.verification_attempts, 4); + assert!(enrollment.is_max_attempts_reached()); +} + +#[test] +fn test_enrollment_session_not_expired() { + let user_id = Uuid::new_v4(); + let mut enrollment = MfaEnrollment::new(user_id); + + // Session expires in future + let session = EnrollmentSession { + session_id: Uuid::new_v4(), + user_id, + qr_code_uri: "otpauth://totp/test".to_string(), + qr_code_png: vec![], + manual_entry_key: "TEST123".to_string(), + expires_at: Utc::now() + Duration::hours(1), + }; + enrollment.start(session); + + assert!(!enrollment.is_expired()); +} + +#[test] +fn test_enrollment_session_expired() { + let user_id = Uuid::new_v4(); + let mut enrollment = MfaEnrollment::new(user_id); + + // Session already expired + let session = EnrollmentSession { + session_id: Uuid::new_v4(), + user_id, + qr_code_uri: "otpauth://totp/test".to_string(), + qr_code_png: vec![], + manual_entry_key: "TEST123".to_string(), + expires_at: Utc::now() - Duration::hours(1), + }; + enrollment.start(session); + + assert!(enrollment.is_expired()); +} + +#[test] +fn test_enrollment_no_session_not_expired() { + let user_id = Uuid::new_v4(); + let enrollment = MfaEnrollment::new(user_id); + + // No session means not expired + assert!(!enrollment.is_expired()); +} + +#[test] +fn test_enrollment_state_transitions() { + let user_id = Uuid::new_v4(); + let mut enrollment = MfaEnrollment::new(user_id); + + // NotStarted → InProgress + assert_eq!(enrollment.status, EnrollmentStatus::NotStarted); + + let session = EnrollmentSession { + session_id: Uuid::new_v4(), + user_id, + qr_code_uri: "otpauth://totp/test".to_string(), + qr_code_png: vec![], + manual_entry_key: "TEST123".to_string(), + expires_at: Utc::now() + Duration::minutes(15), + }; + enrollment.start(session); + assert_eq!(enrollment.status, EnrollmentStatus::InProgress); + + // InProgress → Completed + enrollment.complete(); + assert_eq!(enrollment.status, EnrollmentStatus::Completed); + + // Can restart after completion + let new_session = EnrollmentSession { + session_id: Uuid::new_v4(), + user_id, + qr_code_uri: "otpauth://totp/test2".to_string(), + qr_code_png: vec![], + manual_entry_key: "TEST456".to_string(), + expires_at: Utc::now() + Duration::minutes(15), + }; + enrollment.start(new_session); + assert_eq!(enrollment.status, EnrollmentStatus::InProgress); + + // InProgress → Failed + enrollment.fail("Test failure".to_string()); + match &enrollment.status { + EnrollmentStatus::Failed(reason) => assert_eq!(reason, "Test failure"), + _ => panic!("Expected Failed status"), + } +} + +#[test] +fn test_enrollment_session_data_integrity() { + let user_id = Uuid::new_v4(); + let mut enrollment = MfaEnrollment::new(user_id); + + let session_id = Uuid::new_v4(); + let qr_uri = "otpauth://totp/FoxhuntHFT:user@example.com".to_string(); + let qr_png = vec![1, 2, 3, 4, 5]; + let manual_key = "JBSWY3DPEHPK3PXP".to_string(); + let expires = Utc::now() + Duration::minutes(10); + + let session = EnrollmentSession { + session_id, + user_id, + qr_code_uri: qr_uri.clone(), + qr_code_png: qr_png.clone(), + manual_entry_key: manual_key.clone(), + expires_at: expires, + }; + + enrollment.start(session); + + // Verify all session data preserved + let stored_session = enrollment.session.as_ref().expect("INVARIANT: Option should be Some"); + assert_eq!(stored_session.session_id, session_id); + assert_eq!(stored_session.user_id, user_id); + assert_eq!(stored_session.qr_code_uri, qr_uri); + assert_eq!(stored_session.qr_code_png, qr_png); + assert_eq!(stored_session.manual_entry_key, manual_key); + assert_eq!(stored_session.expires_at, expires); +} + +// ============================================================================ +// SECTION 4: VERIFICATION FLOW TESTS (10 tests) +// ============================================================================ + +#[test] +fn test_verification_result_success_creation() { + let user_id = Uuid::new_v4(); + let metadata = VerificationMetadata { + failed_attempts: 0, + is_locked: false, + locked_until: None, + backup_codes_remaining: Some(10), + totp_drift: None, + }; + + let result = VerificationResult::success(user_id, VerificationMethod::Totp, metadata); + + assert!(result.success); + assert_eq!(result.user_id, user_id); + assert_eq!(result.method, VerificationMethod::Totp); + assert_eq!(result.metadata.failed_attempts, 0); + assert!(!result.metadata.is_locked); +} + +#[test] +fn test_verification_result_failure_creation() { + let user_id = Uuid::new_v4(); + let metadata = VerificationMetadata { + failed_attempts: 3, + is_locked: false, + locked_until: None, + backup_codes_remaining: Some(8), + totp_drift: Some(1), + }; + + let result = VerificationResult::failure(user_id, VerificationMethod::Totp, metadata); + + assert!(!result.success); + assert_eq!(result.metadata.failed_attempts, 3); + assert_eq!(result.metadata.totp_drift, Some(1)); +} + +#[test] +fn test_verification_metadata_lockout_info() { + let user_id = Uuid::new_v4(); + let locked_until = Utc::now() + Duration::minutes(15); + + let metadata = VerificationMetadata { + failed_attempts: 5, + is_locked: true, + locked_until: Some(locked_until), + backup_codes_remaining: Some(10), + totp_drift: None, + }; + + let result = VerificationResult::failure(user_id, VerificationMethod::Totp, metadata); + + assert!(result.metadata.is_locked); + assert_eq!(result.metadata.locked_until, Some(locked_until)); + assert_eq!(result.metadata.failed_attempts, 5); +} + +#[test] +fn test_verification_backup_codes_remaining() { + let user_id = Uuid::new_v4(); + + // With backup codes + let metadata = VerificationMetadata { + failed_attempts: 0, + is_locked: false, + locked_until: None, + backup_codes_remaining: Some(5), + totp_drift: None, + }; + let result = VerificationResult::success(user_id, VerificationMethod::BackupCode, metadata); + assert_eq!(result.metadata.backup_codes_remaining, Some(5)); + + // Without backup codes (TOTP verification) + let metadata2 = VerificationMetadata { + failed_attempts: 0, + is_locked: false, + locked_until: None, + backup_codes_remaining: None, + totp_drift: Some(0), + }; + let result2 = VerificationResult::success(user_id, VerificationMethod::Totp, metadata2); + assert_eq!(result2.metadata.backup_codes_remaining, None); +} + +#[test] +fn test_verification_totp_drift_tracking() { + let user_id = Uuid::new_v4(); + + // No drift (exact match) + let metadata = VerificationMetadata { + failed_attempts: 0, + is_locked: false, + locked_until: None, + backup_codes_remaining: None, + totp_drift: Some(0), + }; + let result = VerificationResult::success(user_id, VerificationMethod::Totp, metadata); + assert_eq!(result.metadata.totp_drift, Some(0)); + + // Drift of 1 period + let metadata2 = VerificationMetadata { + failed_attempts: 0, + is_locked: false, + locked_until: None, + backup_codes_remaining: None, + totp_drift: Some(1), + }; + let result2 = VerificationResult::success(user_id, VerificationMethod::Totp, metadata2); + assert_eq!(result2.metadata.totp_drift, Some(1)); +} + +#[test] +fn test_verification_method_serialization() { + // Test JSON serialization + let method_totp = VerificationMethod::Totp; + let json = serde_json::to_string(&method_totp).expect("INVARIANT: Serialization should succeed for valid types"); + assert_eq!(json, "\"totp\""); + + let method_backup = VerificationMethod::BackupCode; + let json = serde_json::to_string(&method_backup).expect("INVARIANT: Serialization should succeed for valid types"); + assert_eq!(json, "\"backup_code\""); + + let method_device = VerificationMethod::TrustedDevice; + let json = serde_json::to_string(&method_device).expect("INVARIANT: Serialization should succeed for valid types"); + assert_eq!(json, "\"trusted_device\""); +} + +#[test] +fn test_verification_timestamp_recent() { + let user_id = Uuid::new_v4(); + let metadata = VerificationMetadata { + failed_attempts: 0, + is_locked: false, + locked_until: None, + backup_codes_remaining: Some(10), + totp_drift: None, + }; + + let result = VerificationResult::success(user_id, VerificationMethod::Totp, metadata); + + // Timestamp should be very recent (within last second) + let now = Utc::now(); + let diff = now.signed_duration_since(result.timestamp); + assert!(diff.num_seconds() < 2, "Timestamp should be recent"); +} + +#[test] +fn test_verification_request_structure() { + let user_id = Uuid::new_v4(); + + let verification = MfaVerification { + user_id, + method: VerificationMethod::Totp, + code: "123456".to_string(), + ip_address: Some("192.168.1.100".to_string()), + user_agent: Some("Mozilla/5.0".to_string()), + }; + + assert_eq!(verification.user_id, user_id); + assert_eq!(verification.method, VerificationMethod::Totp); + assert_eq!(verification.code, "123456"); + assert_eq!(verification.ip_address, Some("192.168.1.100".to_string())); + assert_eq!(verification.user_agent, Some("Mozilla/5.0".to_string())); +} + +#[test] +fn test_verification_multiple_methods() { + let user_id = Uuid::new_v4(); + let metadata = VerificationMetadata { + failed_attempts: 0, + is_locked: false, + locked_until: None, + backup_codes_remaining: Some(10), + totp_drift: Some(0), + }; + + // TOTP verification + let result_totp = + VerificationResult::success(user_id, VerificationMethod::Totp, metadata.clone()); + assert_eq!(result_totp.method, VerificationMethod::Totp); + + // Backup code verification + let result_backup = + VerificationResult::success(user_id, VerificationMethod::BackupCode, metadata.clone()); + assert_eq!(result_backup.method, VerificationMethod::BackupCode); + + // Trusted device verification + let result_device = + VerificationResult::success(user_id, VerificationMethod::TrustedDevice, metadata); + assert_eq!(result_device.method, VerificationMethod::TrustedDevice); +} + +#[test] +fn test_verification_failed_attempts_progression() { + let user_id = Uuid::new_v4(); + + // Attempt 1 + let metadata1 = VerificationMetadata { + failed_attempts: 1, + is_locked: false, + locked_until: None, + backup_codes_remaining: Some(10), + totp_drift: None, + }; + let result1 = VerificationResult::failure(user_id, VerificationMethod::Totp, metadata1); + assert_eq!(result1.metadata.failed_attempts, 1); + assert!(!result1.metadata.is_locked); + + // Attempt 3 (still not locked) + let metadata3 = VerificationMetadata { + failed_attempts: 3, + is_locked: false, + locked_until: None, + backup_codes_remaining: Some(10), + totp_drift: None, + }; + let result3 = VerificationResult::failure(user_id, VerificationMethod::Totp, metadata3); + assert_eq!(result3.metadata.failed_attempts, 3); + assert!(!result3.metadata.is_locked); + + // Attempt 5 (now locked) + let locked_until = Utc::now() + Duration::minutes(15); + let metadata5 = VerificationMetadata { + failed_attempts: 5, + is_locked: true, + locked_until: Some(locked_until), + backup_codes_remaining: Some(10), + totp_drift: None, + }; + let result5 = VerificationResult::failure(user_id, VerificationMethod::Totp, metadata5); + assert_eq!(result5.metadata.failed_attempts, 5); + assert!(result5.metadata.is_locked); + assert!(result5.metadata.locked_until.is_some()); +} + +// ============================================================================ +// SECTION 5: INTEGRATION & SECURITY TESTS (8 tests) +// ============================================================================ + +#[test] +fn test_integration_full_enrollment_flow() { + let user_id = Uuid::new_v4(); + let generator = TotpGenerator::new(); + + // Step 1: Generate TOTP secret + let secret = generator.generate_secret().unwrap(); + + // Step 2: Generate QR code URI + let qr_uri = generator + .generate_qr_uri(&secret, "FoxhuntHFT", "trader@example.com") + .unwrap(); + assert!(qr_uri.starts_with("otpauth://totp/")); + + // Step 3: Create enrollment session + let mut enrollment = MfaEnrollment::new(user_id); + let session = EnrollmentSession { + session_id: Uuid::new_v4(), + user_id, + qr_code_uri: qr_uri, + qr_code_png: vec![], + manual_entry_key: secret.expose_secret().to_string(), + expires_at: Utc::now() + Duration::minutes(15), + }; + enrollment.start(session); + assert_eq!(enrollment.status, EnrollmentStatus::InProgress); + + // Step 4: User scans QR code and enters code + let time = 1234567890u64; + let code = generator + .generate_code_at_time(secret.expose_secret(), time) + .unwrap(); + + // Step 5: Verify code + let verifier = TotpVerifier::new(); + let is_valid = verifier + .verify_at_time(secret.expose_secret(), &code, time, 1) + .unwrap(); + assert!(is_valid); + + // Step 6: Complete enrollment + enrollment.complete(); + assert_eq!(enrollment.status, EnrollmentStatus::Completed); +} + +#[test] +fn test_integration_backup_codes_generation() { + let user_id = Uuid::new_v4(); + let generator = BackupCodeGenerator::new(); + + // Generate 10 backup codes during enrollment + let codes = generator.generate_codes(10).unwrap(); + assert_eq!(codes.len(), 10); + + // All codes should be valid format + for code in &codes { + assert_eq!(code.expose().len(), 16); + assert_eq!(code.hint.len(), 4); + assert!(code.display.contains('-')); + } + + // Hash all codes for storage + let hashed_codes: Vec = codes.iter().map(|c| hash_backup_code(c.expose())).collect(); + + // All hashes should be unique + let unique_hashes: std::collections::HashSet<_> = hashed_codes.iter().collect(); + assert_eq!(unique_hashes.len(), 10); +} + +#[test] +fn test_integration_totp_and_backup_codes() { + let user_id = Uuid::new_v4(); + + // TOTP setup + let totp_generator = TotpGenerator::new(); + let totp_secret = totp_generator.generate_secret().unwrap(); + + // Backup codes setup + let backup_generator = BackupCodeGenerator::new(); + let backup_codes = backup_generator.generate_codes(10).unwrap(); + + // Both should be independent + assert_ne!( + totp_secret.expose_secret().len(), + backup_codes[0].expose().len() + ); + + // TOTP verification + let time = 1234567890u64; + let totp_code = totp_generator + .generate_code_at_time(totp_secret.expose_secret(), time) + .unwrap(); + let verifier = TotpVerifier::new(); + assert!(verifier + .verify_at_time(totp_secret.expose_secret(), &totp_code, time, 1) + .unwrap()); + + // Backup codes should be available as fallback + assert_eq!(backup_codes.len(), 10); +} + +#[test] +fn test_security_timing_attack_resistance() { + let verifier = TotpVerifier::new(); + let secret = "JBSWY3DPEHPK3PXP"; + let time = 1234567890u64; + + let generator = TotpGenerator::new(); + let valid_code = generator.generate_code_at_time(secret, time).unwrap(); + + // All verifications should take similar time (constant-time comparison) + let wrong_codes = vec![ + "000000".to_string(), // All wrong + format!("{}00000", &valid_code[0..1]), // First digit correct + format!("{}0000", &valid_code[0..2]), // First two correct + format!("{}000", &valid_code[0..3]), // First three correct + format!("{}00", &valid_code[0..4]), // First four correct + format!("{}0", &valid_code[0..5]), // First five correct + ]; + + for wrong_code in &wrong_codes { + let result = verifier + .verify_at_time(secret, wrong_code, time, 1) + .unwrap(); + assert!(!result, "Wrong code should fail: {}", wrong_code); + } + + // Valid code should succeed + assert!(verifier + .verify_at_time(secret, &valid_code, time, 1) + .unwrap()); +} + +#[test] +fn test_security_backup_code_entropy_analysis() { + let generator = BackupCodeGenerator::new(); + + // Generate large sample for entropy analysis + let codes = generator.generate_codes(20).unwrap(); + + // Collect all characters + let mut all_chars = String::new(); + for code in &codes { + all_chars.push_str(code.expose()); + } + + // Calculate character frequency + let mut freq_map = std::collections::HashMap::new(); + for c in all_chars.chars() { + *freq_map.entry(c).or_insert(0) += 1; + } + + // Shannon entropy calculation + let total = all_chars.len() as f64; + let mut entropy = 0.0; + for count in freq_map.values() { + let p = (*count as f64) / total; + entropy -= p * p.log2(); + } + + // Entropy should be > 4.0 bits per character (good randomness) + // Theoretical maximum for charset of 32 characters is 5.0 bits + assert!( + entropy > 4.0, + "Entropy too low: {:.2} bits per character", + entropy + ); +} + +#[test] +fn test_security_session_expiration_enforcement() { + let user_id = Uuid::new_v4(); + let mut enrollment = MfaEnrollment::new(user_id); + + // Create session that expires in 1 second + let session = EnrollmentSession { + session_id: Uuid::new_v4(), + user_id, + qr_code_uri: "otpauth://totp/test".to_string(), + qr_code_png: vec![], + manual_entry_key: "TEST123".to_string(), + expires_at: Utc::now() + Duration::seconds(1), + }; + enrollment.start(session); + + // Initially not expired + assert!(!enrollment.is_expired()); + + // Wait 2 seconds + std::thread::sleep(std::time::Duration::from_secs(2)); + + // Now should be expired + assert!(enrollment.is_expired()); +} + +#[test] +fn test_security_qr_code_uri_injection() { + let generator = TotpGenerator::new(); + let secret = SecretString::new("JBSWY3DPEHPK3PXP".to_string().into()); + + // Try injection attacks in issuer/account + let malicious_issuer = "FoxhuntHFT&secret=HACKED"; + let malicious_account = "user@example.com?secret=STOLEN"; + + let uri = generator + .generate_qr_uri(&secret, malicious_issuer, malicious_account) + .unwrap(); + + // Should URL-encode special characters + assert!(uri.contains("%26")); // '&' encoded + assert!(uri.contains("%3F")); // '?' encoded + + // Original secret should still be present + assert!(uri.contains("secret=JBSWY3DPEHPK3PXP")); + + // Malicious parts should be encoded (not interpreted) + assert!(!uri.contains("secret=HACKED")); + assert!(!uri.contains("secret=STOLEN")); +} + +#[test] +fn test_security_hash_collision_resistance() { + // Test that similar backup codes produce different hashes + let codes = vec![ + "ABCD2345EFGH6789", + "ABCD2345EFGH678A", // Last char different + "ABCD2345EFGH679A", // Second-to-last different + "ABCD2345EFGH689A", // Third-to-last different + ]; + + let hashes: Vec = codes.iter().map(|c| hash_backup_code(c)).collect(); + + // All hashes should be unique + let unique: std::collections::HashSet<_> = hashes.iter().collect(); + assert_eq!(unique.len(), codes.len()); + + // Small changes should produce completely different hashes (avalanche effect) + let hash_diff_1_2 = hamming_distance(&hashes[0], &hashes[1]); + let hash_diff_2_3 = hamming_distance(&hashes[1], &hashes[2]); + + // At least 30% of bits should differ (good avalanche) + assert!( + hash_diff_1_2 > 19, + "Hash avalanche too weak: {} bits different", + hash_diff_1_2 + ); + assert!( + hash_diff_2_3 > 19, + "Hash avalanche too weak: {} bits different", + hash_diff_2_3 + ); +} + +// ============================================================================ +// HELPER FUNCTIONS +// ============================================================================ + +/// Calculate Hamming distance between two hex strings (number of different characters) +fn hamming_distance(s1: &str, s2: &str) -> usize { + if s1.len() != s2.len() { + return usize::MAX; + } + + s1.chars() + .zip(s2.chars()) + .filter(|(c1, c2)| c1 != c2) + .count() +} + +#[test] +fn test_helper_hamming_distance() { + assert_eq!(hamming_distance("abc", "abc"), 0); + assert_eq!(hamming_distance("abc", "abd"), 1); + assert_eq!(hamming_distance("abc", "xyz"), 3); + assert_eq!(hamming_distance("abc", "abcd"), usize::MAX); +} diff --git a/services/api/tests/mfa_enrollment_integration_test.rs b/services/api/tests/mfa_enrollment_integration_test.rs new file mode 100644 index 000000000..692f9a401 --- /dev/null +++ b/services/api/tests/mfa_enrollment_integration_test.rs @@ -0,0 +1,449 @@ +#![cfg(feature = "mfa")] +//! MFA Enrollment Integration Test +//! +//! Agent H3: Tests complete MFA enrollment flow including: +//! - QR code generation +//! - TOTP verification +//! - Backup code generation +//! - Account lockout after failed attempts +//! - Admin enforcement + +use anyhow::Result; +use secrecy::ExposeSecret; +use sqlx::PgPool; +use uuid::Uuid; + +use api::auth::mfa::MfaManager; + +/// Helper to create test database connection +async fn setup_test_db() -> Result { + let database_url = std::env::var("DATABASE_URL").unwrap_or_else(|_| { + "postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt".to_string() + }); + + let pool = PgPool::connect(&database_url).await?; + Ok(pool) +} + +/// Helper to create test user with admin role +async fn create_test_admin_user(pool: &PgPool) -> Result { + let user_id = Uuid::new_v4(); + let username = format!( + "test_admin_{}", + Uuid::new_v4().to_string().split('-').next().expect("INVARIANT: Iterator should have next element") + ); + let email = format!("{}@test.local", username); + + // Create user + sqlx::query( + r#" + INSERT INTO users ( + id, username, email, password_hash, salt, must_change_password, active + ) VALUES ($1, $2, $3, '$2b$12$test_hash', 'test_salt', FALSE, TRUE) + "#, + ) + .bind(user_id) + .bind(&username) + .bind(&email) + .execute(pool) + .await?; + + // Assign system_admin role + let admin_role_id = Uuid::parse_str("00000000-0000-0000-0000-000000000001")?; + sqlx::query("INSERT INTO user_roles (user_id, role_id, granted_by) VALUES ($1, $2, $1)") + .bind(user_id) + .bind(admin_role_id) + .execute(pool) + .await?; + + Ok(user_id) +} + +/// Helper to cleanup test user +async fn cleanup_test_user(pool: &PgPool, user_id: Uuid) -> Result<()> { + // Foreign key constraints will cascade delete related records + sqlx::query("DELETE FROM users WHERE id = $1") + .bind(user_id) + .execute(pool) + .await?; + + Ok(()) +} + +#[tokio::test] +async fn test_mfa_enrollment_complete_flow() -> Result<()> { + let pool = setup_test_db().await?; + let user_id = create_test_admin_user(&pool).await?; + + // Create MFA manager + let encryption_key = "test_encryption_key_32_bytes_long!!".to_string(); + let mfa_manager = MfaManager::new(pool.clone(), encryption_key)?; + + // Step 1: Start enrollment + let enrollment = mfa_manager + .start_enrollment(user_id, "Foxhunt Test", "test_user@test.local") + .await?; + + println!("✓ MFA enrollment session created"); + println!(" Session ID: {}", enrollment.session_id); + println!(" QR Code URI: {}", enrollment.qr_code_uri); + println!(" QR Code PNG: {} bytes", enrollment.qr_code_png.len()); + println!(" Manual Entry Key: {}", enrollment.manual_entry_key); + + assert!( + !enrollment.qr_code_uri.is_empty(), + "QR code URI should not be empty" + ); + assert!( + !enrollment.qr_code_png.is_empty(), + "QR code PNG should not be empty" + ); + assert!( + enrollment.qr_code_png.len() > 100, + "QR code PNG should be at least 100 bytes" + ); + assert!( + enrollment.manual_entry_key.len() >= 16, + "TOTP secret should be at least 16 characters" + ); + + // Step 2: Generate valid TOTP code from secret + use totp_rs::{Algorithm, TOTP}; + let totp = TOTP::new( + Algorithm::SHA1, + 6, + 1, + 30, + enrollment.manual_entry_key.as_bytes().to_vec(), + )?; + + let valid_code = totp.generate_current()?; + println!("✓ Generated TOTP code: {}", valid_code); + + // Step 3: Complete enrollment with valid TOTP code + let backup_codes = mfa_manager + .complete_enrollment(enrollment.session_id, user_id, &valid_code) + .await?; + + println!("✓ MFA enrollment completed"); + println!(" Backup codes generated: {}", backup_codes.len()); + + assert_eq!(backup_codes.len(), 10, "Should generate 10 backup codes"); + + for (i, code) in backup_codes.iter().enumerate() { + println!( + " Backup Code {}: {} (hint: {})", + i + 1, + code.code.expose_secret(), + code.hint + ); + assert!( + code.code.expose_secret().len() >= 8, + "Backup code should be at least 8 characters" + ); + assert_eq!(code.hint.len(), 4, "Hint should be 4 characters"); + } + + // Step 4: Verify MFA is enabled + let config = mfa_manager.get_mfa_config(user_id).await?; + assert!(config.is_some(), "MFA config should exist"); + + let config = config.unwrap(); + assert!(config.is_enabled, "MFA should be enabled"); + assert!(config.is_verified, "MFA should be verified"); + assert_eq!( + config.backup_codes_remaining, 10, + "Should have 10 backup codes" + ); + + println!("✓ MFA config verified"); + println!(" Enabled: {}", config.is_enabled); + println!(" Verified: {}", config.is_verified); + println!( + " Backup codes remaining: {}", + config.backup_codes_remaining + ); + + // Cleanup + cleanup_test_user(&pool, user_id).await?; + + Ok(()) +} + +#[tokio::test] +async fn test_mfa_totp_verification() -> Result<()> { + let pool = setup_test_db().await?; + let user_id = create_test_admin_user(&pool).await?; + + let encryption_key = "test_encryption_key_32_bytes_long!!".to_string(); + let mfa_manager = MfaManager::new(pool.clone(), encryption_key)?; + + // Enroll user in MFA + let enrollment = mfa_manager + .start_enrollment(user_id, "Foxhunt Test", "test@test.local") + .await?; + + use totp_rs::{Algorithm, TOTP}; + let totp = TOTP::new( + Algorithm::SHA1, + 6, + 1, + 30, + enrollment.manual_entry_key.as_bytes().to_vec(), + )?; + + let valid_code = totp.generate_current()?; + let _backup_codes = mfa_manager + .complete_enrollment(enrollment.session_id, user_id, &valid_code) + .await?; + + // Test valid TOTP verification + let new_code = totp.generate_current()?; + let is_valid = mfa_manager + .verify_totp(user_id, &new_code, Some("127.0.0.1".to_string())) + .await?; + + assert!(is_valid, "Valid TOTP code should verify successfully"); + println!("✓ Valid TOTP code verified"); + + // Test invalid TOTP code + let invalid_result = mfa_manager + .verify_totp(user_id, "000000", Some("127.0.0.1".to_string())) + .await; + + assert!( + invalid_result.is_ok(), + "Invalid code should return Ok(false)" + ); + assert!( + !invalid_result.unwrap(), + "Invalid TOTP code should not verify" + ); + println!("✓ Invalid TOTP code rejected"); + + // Cleanup + cleanup_test_user(&pool, user_id).await?; + + Ok(()) +} + +#[tokio::test] +async fn test_mfa_backup_code_recovery() -> Result<()> { + let pool = setup_test_db().await?; + let user_id = create_test_admin_user(&pool).await?; + + let encryption_key = "test_encryption_key_32_bytes_long!!".to_string(); + let mfa_manager = MfaManager::new(pool.clone(), encryption_key)?; + + // Enroll user in MFA + let enrollment = mfa_manager + .start_enrollment(user_id, "Foxhunt Test", "test@test.local") + .await?; + + use totp_rs::{Algorithm, TOTP}; + let totp = TOTP::new( + Algorithm::SHA1, + 6, + 1, + 30, + enrollment.manual_entry_key.as_bytes().to_vec(), + )?; + + let valid_code = totp.generate_current()?; + let backup_codes = mfa_manager + .complete_enrollment(enrollment.session_id, user_id, &valid_code) + .await?; + + // Test backup code usage + let first_backup_code = backup_codes[0].code.expose_secret(); + let is_valid = mfa_manager + .verify_backup_code(user_id, first_backup_code, Some("127.0.0.1".to_string())) + .await?; + + assert!(is_valid, "Valid backup code should verify successfully"); + println!("✓ Backup code verified and consumed"); + + // Verify backup code count decreased + let status = mfa_manager.get_backup_codes_status(user_id).await?; + assert_eq!(status.remaining, 9, "Should have 9 remaining backup codes"); + assert_eq!(status.used, 1, "Should have 1 used backup code"); + println!("✓ Backup code status updated (9 remaining, 1 used)"); + + // Test reusing same backup code (should fail) + let is_valid = mfa_manager + .verify_backup_code(user_id, first_backup_code, Some("127.0.0.1".to_string())) + .await?; + + assert!(!is_valid, "Used backup code should not verify again"); + println!("✓ Reused backup code rejected"); + + // Cleanup + cleanup_test_user(&pool, user_id).await?; + + Ok(()) +} + +#[tokio::test] +async fn test_mfa_account_lockout() -> Result<()> { + let pool = setup_test_db().await?; + let user_id = create_test_admin_user(&pool).await?; + + let encryption_key = "test_encryption_key_32_bytes_long!!".to_string(); + let mfa_manager = MfaManager::new(pool.clone(), encryption_key)?; + + // Enroll user in MFA + let enrollment = mfa_manager + .start_enrollment(user_id, "Foxhunt Test", "test@test.local") + .await?; + + use totp_rs::{Algorithm, TOTP}; + let totp = TOTP::new( + Algorithm::SHA1, + 6, + 1, + 30, + enrollment.manual_entry_key.as_bytes().to_vec(), + )?; + + let valid_code = totp.generate_current()?; + let _backup_codes = mfa_manager + .complete_enrollment(enrollment.session_id, user_id, &valid_code) + .await?; + + // Make 5 failed attempts to trigger lockout + for i in 1..=5 { + let result = mfa_manager + .verify_totp(user_id, "000000", Some("127.0.0.1".to_string())) + .await; + + if i < 5 { + assert!( + result.is_ok(), + "Failed attempt {} should not lock account yet", + i + ); + println!("✓ Failed attempt {} recorded", i); + } + } + + // Check if account is locked + let is_locked = mfa_manager.is_mfa_locked(user_id).await?; + assert!( + is_locked, + "Account should be locked after 5 failed attempts" + ); + println!("✓ Account locked after 5 failed attempts"); + + // Verify locked account cannot authenticate even with valid code + let new_code = totp.generate_current()?; + let locked_result = mfa_manager + .verify_totp(user_id, &new_code, Some("127.0.0.1".to_string())) + .await; + + assert!( + locked_result.is_err(), + "Locked account should not allow authentication" + ); + assert!( + locked_result.unwrap_err().to_string().contains("locked"), + "Error should mention account lock" + ); + println!("✓ Locked account rejected valid TOTP code"); + + // Cleanup + cleanup_test_user(&pool, user_id).await?; + + Ok(()) +} + +#[tokio::test] +async fn test_mfa_admin_enforcement() -> Result<()> { + let pool = setup_test_db().await?; + let user_id = create_test_admin_user(&pool).await?; + + // Test 1: Verify MFA is required for admin user + let is_required: bool = sqlx::query_scalar("SELECT is_mfa_required($1)") + .bind(user_id) + .fetch_one(&pool) + .await?; + + assert!(is_required, "MFA should be required for admin user"); + println!("✓ MFA requirement detected for admin user"); + + // Test 2: Verify session creation is blocked without MFA + let session_id = Uuid::new_v4(); + let token_hash = format!("test_token_{}", Uuid::new_v4()); + + let result = sqlx::query( + r#" + INSERT INTO sessions ( + id, token_hash, user_id, expires_at, client_ip, session_type + ) VALUES ($1, $2, $3, NOW() + INTERVAL '1 hour', '127.0.0.1'::inet, 'test') + "#, + ) + .bind(session_id) + .bind(token_hash) + .bind(user_id) + .execute(&pool) + .await; + + assert!(result.is_err(), "Session creation should fail without MFA"); + let error_msg = result.unwrap_err().to_string(); + assert!( + error_msg.contains("MFA_REQUIRED"), + "Error should indicate MFA requirement" + ); + println!("✓ Session creation blocked without MFA enrollment"); + + // Test 3: Enroll in MFA and verify session creation succeeds + let encryption_key = "test_encryption_key_32_bytes_long!!".to_string(); + let mfa_manager = MfaManager::new(pool.clone(), encryption_key)?; + + let enrollment = mfa_manager + .start_enrollment(user_id, "Foxhunt Test", "test@test.local") + .await?; + + use totp_rs::{Algorithm, TOTP}; + let totp = TOTP::new( + Algorithm::SHA1, + 6, + 1, + 30, + enrollment.manual_entry_key.as_bytes().to_vec(), + )?; + + let valid_code = totp.generate_current()?; + let _backup_codes = mfa_manager + .complete_enrollment(enrollment.session_id, user_id, &valid_code) + .await?; + + println!("✓ MFA enrollment completed"); + + // Now session creation should succeed + let session_id = Uuid::new_v4(); + let token_hash = format!("test_token_{}", Uuid::new_v4()); + + let result = sqlx::query( + r#" + INSERT INTO sessions ( + id, token_hash, user_id, expires_at, client_ip, session_type + ) VALUES ($1, $2, $3, NOW() + INTERVAL '1 hour', '127.0.0.1'::inet, 'test') + "#, + ) + .bind(session_id) + .bind(token_hash) + .bind(user_id) + .execute(&pool) + .await; + + assert!( + result.is_ok(), + "Session creation should succeed with MFA enrolled" + ); + println!("✓ Session creation allowed after MFA enrollment"); + + // Cleanup + cleanup_test_user(&pool, user_id).await?; + + Ok(()) +} diff --git a/services/api/tests/ml_trading_integration_tests.rs b/services/api/tests/ml_trading_integration_tests.rs new file mode 100644 index 000000000..b4df15a66 --- /dev/null +++ b/services/api/tests/ml_trading_integration_tests.rs @@ -0,0 +1,922 @@ +//! ML Trading Integration Tests +//! +//! End-to-end tests for ML trading flow through API Gateway: +//! 1. API Gateway receives ML trading request from client +//! 2. API Gateway validates JWT and permissions +//! 3. API Gateway proxies request to Trading Service +//! 4. Trading Service processes ML ensemble prediction +//! 5. Trading Service returns response with order details +//! 6. API Gateway forwards response to client +//! +//! Test Coverage: +//! - SubmitMLOrder: Submit orders based on ensemble predictions +//! - GetMLPredictions: Query prediction history with filters +//! - GetMLPerformance: Get model performance metrics +//! - Permission checks: trading.submit, trading.view +//! - Rate limiting: 100 req/min for ML operations +//! - Error handling: invalid inputs, backend failures +//! +//! Requirements: +//! - API Gateway running on localhost:50051 +//! - Trading Service running on localhost:50052 +//! - PostgreSQL with ensemble_predictions table +//! - Redis for rate limiting + +#[path = "common/mod.rs"] +mod common; + +use anyhow::Result; +use common::{cleanup_redis, generate_test_token, wait_for_redis}; +use std::time::Instant; +use tonic::transport::Channel; +use tonic::{Code, Request}; + +// Import Trading Service proto +use api::trading_backend::{ + trading_service_client::TradingServiceClient, MlOrderRequest, MlPerformanceRequest, + MlPredictionsRequest, +}; + +const REDIS_URL: &str = "redis://localhost:6379"; + +// ============================================================================ +// SECTION 1: ML ORDER SUBMISSION TESTS (5 tests) +// ============================================================================ + +#[tokio::test] +async fn test_submit_ml_order_success() -> Result<()> { + println!("\n=== Test: Submit ML Order - Success ==="); + + // Setup: Connect to Trading Service backend (simulating API Gateway proxy) + let channel = Channel::from_static("http://localhost:50052") + .connect_timeout(std::time::Duration::from_secs(5)) + .connect() + .await; + + if channel.is_err() { + println!("⚠️ Trading Service not running on localhost:50052"); + println!(" This is an integration test - skipping"); + return Ok(()); + } + + let mut client = TradingServiceClient::new(channel.unwrap()); + + // Create ML order request + let request = Request::new(MlOrderRequest { + symbol: "ES.FUT".to_string(), + account_id: "test_account_ml_001".to_string(), + use_ensemble: true, + model_name: None, // Use ensemble mode + features: vec![ + // 26 features: 5 OHLCV + 10 technical + 11 microstructure + 100.0, 101.0, 99.0, 100.5, 1000.0, // OHLCV + 50.0, 0.5, 1.0, 1.5, 2.0, // RSI, MACD, BB, ATR, EMA + 0.2, 0.3, 0.4, 0.5, 0.6, // Stoch, CCI, ADX, OBV, VWAP + 100.2, 100.1, 500.0, 100.0, 99.9, 450.0, // Order book levels + 100.15, 0.05, 0.1, 10.0, 15.0, // Spread, imbalance, urgency, depth + ], + }); + + let start = Instant::now(); + let response = client.submit_ml_order(request).await; + let elapsed = start.elapsed(); + + println!(" Response time: {:?}", elapsed); + + assert!(response.is_ok(), "ML order submission should succeed"); + + let order_response = response.unwrap().into_inner(); + + println!(" ✓ Order ID: {}", order_response.order_id); + println!(" ✓ Prediction ID: {}", order_response.prediction_id); + println!(" ✓ Action: {}", order_response.action); + println!(" ✓ Confidence: {:.2}%", order_response.confidence * 100.0); + println!(" ✓ Executed: {}", order_response.executed); + + // Assertions + assert!(!order_response.order_id.is_empty() || order_response.action == "HOLD"); + assert!(!order_response.prediction_id.is_empty()); + assert!(["BUY", "SELL", "HOLD"].contains(&order_response.action.as_str())); + assert!(order_response.confidence >= 0.0 && order_response.confidence <= 1.0); + + Ok(()) +} + +#[tokio::test] +async fn test_submit_ml_order_specific_model() -> Result<()> { + println!("\n=== Test: Submit ML Order - Specific Model (DQN) ==="); + + let channel = Channel::from_static("http://localhost:50052") + .connect_timeout(std::time::Duration::from_secs(5)) + .connect() + .await; + + if channel.is_err() { + println!("⚠️ Trading Service not running - skipping"); + return Ok(()); + } + + let mut client = TradingServiceClient::new(channel.unwrap()); + + let request = Request::new(MlOrderRequest { + symbol: "NQ.FUT".to_string(), + account_id: "test_account_ml_002".to_string(), + use_ensemble: false, + model_name: Some("DQN".to_string()), // Use specific model + features: vec![0.0; 26], // Placeholder features + }); + + let response = client.submit_ml_order(request).await; + + if response.is_ok() { + let order_response = response.unwrap().into_inner(); + println!(" ✓ Model used: DQN"); + println!(" ✓ Action: {}", order_response.action); + println!(" ✓ Confidence: {:.2}%", order_response.confidence * 100.0); + assert!(!order_response.prediction_id.is_empty()); + } else { + // Model might not be loaded yet - this is acceptable in dev + println!(" ⚠️ DQN model not loaded (expected in development)"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_submit_ml_order_invalid_symbol() -> Result<()> { + println!("\n=== Test: Submit ML Order - Invalid Symbol ==="); + + let channel = Channel::from_static("http://localhost:50052") + .connect_timeout(std::time::Duration::from_secs(5)) + .connect() + .await; + + if channel.is_err() { + println!("⚠️ Trading Service not running - skipping"); + return Ok(()); + } + + let mut client = TradingServiceClient::new(channel.unwrap()); + + let request = Request::new(MlOrderRequest { + symbol: "INVALID_SYM".to_string(), + account_id: "test_account_ml_003".to_string(), + use_ensemble: true, + model_name: None, + features: vec![0.0; 26], + }); + + let response = client.submit_ml_order(request).await; + + // Should either fail validation or return HOLD + if let Ok(order_response) = response { + let inner = order_response.into_inner(); + println!(" ✓ Invalid symbol handled: action={}", inner.action); + // Typically returns HOLD for invalid/unsupported symbols + assert_eq!(inner.action, "HOLD"); + } else { + println!(" ✓ Invalid symbol rejected by validation"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_submit_ml_order_wrong_feature_count() -> Result<()> { + println!("\n=== Test: Submit ML Order - Wrong Feature Count ==="); + + let channel = Channel::from_static("http://localhost:50052") + .connect_timeout(std::time::Duration::from_secs(5)) + .connect() + .await; + + if channel.is_err() { + println!("⚠️ Trading Service not running - skipping"); + return Ok(()); + } + + let mut client = TradingServiceClient::new(channel.unwrap()); + + let request = Request::new(MlOrderRequest { + symbol: "ES.FUT".to_string(), + account_id: "test_account_ml_004".to_string(), + use_ensemble: true, + model_name: None, + features: vec![0.0; 10], // Wrong count (should be 26) + }); + + let response = client.submit_ml_order(request).await; + + // Should fail with InvalidArgument + if let Err(status) = response { + println!(" ✓ Wrong feature count rejected"); + println!(" ✓ Error: {}", status.message()); + assert_eq!(status.code(), Code::InvalidArgument); + } else { + // Fallback handler might return HOLD + let inner = response.unwrap().into_inner(); + println!(" ✓ Fallback returned HOLD for invalid features"); + assert_eq!(inner.action, "HOLD"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_submit_ml_order_empty_account_id() -> Result<()> { + println!("\n=== Test: Submit ML Order - Empty Account ID ==="); + + let channel = Channel::from_static("http://localhost:50052") + .connect_timeout(std::time::Duration::from_secs(5)) + .connect() + .await; + + if channel.is_err() { + println!("⚠️ Trading Service not running - skipping"); + return Ok(()); + } + + let mut client = TradingServiceClient::new(channel.unwrap()); + + let request = Request::new(MlOrderRequest { + symbol: "ES.FUT".to_string(), + account_id: "".to_string(), // Empty account ID + use_ensemble: true, + model_name: None, + features: vec![0.0; 26], + }); + + let response = client.submit_ml_order(request).await; + + // Should fail validation + assert!(response.is_err(), "Empty account ID should be rejected"); + + if let Err(status) = response { + println!(" ✓ Empty account ID rejected"); + println!(" ✓ Error code: {:?}", status.code()); + assert_eq!(status.code(), Code::InvalidArgument); + } + + Ok(()) +} + +// ============================================================================ +// SECTION 2: ML PREDICTIONS QUERY TESTS (3 tests) +// ============================================================================ + +#[tokio::test] +async fn test_get_ml_predictions_with_filters() -> Result<()> { + println!("\n=== Test: Get ML Predictions - With Filters ==="); + + let channel = Channel::from_static("http://localhost:50052") + .connect_timeout(std::time::Duration::from_secs(5)) + .connect() + .await; + + if channel.is_err() { + println!("⚠️ Trading Service not running - skipping"); + return Ok(()); + } + + let mut client = TradingServiceClient::new(channel.unwrap()); + + let request = Request::new(MlPredictionsRequest { + symbol: "ES.FUT".to_string(), + model_name: Some("DQN".to_string()), // Filter by model + limit: 5, + start_time: None, + end_time: None, + }); + + let response = client.get_ml_predictions(request).await; + + if response.is_ok() { + let predictions_response = response.unwrap().into_inner(); + + println!( + " ✓ Predictions retrieved: {} records", + predictions_response.predictions.len() + ); + + // Verify pagination limit + assert!(predictions_response.predictions.len() <= 5); + + // Verify all predictions match filter + for pred in &predictions_response.predictions { + println!( + " - ID: {}, Symbol: {}, Action: {}, Confidence: {:.2}%", + pred.id, + pred.symbol, + pred.ensemble_action, + pred.ensemble_confidence * 100.0 + ); + + assert_eq!(pred.symbol, "ES.FUT"); + assert!(pred.ensemble_confidence >= 0.0 && pred.ensemble_confidence <= 1.0); + } + } else { + println!(" ⚠️ No predictions found (expected in fresh database)"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_get_ml_predictions_all_models() -> Result<()> { + println!("\n=== Test: Get ML Predictions - All Models ==="); + + let channel = Channel::from_static("http://localhost:50052") + .connect_timeout(std::time::Duration::from_secs(5)) + .connect() + .await; + + if channel.is_err() { + println!("⚠️ Trading Service not running - skipping"); + return Ok(()); + } + + let mut client = TradingServiceClient::new(channel.unwrap()); + + let request = Request::new(MlPredictionsRequest { + symbol: "ES.FUT".to_string(), + model_name: None, // All models + limit: 10, + start_time: None, + end_time: None, + }); + + let response = client.get_ml_predictions(request).await; + + if response.is_ok() { + let predictions_response = response.unwrap().into_inner(); + + println!( + " ✓ Total predictions: {}", + predictions_response.predictions.len() + ); + + // Verify data structure + for pred in predictions_response.predictions.iter().take(3) { + println!(" Prediction ID: {}", pred.id); + println!( + " Ensemble: {} (signal={:.2}, confidence={:.2}%)", + pred.ensemble_action, + pred.ensemble_signal, + pred.ensemble_confidence * 100.0 + ); + + if !pred.model_predictions.is_empty() { + println!(" Individual models:"); + for model_pred in &pred.model_predictions { + println!( + " - {}: signal={:.2}, confidence={:.2}%", + model_pred.model_name, + model_pred.signal, + model_pred.confidence * 100.0 + ); + } + } + } + } else { + println!(" ⚠️ No predictions found (expected in fresh database)"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_get_ml_predictions_time_range() -> Result<()> { + println!("\n=== Test: Get ML Predictions - Time Range Filter ==="); + + let channel = Channel::from_static("http://localhost:50052") + .connect_timeout(std::time::Duration::from_secs(5)) + .connect() + .await; + + if channel.is_err() { + println!("⚠️ Trading Service not running - skipping"); + return Ok(()); + } + + let mut client = TradingServiceClient::new(channel.unwrap()); + + // Query last 24 hours + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_nanos() as i64; + let one_day_ago = now - (24 * 3600 * 1_000_000_000); + + let request = Request::new(MlPredictionsRequest { + symbol: "ES.FUT".to_string(), + model_name: None, + limit: 100, + start_time: Some(one_day_ago), + end_time: Some(now), + }); + + let response = client.get_ml_predictions(request).await; + + if response.is_ok() { + let predictions_response = response.unwrap().into_inner(); + + println!( + " ✓ Predictions in last 24h: {}", + predictions_response.predictions.len() + ); + + // Verify all timestamps are within range + for pred in &predictions_response.predictions { + assert!(pred.timestamp >= one_day_ago); + assert!(pred.timestamp <= now); + } + } else { + println!(" ⚠️ No predictions in last 24h (expected in dev)"); + } + + Ok(()) +} + +// ============================================================================ +// SECTION 3: ML PERFORMANCE METRICS TESTS (3 tests) +// ============================================================================ + +#[tokio::test] +async fn test_get_ml_performance_all_models() -> Result<()> { + println!("\n=== Test: Get ML Performance - All Models ==="); + + let channel = Channel::from_static("http://localhost:50052") + .connect_timeout(std::time::Duration::from_secs(5)) + .connect() + .await; + + if channel.is_err() { + println!("⚠️ Trading Service not running - skipping"); + return Ok(()); + } + + let mut client = TradingServiceClient::new(channel.unwrap()); + + let request = Request::new(MlPerformanceRequest { + model_name: None, // All models + start_time: None, + end_time: None, + }); + + let response = client.get_ml_performance(request).await; + + if response.is_ok() { + let performance_response = response.unwrap().into_inner(); + + println!(" ✓ Models tracked: {}", performance_response.models.len()); + + for model in &performance_response.models { + println!("\n Model: {}", model.model_name); + println!(" Total predictions: {}", model.total_predictions); + println!(" Correct predictions: {}", model.correct_predictions); + println!(" Accuracy: {:.2}%", model.accuracy * 100.0); + println!(" Sharpe ratio: {:.2}", model.sharpe_ratio); + println!(" Avg P&L: ${:.2}", model.avg_pnl); + + // Validate metrics ranges + assert!(model.accuracy >= 0.0 && model.accuracy <= 1.0); + assert!(model.total_predictions >= 0); + assert!(model.correct_predictions >= 0); + assert!(model.correct_predictions <= model.total_predictions); + } + } else { + println!(" ⚠️ No performance data (expected in fresh database)"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_get_ml_performance_specific_model() -> Result<()> { + println!("\n=== Test: Get ML Performance - Specific Model (MAMBA_2) ==="); + + let channel = Channel::from_static("http://localhost:50052") + .connect_timeout(std::time::Duration::from_secs(5)) + .connect() + .await; + + if channel.is_err() { + println!("⚠️ Trading Service not running - skipping"); + return Ok(()); + } + + let mut client = TradingServiceClient::new(channel.unwrap()); + + let request = Request::new(MlPerformanceRequest { + model_name: Some("MAMBA_2".to_string()), + start_time: None, + end_time: None, + }); + + let response = client.get_ml_performance(request).await; + + if response.is_ok() { + let performance_response = response.unwrap().into_inner(); + + // Should only return MAMBA_2 stats + if !performance_response.models.is_empty() { + assert_eq!(performance_response.models.len(), 1); + assert_eq!(performance_response.models[0].model_name, "MAMBA_2"); + + println!(" ✓ MAMBA_2 Performance:"); + println!( + " Total predictions: {}", + performance_response.models[0].total_predictions + ); + println!( + " Accuracy: {:.2}%", + performance_response.models[0].accuracy * 100.0 + ); + } else { + println!(" ⚠️ MAMBA_2 has no predictions yet (expected)"); + } + } else { + println!(" ⚠️ MAMBA_2 performance data not available"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_get_ml_performance_time_range() -> Result<()> { + println!("\n=== Test: Get ML Performance - Time Range ==="); + + let channel = Channel::from_static("http://localhost:50052") + .connect_timeout(std::time::Duration::from_secs(5)) + .connect() + .await; + + if channel.is_err() { + println!("⚠️ Trading Service not running - skipping"); + return Ok(()); + } + + let mut client = TradingServiceClient::new(channel.unwrap()); + + // Last 7 days + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH)? + .as_nanos() as i64; + let seven_days_ago = now - (7 * 24 * 3600 * 1_000_000_000); + + let request = Request::new(MlPerformanceRequest { + model_name: None, + start_time: Some(seven_days_ago), + end_time: Some(now), + }); + + let response = client.get_ml_performance(request).await; + + if response.is_ok() { + let performance_response = response.unwrap().into_inner(); + + println!(" ✓ Performance metrics for last 7 days:"); + println!(" Models tracked: {}", performance_response.models.len()); + + for model in &performance_response.models { + println!( + " - {}: {} predictions, {:.2}% accuracy", + model.model_name, + model.total_predictions, + model.accuracy * 100.0 + ); + } + } else { + println!(" ⚠️ No performance data in last 7 days"); + } + + Ok(()) +} + +// ============================================================================ +// SECTION 4: PERMISSION & RATE LIMITING TESTS (4 tests) +// ============================================================================ + +#[tokio::test] +async fn test_ml_order_requires_trading_submit_permission() -> Result<()> { + println!("\n=== Test: ML Order - Permission Check ==="); + + // Generate token WITHOUT trading.submit permission + let (token, _jti) = generate_test_token( + "user_viewer", + vec!["viewer".to_string()], + vec!["trading.view".to_string()], // Only view permission + 3600, + )?; + + println!(" Token scopes: [trading.view] (missing trading.submit)"); + println!(" ✓ In production, API Gateway would reject this with PermissionDenied"); + println!(" ✓ Direct backend call simulates post-authorization"); + + // Note: This test validates the auth flow at API Gateway level + // The Trading Service backend assumes authorization already passed + // Integration tests with full API Gateway stack would validate permissions + + Ok(()) +} + +#[tokio::test] +async fn test_get_ml_predictions_requires_view_permission() -> Result<()> { + println!("\n=== Test: Get ML Predictions - Permission Check ==="); + + // Generate token WITH trading.view permission + let (token, _jti) = generate_test_token( + "user_analyst", + vec!["analyst".to_string()], + vec!["trading.view".to_string()], + 3600, + )?; + + println!(" Token scopes: [trading.view]"); + println!(" ✓ Should allow GetMLPredictions"); + println!(" ✓ Should allow GetMLPerformance"); + println!(" ✗ Should NOT allow SubmitMLOrder (requires trading.submit)"); + + Ok(()) +} + +#[tokio::test] +async fn test_rate_limiting_ml_operations() -> Result<()> { + println!("\n=== Test: Rate Limiting - ML Operations ==="); + + wait_for_redis(REDIS_URL, 50).await?; + cleanup_redis(REDIS_URL).await?; + + let channel = Channel::from_static("http://localhost:50052") + .connect_timeout(std::time::Duration::from_secs(5)) + .connect() + .await; + + if channel.is_err() { + println!("⚠️ Trading Service not running - skipping rate limit test"); + return Ok(()); + } + + let mut client = TradingServiceClient::new(channel.unwrap()); + + // Simulate burst of 105 requests (limit is 100/min) + println!(" Sending 105 rapid ML order requests..."); + let mut success_count = 0; + let mut rate_limited_count = 0; + + let start = Instant::now(); + + for i in 0..105 { + let request = Request::new(MlOrderRequest { + symbol: "ES.FUT".to_string(), + account_id: format!("rate_test_{}", i), + use_ensemble: true, + model_name: None, + features: vec![0.0; 26], + }); + + let result = client.submit_ml_order(request).await; + + if result.is_ok() { + success_count += 1; + } else if let Err(status) = result { + if status.code() == Code::ResourceExhausted { + rate_limited_count += 1; + } + } + } + + let elapsed = start.elapsed(); + + println!("\n Results:"); + println!(" Successful requests: {}", success_count); + println!(" Rate limited: {}", rate_limited_count); + println!(" Total time: {:?}", elapsed); + + // Note: Rate limiting is enforced at API Gateway level + // Direct backend calls bypass rate limiting + println!(" ✓ Backend accepts all requests (rate limiting at API Gateway)"); + + Ok(()) +} + +#[tokio::test] +async fn test_concurrent_ml_requests_different_accounts() -> Result<()> { + println!("\n=== Test: Concurrent ML Requests - Different Accounts ==="); + + let channel = Channel::from_static("http://localhost:50052") + .connect_timeout(std::time::Duration::from_secs(5)) + .connect() + .await; + + if channel.is_err() { + println!("⚠️ Trading Service not running - skipping"); + return Ok(()); + } + + let client = TradingServiceClient::new(channel.unwrap()); + + // Spawn 10 concurrent ML order requests from different accounts + let mut handles = vec![]; + + println!(" Spawning 10 concurrent ML orders..."); + + for i in 0..10 { + let mut client_clone = client.clone(); + let handle = tokio::spawn(async move { + let request = Request::new(MlOrderRequest { + symbol: "ES.FUT".to_string(), + account_id: format!("concurrent_test_{}", i), + use_ensemble: true, + model_name: None, + features: vec![0.0; 26], + }); + + client_clone.submit_ml_order(request).await + }); + handles.push(handle); + } + + // Wait for all requests to complete + let mut success_count = 0; + for handle in handles { + if let Ok(result) = handle.await { + if result.is_ok() { + success_count += 1; + } + } + } + + println!( + " ✓ Concurrent requests completed: {}/10 succeeded", + success_count + ); + assert!( + success_count >= 8, + "At least 80% of concurrent requests should succeed" + ); + + Ok(()) +} + +// ============================================================================ +// SECTION 5: ERROR HANDLING & EDGE CASES (3 tests) +// ============================================================================ + +#[tokio::test] +async fn test_ml_order_with_nan_features() -> Result<()> { + println!("\n=== Test: ML Order - NaN Features ==="); + + let channel = Channel::from_static("http://localhost:50052") + .connect_timeout(std::time::Duration::from_secs(5)) + .connect() + .await; + + if channel.is_err() { + println!("⚠️ Trading Service not running - skipping"); + return Ok(()); + } + + let mut client = TradingServiceClient::new(channel.unwrap()); + + let request = Request::new(MlOrderRequest { + symbol: "ES.FUT".to_string(), + account_id: "test_nan".to_string(), + use_ensemble: true, + model_name: None, + features: vec![f64::NAN; 26], // Invalid NaN features + }); + + let response = client.submit_ml_order(request).await; + + // Should either reject or return HOLD + if let Err(status) = response { + println!(" ✓ NaN features rejected: {}", status.message()); + assert_eq!(status.code(), Code::InvalidArgument); + } else { + let inner = response.unwrap().into_inner(); + println!( + " ✓ NaN features handled gracefully: action={}", + inner.action + ); + assert_eq!(inner.action, "HOLD"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_ml_order_with_infinite_features() -> Result<()> { + println!("\n=== Test: ML Order - Infinite Features ==="); + + let channel = Channel::from_static("http://localhost:50052") + .connect_timeout(std::time::Duration::from_secs(5)) + .connect() + .await; + + if channel.is_err() { + println!("⚠️ Trading Service not running - skipping"); + return Ok(()); + } + + let mut client = TradingServiceClient::new(channel.unwrap()); + + let request = Request::new(MlOrderRequest { + symbol: "ES.FUT".to_string(), + account_id: "test_inf".to_string(), + use_ensemble: true, + model_name: None, + features: vec![f64::INFINITY; 26], // Invalid infinite features + }); + + let response = client.submit_ml_order(request).await; + + // Should either reject or return HOLD + if let Err(status) = response { + println!(" ✓ Infinite features rejected: {}", status.message()); + assert_eq!(status.code(), Code::InvalidArgument); + } else { + let inner = response.unwrap().into_inner(); + println!(" ✓ Infinite features handled: action={}", inner.action); + assert_eq!(inner.action, "HOLD"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_backend_connection_failure_handling() -> Result<()> { + println!("\n=== Test: Backend Connection Failure ==="); + + // Try to connect to non-existent backend + let channel = Channel::from_static("http://localhost:59999") // Wrong port + .connect_timeout(std::time::Duration::from_millis(500)) + .connect() + .await; + + assert!( + channel.is_err(), + "Connection to non-existent backend should fail" + ); + + if let Err(e) = channel { + println!(" ✓ Connection failure handled gracefully"); + println!(" ✓ Error: {}", e); + } + + // In production, API Gateway circuit breaker would: + // 1. Detect repeated failures + // 2. Open circuit after threshold (e.g., 5 failures) + // 3. Return 503 Service Unavailable to clients + // 4. Attempt recovery after timeout (e.g., 30s) + + println!(" ✓ Circuit breaker would prevent cascade failures"); + + Ok(()) +} + +// ============================================================================ +// TEST SUMMARY +// ============================================================================ + +#[tokio::test] +async fn test_summary() { + println!("\n"); + println!("╔═══════════════════════════════════════════════════════════════════╗"); + println!("║ ML TRADING INTEGRATION TEST SUITE SUMMARY ║"); + println!("╠═══════════════════════════════════════════════════════════════════╣"); + println!("║ ║"); + println!("║ Section 1: ML Order Submission (5 tests) ║"); + println!("║ ✓ Submit ML order - success ║"); + println!("║ ✓ Submit ML order - specific model ║"); + println!("║ ✓ Submit ML order - invalid symbol ║"); + println!("║ ✓ Submit ML order - wrong feature count ║"); + println!("║ ✓ Submit ML order - empty account ID ║"); + println!("║ ║"); + println!("║ Section 2: ML Predictions Query (3 tests) ║"); + println!("║ ✓ Get predictions with filters ║"); + println!("║ ✓ Get predictions - all models ║"); + println!("║ ✓ Get predictions - time range ║"); + println!("║ ║"); + println!("║ Section 3: ML Performance Metrics (3 tests) ║"); + println!("║ ✓ Get performance - all models ║"); + println!("║ ✓ Get performance - specific model ║"); + println!("║ ✓ Get performance - time range ║"); + println!("║ ║"); + println!("║ Section 4: Permission & Rate Limiting (4 tests) ║"); + println!("║ ✓ ML order requires trading.submit permission ║"); + println!("║ ✓ Get predictions requires trading.view permission ║"); + println!("║ ✓ Rate limiting - ML operations ║"); + println!("║ ✓ Concurrent requests - different accounts ║"); + println!("║ ║"); + println!("║ Section 5: Error Handling & Edge Cases (3 tests) ║"); + println!("║ ✓ ML order with NaN features ║"); + println!("║ ✓ ML order with infinite features ║"); + println!("║ ✓ Backend connection failure ║"); + println!("║ ║"); + println!("╠═══════════════════════════════════════════════════════════════════╣"); + println!("║ TOTAL TESTS: 18 ║"); + println!("║ COVERAGE: ML Trading Flow (API Gateway → Trading Service) ║"); + println!("║ REQUIREMENTS: API Gateway + Trading Service + PostgreSQL + Redis║"); + println!("╚═══════════════════════════════════════════════════════════════════╝"); + println!(); +} diff --git a/services/api/tests/proxy_latency_test.rs b/services/api/tests/proxy_latency_test.rs new file mode 100644 index 000000000..03de910e7 --- /dev/null +++ b/services/api/tests/proxy_latency_test.rs @@ -0,0 +1,263 @@ +//! API Gateway Proxy Latency Test +//! +//! Quick validation test for proxy latency against <1ms target +//! Measures REAL gRPC calls: API Gateway (50051) → Trading Service (50052) + +#[path = "common/mod.rs"] +mod common; + +use anyhow::Result; +use std::time::Instant; +use tonic::{metadata::MetadataValue, Request}; +use uuid::Uuid; + +use api::foxhunt::tli::{ + trading_service_client::TradingServiceClient, OrderSide, OrderType, SubmitOrderRequest, +}; + +/// Create authenticated request with JWT +fn create_test_request() -> Request { + let (token, _jti) = common::generate_test_token( + "bench-user", + vec!["trader".to_string()], + vec!["trading.submit_order".to_string()], + 3600, + ) + .unwrap(); + + let order = SubmitOrderRequest { + symbol: "BTC/USD".to_string(), + side: OrderSide::Buy as i32, + order_type: OrderType::Limit as i32, + quantity: 1.0, + price: Some(50000.0), + stop_price: None, + time_in_force: "GTC".to_string(), + client_order_id: Uuid::new_v4().to_string(), + }; + + let mut request = Request::new(order); + let auth_value = MetadataValue::try_from(format!("Bearer {}", token)).unwrap(); + request.metadata_mut().insert("authorization", auth_value); + + request +} + +#[tokio::test] +#[ignore = "Run manually: cargo test -p api proxy_latency --ignored -- --nocapture"] +async fn test_proxy_cold_start_latency() -> Result<()> { + println!("\n=== Test 1: Cold Start Latency ==="); + + let mut latencies = Vec::new(); + + // Measure 10 cold starts + for i in 0..10 { + let start = Instant::now(); + + let mut client = TradingServiceClient::connect("http://localhost:50051").await?; + let request = create_test_request(); + + let _ = client.submit_order(request).await; + + let elapsed = start.elapsed(); + latencies.push(elapsed); + + println!(" Cold start {}: {:?}", i + 1, elapsed); + } + + latencies.sort(); + let median = latencies[latencies.len() / 2]; + let p99 = latencies[(latencies.len() as f64 * 0.99) as usize]; + + println!("\n 📊 Cold Start Statistics:"); + println!(" Median: {:?}", median); + println!(" P99: {:?}", p99); + println!(" Target: <10ms (cold start allowance)"); + + assert!( + p99.as_millis() < 10, + "P99 cold start latency {} ms exceeds 10ms target", + p99.as_millis() + ); + + Ok(()) +} + +#[tokio::test] +#[ignore = "Run manually: cargo test -p api proxy_latency --ignored -- --nocapture"] +async fn test_proxy_warm_cache_latency() -> Result<()> { + println!("\n=== Test 2: Warm Cache Latency ==="); + + // Setup persistent connection + let mut client = TradingServiceClient::connect("http://localhost:50051").await?; + + // Warmup: 100 requests + println!(" Warming up with 100 requests..."); + for _ in 0..100 { + let request = create_test_request(); + let _ = client.submit_order(request).await; + } + + // Measure 1000 warm requests + let mut latencies = Vec::new(); + println!(" Measuring 1000 warm requests..."); + + for _ in 0..1000 { + let start = Instant::now(); + + let request = create_test_request(); + let _ = client.submit_order(request).await; + + latencies.push(start.elapsed()); + } + + latencies.sort(); + let p50 = latencies[latencies.len() / 2]; + let p95 = latencies[(latencies.len() as f64 * 0.95) as usize]; + let p99 = latencies[(latencies.len() as f64 * 0.99) as usize]; + let min = latencies[0]; + let max = latencies[latencies.len() - 1]; + + println!("\n 📊 Warm Cache Statistics:"); + println!(" Min: {:>8} μs", min.as_micros()); + println!(" P50: {:>8} μs", p50.as_micros()); + println!(" P95: {:>8} μs", p95.as_micros()); + println!(" P99: {:>8} μs", p99.as_micros()); + println!(" Max: {:>8} μs", max.as_micros()); + println!(" Target: < 1,000 μs (1ms)"); + println!("\n Wave 132 Baseline: 21-488μs warm"); + + if p99.as_micros() < 1000 { + println!(" ✅ PASS: P99 {} μs < 1ms target", p99.as_micros()); + } else { + println!(" ❌ FAIL: P99 {} μs >= 1ms target", p99.as_micros()); + } + + assert!( + p99.as_micros() < 1000, + "P99 latency {} μs exceeds 1ms target", + p99.as_micros() + ); + + Ok(()) +} + +#[tokio::test] +#[ignore = "Run manually: cargo test -p api proxy_latency --ignored -- --nocapture"] +async fn test_proxy_overhead_comparison() -> Result<()> { + println!("\n=== Test 3: Proxy Overhead (Proxied vs Direct) ==="); + + // Setup both clients + let mut proxy_client = TradingServiceClient::connect("http://localhost:50051").await?; + let mut direct_client = TradingServiceClient::connect("http://localhost:50052").await?; + + // Warmup both + println!(" Warming up proxy and direct clients..."); + for _ in 0..100 { + let r1 = create_test_request(); + let r2 = create_test_request(); + let _ = proxy_client.submit_order(r1).await; + let _ = direct_client.submit_order(r2).await; + } + + // Measure proxy latency + let mut proxy_latencies = Vec::new(); + for _ in 0..1000 { + let start = Instant::now(); + let request = create_test_request(); + let _ = proxy_client.submit_order(request).await; + proxy_latencies.push(start.elapsed()); + } + + // Measure direct latency + let mut direct_latencies = Vec::new(); + for _ in 0..1000 { + let start = Instant::now(); + let request = create_test_request(); + let _ = direct_client.submit_order(request).await; + direct_latencies.push(start.elapsed()); + } + + proxy_latencies.sort(); + direct_latencies.sort(); + + let proxy_p50 = proxy_latencies[proxy_latencies.len() / 2]; + let direct_p50 = direct_latencies[direct_latencies.len() / 2]; + let proxy_p99 = proxy_latencies[(proxy_latencies.len() as f64 * 0.99) as usize]; + let direct_p99 = direct_latencies[(direct_latencies.len() as f64 * 0.99) as usize]; + + let overhead_p50 = proxy_p50.saturating_sub(direct_p50); + let overhead_p99 = proxy_p99.saturating_sub(direct_p99); + + let overhead_percent_p50 = if direct_p50.as_micros() > 0 { + ((overhead_p50.as_micros() as f64 / direct_p50.as_micros() as f64) * 100.0) as u32 + } else { + 0 + }; + + println!("\n 📊 Proxy vs Direct Comparison:"); + println!(" Direct P50: {:>8} μs", direct_p50.as_micros()); + println!(" Proxy P50: {:>8} μs", proxy_p50.as_micros()); + println!( + " Overhead P50: {:>8} μs ({}%)", + overhead_p50.as_micros(), + overhead_percent_p50 + ); + println!(); + println!(" Direct P99: {:>8} μs", direct_p99.as_micros()); + println!(" Proxy P99: {:>8} μs", proxy_p99.as_micros()); + println!(" Overhead P99: {:>8} μs", overhead_p99.as_micros()); + println!("\n Target: Proxy overhead < 100μs"); + + if overhead_p99.as_micros() < 100 { + println!( + " ✅ PASS: Overhead {} μs < 100μs", + overhead_p99.as_micros() + ); + } else { + println!( + " ⚠️ WARNING: Overhead {} μs >= 100μs", + overhead_p99.as_micros() + ); + } + + Ok(()) +} + +#[tokio::test] +#[ignore = "Run manually: cargo test -p api proxy_latency --ignored -- --nocapture"] +async fn test_connection_pool_impact() -> Result<()> { + println!("\n=== Test 4: Connection Pool Impact ==="); + + for concurrency in [1, 10, 50, 100] { + let start = Instant::now(); + + let mut handles = vec![]; + for _ in 0..concurrency { + let handle = tokio::spawn(async move { + let mut client = TradingServiceClient::connect("http://localhost:50051") + .await + .unwrap(); + let request = create_test_request(); + let _ = client.submit_order(request).await; + }); + handles.push(handle); + } + + for handle in handles { + let _ = handle.await; + } + + let elapsed = start.elapsed(); + let avg_per_request = elapsed / concurrency; + + println!( + " Concurrency {:<3}: Total {:>6?}, Avg/req {:>6?}", + concurrency, elapsed, avg_per_request + ); + } + + println!("\n ✅ Connection pool test complete"); + + Ok(()) +} diff --git a/services/api/tests/rate_limiter_advanced_tests.rs b/services/api/tests/rate_limiter_advanced_tests.rs new file mode 100644 index 000000000..ceb54ea41 --- /dev/null +++ b/services/api/tests/rate_limiter_advanced_tests.rs @@ -0,0 +1,654 @@ +//! Advanced Rate Limiter Tests - Wave 17 Agent 17.10 +//! +//! Comprehensive tests for token bucket algorithm, cache management, +//! endpoint configuration, and error handling with Redis backend. +//! +//! Coverage targets: +//! - Token bucket refill mechanics +//! - LRU cache eviction +//! - Endpoint configuration updates +//! - Redis connection handling +//! - Concurrent access patterns + +use anyhow::Result; +use std::time::Duration; +use uuid::Uuid; + +use api::routing::{RateLimitConfig, RateLimiter}; + +const REDIS_URL: &str = "redis://localhost:6379"; + +// ============================================================================ +// Token Bucket Mechanics Tests (5 tests) +// ============================================================================ + +#[tokio::test] +async fn test_token_bucket_capacity_enforcement() -> Result<()> { + println!("\n=== Test: Token Bucket Capacity Enforcement ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // config.update has 10 req/s capacity + let mut allowed = 0; + for _ in 0..20 { + if rate_limiter.check_limit(&user_id, "config.update").await? { + allowed += 1; + } else { + break; + } + } + + println!(" Allowed: {}/20", allowed); + assert!( + allowed >= 10 && allowed <= 12, + "Should allow ~10 requests (capacity)" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_token_bucket_refill_rate() -> Result<()> { + println!("\n=== Test: Token Bucket Refill Rate ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Exhaust bucket + for _ in 0..15 { + let _ = rate_limiter.check_limit(&user_id, "config.update").await?; + } + + // Wait for partial refill (0.5s = ~5 tokens at 10 req/s) + tokio::time::sleep(Duration::from_millis(500)).await; + + let mut refilled = 0; + for _ in 0..10 { + if rate_limiter.check_limit(&user_id, "config.update").await? { + refilled += 1; + } else { + break; + } + } + + println!(" Refilled: {} tokens in 500ms", refilled); + assert!(refilled >= 4 && refilled <= 6, "Should refill ~5 tokens"); + + Ok(()) +} + +#[tokio::test] +async fn test_token_bucket_burst_handling() -> Result<()> { + println!("\n=== Test: Token Bucket Burst Handling ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // trading.submit_order has 100 req/s capacity + let mut burst_allowed = 0; + + // Make burst of 150 requests + for _ in 0..150 { + if rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await? + { + burst_allowed += 1; + } else { + break; + } + } + + println!(" Burst allowed: {}/150", burst_allowed); + assert!( + burst_allowed >= 95 && burst_allowed <= 105, + "Should handle burst up to capacity" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_token_bucket_multiple_endpoints() -> Result<()> { + println!("\n=== Test: Multiple Endpoints Independent Limits ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Exhaust one endpoint + for _ in 0..20 { + let _ = rate_limiter.check_limit(&user_id, "config.update").await?; + } + + // Other endpoint should have full capacity + let mut trading_allowed = 0; + for _ in 0..50 { + if rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await? + { + trading_allowed += 1; + } else { + break; + } + } + + println!(" Trading endpoint allowed: {}", trading_allowed); + assert!(trading_allowed >= 45, "Should have independent limit"); + + Ok(()) +} + +#[tokio::test] +async fn test_token_bucket_slow_refill() -> Result<()> { + println!("\n=== Test: Slow Refill Rate (Backtesting) ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // backtesting.run has 5 req/min (very slow refill) + let mut initial = 0; + for _ in 0..10 { + if rate_limiter + .check_limit(&user_id, "backtesting.run") + .await? + { + initial += 1; + } else { + break; + } + } + + println!(" Initial allowed: {}", initial); + assert!(initial <= 6, "Should be rate limited"); + + // Wait 12 seconds (should refill ~1 token) + tokio::time::sleep(Duration::from_secs(12)).await; + + let mut refilled = 0; + for _ in 0..5 { + if rate_limiter + .check_limit(&user_id, "backtesting.run") + .await? + { + refilled += 1; + } else { + break; + } + } + + println!(" After 12s: {} new tokens", refilled); + assert!(refilled >= 0 && refilled <= 2, "Should refill slowly"); + + Ok(()) +} + +// ============================================================================ +// Cache Management Tests (5 tests) +// ============================================================================ + +#[tokio::test] +async fn test_cache_hit_after_first_check() -> Result<()> { + println!("\n=== Test: Cache Hit After First Check ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // First check (cache miss, hits Redis) + let start = std::time::Instant::now(); + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + let first_duration = start.elapsed(); + + // Second check (cache hit, <8ns expected) + let start = std::time::Instant::now(); + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + let second_duration = start.elapsed(); + + println!(" First check: {:?}", first_duration); + println!(" Second check: {:?}", second_duration); + + // Second check should be much faster + assert!(second_duration < first_duration); + + Ok(()) +} + +#[tokio::test] +async fn test_cache_expiration() -> Result<()> { + println!("\n=== Test: Cache TTL Expiration ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Make initial check + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + + // Wait for cache TTL to expire (1 second) + tokio::time::sleep(Duration::from_millis(1100)).await; + + // Next check should be cache miss (go to Redis) + let start = std::time::Instant::now(); + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + let duration = start.elapsed(); + + println!(" After TTL expiry: {:?}", duration); + // Should hit Redis again + assert!(duration > Duration::from_micros(100)); + + Ok(()) +} + +#[tokio::test] +async fn test_cache_size_limit_and_eviction() -> Result<()> { + println!("\n=== Test: Cache LRU Eviction ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // Fill cache with many different users (10,000+ entries) + println!(" Creating 10,500 cache entries..."); + for i in 0..10_500 { + let user_id = Uuid::new_v4(); + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + + if i % 1000 == 0 { + println!(" Created {} entries", i); + } + } + + let stats = rate_limiter.get_cache_stats().await; + + println!(" Cache size: {}/{}", stats.size, stats.max_size); + println!(" LRU eviction: {} entries removed", 10_500 - stats.size); + + // Cache should not exceed max size + assert!( + stats.size <= stats.max_size, + "Cache should not exceed max size" + ); + assert!(stats.size >= 9_000, "Cache should keep most recent entries"); + + Ok(()) +} + +#[tokio::test] +async fn test_cache_clear_operation() -> Result<()> { + println!("\n=== Test: Cache Clear Operation ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // Populate cache + for _ in 0..100 { + let user_id = Uuid::new_v4(); + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + } + + let stats_before = rate_limiter.get_cache_stats().await; + println!(" Cache before clear: {} entries", stats_before.size); + + // Clear cache + rate_limiter.clear_cache().await; + + let stats_after = rate_limiter.get_cache_stats().await; + println!(" Cache after clear: {} entries", stats_after.size); + + assert_eq!(stats_after.size, 0, "Cache should be empty after clear"); + + Ok(()) +} + +#[tokio::test] +async fn test_cache_concurrent_access() -> Result<()> { + println!("\n=== Test: Cache Concurrent Access ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Spawn 100 concurrent tasks accessing same cache entry + let mut handles = vec![]; + + for _ in 0..100 { + let limiter = rate_limiter.clone(); + let uid = user_id; + + let handle = + tokio::spawn(async move { limiter.check_limit(&uid, "trading.submit_order").await }); + + handles.push(handle); + } + + // Collect results + let mut success_count = 0; + let mut error_count = 0; + + for handle in handles { + match handle.await { + Ok(Ok(_)) => success_count += 1, + Ok(Err(_)) => error_count += 1, + Err(_) => error_count += 1, + } + } + + println!(" Success: {}, Errors: {}", success_count, error_count); + assert_eq!( + success_count + error_count, + 100, + "All tasks should complete" + ); + assert_eq!(error_count, 0, "No errors should occur"); + + Ok(()) +} + +// ============================================================================ +// Endpoint Configuration Tests (5 tests) +// ============================================================================ + +#[tokio::test] +async fn test_default_endpoint_config() -> Result<()> { + println!("\n=== Test: Default Endpoint Configuration ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Unknown endpoint should use default config (50 req/s) + let mut allowed = 0; + for _ in 0..75 { + if rate_limiter + .check_limit(&user_id, "unknown.endpoint") + .await? + { + allowed += 1; + } else { + break; + } + } + + println!(" Default endpoint allowed: {}/75", allowed); + assert!( + allowed >= 45 && allowed <= 55, + "Should use default limit (50 req/s)" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_update_endpoint_config() -> Result<()> { + println!("\n=== Test: Dynamic Endpoint Configuration ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // Create custom config + let custom_config = RateLimitConfig { + endpoint: "custom.endpoint".to_string(), + capacity: 20.0, + refill_rate: 20.0, + burst_size: 5, + }; + + // Update configuration + rate_limiter.set_endpoint_config(custom_config).await; + + let user_id = Uuid::new_v4(); + + // Test custom limit + let mut allowed = 0; + for _ in 0..30 { + if rate_limiter + .check_limit(&user_id, "custom.endpoint") + .await? + { + allowed += 1; + } else { + break; + } + } + + println!(" Custom endpoint allowed: {}/30", allowed); + assert!( + allowed >= 18 && allowed <= 22, + "Should use custom limit (20 req/s)" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_trading_endpoint_high_capacity() -> Result<()> { + println!("\n=== Test: Trading Endpoint High Capacity ==="); + + let config = RateLimitConfig::trading_submit_order(); + + assert_eq!(config.capacity, 100.0); + assert_eq!(config.refill_rate, 100.0); + assert_eq!(config.burst_size, 10); + println!( + " ✓ Trading config: {} req/s, burst {}", + config.refill_rate, config.burst_size + ); + + Ok(()) +} + +#[tokio::test] +async fn test_config_endpoint_low_capacity() -> Result<()> { + println!("\n=== Test: Config Update Endpoint Low Capacity ==="); + + let config = RateLimitConfig::config_update(); + + assert_eq!(config.capacity, 10.0); + assert_eq!(config.refill_rate, 10.0); + assert_eq!(config.burst_size, 2); + println!( + " ✓ Config update: {} req/s, burst {}", + config.refill_rate, config.burst_size + ); + + Ok(()) +} + +#[tokio::test] +async fn test_backtesting_endpoint_very_low_rate() -> Result<()> { + println!("\n=== Test: Backtesting Endpoint Very Low Rate ==="); + + let config = RateLimitConfig::backtesting_run(); + + assert_eq!(config.capacity, 5.0); + assert!(config.refill_rate < 0.1); // 5 requests per minute + assert_eq!(config.burst_size, 1); + println!( + " ✓ Backtesting: {:.4} req/s (5 req/min), burst {}", + config.refill_rate, config.burst_size + ); + + Ok(()) +} + +// ============================================================================ +// Redis Integration Tests (5 tests) +// ============================================================================ + +#[tokio::test] +async fn test_redis_state_shared_across_instances() -> Result<()> { + println!("\n=== Test: Redis State Sharing ==="); + + let limiter1 = RateLimiter::new(REDIS_URL).await?; + let limiter2 = RateLimiter::new(REDIS_URL).await?; + + let user_id = Uuid::new_v4(); + + // Use first instance to exhaust tokens + let mut count1 = 0; + for _ in 0..10 { + if limiter1.check_limit(&user_id, "config.update").await? { + count1 += 1; + } else { + break; + } + } + + println!(" Instance 1 allowed: {}", count1); + + // Second instance should see same state + let mut count2 = 0; + for _ in 0..10 { + if limiter2.check_limit(&user_id, "config.update").await? { + count2 += 1; + } else { + break; + } + } + + println!(" Instance 2 allowed: {}", count2); + + // Total should not exceed capacity + assert!( + count1 + count2 <= 12, + "Total should respect shared Redis state" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_redis_lua_script_atomicity() -> Result<()> { + println!("\n=== Test: Redis Lua Script Atomicity ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Launch 200 concurrent requests + let mut handles = vec![]; + + for _ in 0..200 { + let limiter = rate_limiter.clone(); + let uid = user_id; + + let handle = tokio::spawn(async move { limiter.check_limit(&uid, "config.update").await }); + + handles.push(handle); + } + + // Collect results + let mut allowed = 0; + let mut denied = 0; + + for handle in handles { + match handle.await? { + Ok(true) => allowed += 1, + Ok(false) => denied += 1, + Err(_) => {}, + } + } + + println!(" Allowed: {}, Denied: {}", allowed, denied); + + // Lua script should ensure exact limit (10 req/s for config.update) + assert_eq!(allowed + denied, 200, "All requests should complete"); + assert!( + allowed >= 9 && allowed <= 12, + "Should allow ~10 requests atomically" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_redis_key_ttl_set() -> Result<()> { + println!("\n=== Test: Redis Key TTL (300s) ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Make request to create Redis key + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + + println!(" ✓ Redis key created with 300s TTL"); + println!(" (Manual verification: redis-cli TTL ratelimit:...)"); + + Ok(()) +} + +#[tokio::test] +async fn test_redis_multiple_users_isolated() -> Result<()> { + println!("\n=== Test: Per-User Rate Limit Isolation ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // Create 10 users + let users: Vec = (0..10).map(|_| Uuid::new_v4()).collect(); + + // Each user should have independent rate limit + for (i, user_id) in users.iter().enumerate() { + let mut allowed = 0; + for _ in 0..15 { + if rate_limiter.check_limit(user_id, "config.update").await? { + allowed += 1; + } else { + break; + } + } + + println!(" User {} allowed: {}", i, allowed); + assert!( + allowed >= 9 && allowed <= 12, + "Each user should have independent limit" + ); + } + + Ok(()) +} + +#[tokio::test] +async fn test_redis_connection_reuse() -> Result<()> { + println!("\n=== Test: Redis Connection Pool Reuse ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // Make 1000 requests to test connection pooling + let mut handles = vec![]; + + for _ in 0..1000 { + let limiter = rate_limiter.clone(); + let user_id = Uuid::new_v4(); + + let handle = + tokio::spawn( + async move { limiter.check_limit(&user_id, "trading.submit_order").await }, + ); + + handles.push(handle); + } + + // All should complete without errors + let mut success = 0; + let mut errors = 0; + + for handle in handles { + match handle.await { + Ok(Ok(_)) => success += 1, + _ => errors += 1, + } + } + + println!(" Success: {}, Errors: {}", success, errors); + assert_eq!(success + errors, 1000); + assert_eq!(errors, 0, "Connection pool should handle all requests"); + + Ok(()) +} diff --git a/services/api/tests/rate_limiter_stress_test.rs b/services/api/tests/rate_limiter_stress_test.rs new file mode 100644 index 000000000..4268f02b5 --- /dev/null +++ b/services/api/tests/rate_limiter_stress_test.rs @@ -0,0 +1,480 @@ +//! Rate Limiter Stress Test & Backpressure Validation +//! +//! WAVE 73 AGENT 10: Comprehensive rate limiting stress test +//! +//! Tests: +//! 1. Single user exceeding limit +//! 2. Multiple users at limit +//! 3. Global limit breach attempt +//! 4. Distributed attack simulation +//! 5. Burst traffic handling +//! 6. Token bucket algorithm correctness +//! 7. Performance validation (<50ns target) + +use anyhow::Result; +use std::collections::HashMap; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +// Import rate limiters from both implementations +use api::auth::RateLimiter as AuthRateLimiter; + +#[tokio::test] +async fn stress_test_single_user_exceeding_limit() -> Result<()> { + println!("\n=== STRESS TEST 1: Single User Exceeding Limit ==="); + + let rate_limiter = AuthRateLimiter::new(100).map_err(|e| anyhow::anyhow!(e))?; // 100 req/s + let user_id = "stress_user_1"; + + // Attempt 10,000 requests in burst + let mut allowed = 0; + let mut denied = 0; + let start = Instant::now(); + + for _ in 0..10_000 { + if rate_limiter.check_rate_limit(user_id) { + allowed += 1; + } else { + denied += 1; + } + } + + let duration = start.elapsed(); + + println!(" Results:"); + println!(" ├─ Allowed: {}", allowed); + println!(" ├─ Denied: {}", denied); + println!(" ├─ Duration: {:?}", duration); + println!( + " └─ Rate: {:.0} req/s", + allowed as f64 / duration.as_secs_f64() + ); + + // Should allow around 100 requests + assert!( + allowed <= 110, + "Should not exceed rate limit by more than 10%" + ); + assert!(denied > 9_800, "Should deny most excess requests"); + + println!(" ✓ Single user rate limit enforced correctly"); + Ok(()) +} + +#[tokio::test] +async fn stress_test_multiple_users_at_limit() -> Result<()> { + println!("\n=== STRESS TEST 2: Multiple Users at Limit ==="); + + let rate_limiter = Arc::new(AuthRateLimiter::new(100).map_err(|e| anyhow::anyhow!(e))?); // 100 req/s per user + let num_users = 100; + let requests_per_user = 150; + + let allowed_counter = Arc::new(AtomicUsize::new(0)); + let denied_counter = Arc::new(AtomicUsize::new(0)); + + let start = Instant::now(); + + // Spawn concurrent tasks for multiple users + let mut handles = Vec::new(); + + for user_idx in 0..num_users { + let limiter = rate_limiter.clone(); + let allowed = allowed_counter.clone(); + let denied = denied_counter.clone(); + + let handle = tokio::spawn(async move { + let user_id = format!("stress_user_{}", user_idx); + let mut local_allowed = 0; + let mut local_denied = 0; + + for _ in 0..requests_per_user { + if limiter.check_rate_limit(&user_id) { + local_allowed += 1; + } else { + local_denied += 1; + } + } + + allowed.fetch_add(local_allowed, Ordering::Relaxed); + denied.fetch_add(local_denied, Ordering::Relaxed); + }); + + handles.push(handle); + } + + // Wait for all tasks to complete + for handle in handles { + handle.await?; + } + + let duration = start.elapsed(); + let allowed = allowed_counter.load(Ordering::Relaxed); + let denied = denied_counter.load(Ordering::Relaxed); + + println!(" Results:"); + println!(" ├─ Users: {}", num_users); + println!(" ├─ Total requests: {}", num_users * requests_per_user); + println!(" ├─ Allowed: {}", allowed); + println!(" ├─ Denied: {}", denied); + println!(" ├─ Duration: {:?}", duration); + println!(" └─ Avg per user: {}", allowed / num_users); + + // Each user should get around 100 requests + let avg_per_user = allowed / num_users; + assert!( + avg_per_user >= 90 && avg_per_user <= 110, + "Average per user should be ~100, got {}", + avg_per_user + ); + + println!(" ✓ Multiple users independently rate limited"); + Ok(()) +} + +#[tokio::test] +async fn stress_test_burst_attack() -> Result<()> { + println!("\n=== STRESS TEST 3: Burst Attack (10K requests in 1 second) ==="); + + let rate_limiter = Arc::new(AuthRateLimiter::new(1000).map_err(|e| anyhow::anyhow!(e))?); // 1000 req/s + let user_id = "burst_attacker"; + + let allowed_counter = Arc::new(AtomicUsize::new(0)); + let denied_counter = Arc::new(AtomicUsize::new(0)); + + let start = Instant::now(); + + // Spawn 100 concurrent tasks, each making 100 requests + let mut handles = Vec::new(); + + for _ in 0..100 { + let limiter = rate_limiter.clone(); + let allowed = allowed_counter.clone(); + let denied = denied_counter.clone(); + + let handle = tokio::spawn(async move { + for _ in 0..100 { + if limiter.check_rate_limit(user_id) { + allowed.fetch_add(1, Ordering::Relaxed); + } else { + denied.fetch_add(1, Ordering::Relaxed); + } + } + }); + + handles.push(handle); + } + + for handle in handles { + handle.await?; + } + + let duration = start.elapsed(); + let allowed = allowed_counter.load(Ordering::Relaxed); + let denied = denied_counter.load(Ordering::Relaxed); + + println!(" Results:"); + println!(" ├─ Total requests: 10,000"); + println!(" ├─ Allowed: {}", allowed); + println!(" ├─ Denied: {}", denied); + println!(" ├─ Duration: {:?}", duration); + println!( + " └─ Effective rate: {:.0} req/s", + allowed as f64 / duration.as_secs_f64() + ); + + // Should block most requests + assert!(allowed <= 1100, "Should limit burst to ~1000 requests"); + assert!(denied >= 8900, "Should deny most burst requests"); + + println!(" ✓ Burst attack successfully blocked"); + Ok(()) +} + +#[tokio::test] +async fn stress_test_sustained_flood() -> Result<()> { + println!("\n=== STRESS TEST 4: Sustained Flood (1M requests over 60s) ==="); + + let rate_limiter = Arc::new(AuthRateLimiter::new(10000).map_err(|e| anyhow::anyhow!(e))?); // 10K req/s + let user_id = "flood_attacker"; + + let allowed_counter = Arc::new(AtomicUsize::new(0)); + let start = Instant::now(); + + println!(" Simulating 60-second sustained flood..."); + + // Make requests as fast as possible for 5 seconds (scaled down test) + let test_duration = Duration::from_secs(5); + let mut handles = Vec::new(); + + for _ in 0..10 { + let limiter = rate_limiter.clone(); + let allowed = allowed_counter.clone(); + let end_time = Instant::now() + test_duration; + + let handle = tokio::spawn(async move { + while Instant::now() < end_time { + if limiter.check_rate_limit(user_id) { + allowed.fetch_add(1, Ordering::Relaxed); + } + } + }); + + handles.push(handle); + } + + for handle in handles { + handle.await?; + } + + let duration = start.elapsed(); + let allowed = allowed_counter.load(Ordering::Relaxed); + + println!(" Results:"); + println!(" ├─ Duration: {:?}", duration); + println!(" ├─ Allowed: {}", allowed); + println!( + " └─ Rate: {:.0} req/s", + allowed as f64 / duration.as_secs_f64() + ); + + // Should maintain consistent rate (allow 20% variance for governor rate limiter) + let rate = allowed as f64 / duration.as_secs_f64(); + assert!( + rate >= 9000.0 && rate <= 13000.0, + "Should maintain ~10K req/s rate, got {:.0}", + rate + ); + + println!(" ✓ Sustained flood successfully rate limited"); + Ok(()) +} + +#[tokio::test] +async fn stress_test_distributed_attack() -> Result<()> { + println!("\n=== STRESS TEST 5: Distributed Attack (100 users at 110% of limit) ==="); + + let rate_limiter = Arc::new(AuthRateLimiter::new(100).map_err(|e| anyhow::anyhow!(e))?); // 100 req/s + let num_attackers = 100; + let requests_per_attacker = 110; // 10% over limit + + let results = Arc::new(tokio::sync::Mutex::new(HashMap::new())); + + let start = Instant::now(); + let mut handles = Vec::new(); + + for attacker_idx in 0..num_attackers { + let limiter = rate_limiter.clone(); + let results_map = results.clone(); + + let handle = tokio::spawn(async move { + let user_id = format!("attacker_{}", attacker_idx); + let mut allowed = 0; + + for _ in 0..requests_per_attacker { + if limiter.check_rate_limit(&user_id) { + allowed += 1; + } + } + + let mut map = results_map.lock().await; + map.insert(user_id, allowed); + }); + + handles.push(handle); + } + + for handle in handles { + handle.await?; + } + + let duration = start.elapsed(); + let results_map = results.lock().await; + + let total_allowed: usize = results_map.values().sum(); + let avg_per_user = total_allowed / num_attackers; + let min_per_user = *results_map.values().min().unwrap_or(&0); + let max_per_user = *results_map.values().max().unwrap_or(&0); + + println!(" Results:"); + println!(" ├─ Attackers: {}", num_attackers); + println!(" ├─ Total allowed: {}", total_allowed); + println!(" ├─ Avg per user: {}", avg_per_user); + println!(" ├─ Min per user: {}", min_per_user); + println!(" ├─ Max per user: {}", max_per_user); + println!(" └─ Duration: {:?}", duration); + + // Each user should be limited to ~100 requests + assert!( + avg_per_user <= 110, + "Average should not exceed limit by >10%" + ); + assert!(min_per_user >= 90, "Min should be at least 90% of limit"); + + println!(" ✓ Distributed attack successfully mitigated"); + Ok(()) +} + +#[tokio::test] +async fn stress_test_performance_validation() -> Result<()> { + println!("\n=== STRESS TEST 6: Performance Validation (<50ns target) ==="); + + let rate_limiter = AuthRateLimiter::new(1_000_000).map_err(|e| anyhow::anyhow!(e))?; // Very high limit + let user_id = "perf_test_user"; + + // Warm-up + for _ in 0..1000 { + let _ = rate_limiter.check_rate_limit(user_id); + } + + // Measure performance + let mut latencies = Vec::with_capacity(10_000); + + for _ in 0..10_000 { + let start = Instant::now(); + let _ = rate_limiter.check_rate_limit(user_id); + latencies.push(start.elapsed()); + } + + // Calculate percentiles + latencies.sort(); + let p50 = latencies[4_999]; + let p95 = latencies[9_499]; + let p99 = latencies[9_899]; + let p999 = latencies[9_989]; + + println!(" Performance Metrics:"); + println!(" ├─ P50: {:?}", p50); + println!(" ├─ P95: {:?}", p95); + println!(" ├─ P99: {:?}", p99); + println!(" ├─ P99.9: {:?}", p999); + println!(" └─ Target: <50ns"); + + // Note: In-process measurements will show higher latency due to measurement overhead + // Actual atomic operation is <50ns, but measurement adds overhead + if p99 > Duration::from_micros(1) { + println!( + " ⚠ NOTE: P99 latency {:?} includes measurement overhead", + p99 + ); + println!(" Actual atomic counter operation is <50ns"); + } + + println!(" ✓ Performance validated (atomic counter implementation)"); + Ok(()) +} + +#[tokio::test] +async fn stress_test_token_bucket_correctness() -> Result<()> { + println!("\n=== STRESS TEST 7: Token Bucket Algorithm Correctness ==="); + + let rate_limiter = AuthRateLimiter::new(10).map_err(|e| anyhow::anyhow!(e))?; // 10 req/s + let user_id = "bucket_test_user"; + + // Phase 1: Exhaust tokens + let mut initial_allowed = 0; + for _ in 0..20 { + if rate_limiter.check_rate_limit(user_id) { + initial_allowed += 1; + } + } + + println!(" Phase 1: Initial burst"); + println!(" ├─ Allowed: {}", initial_allowed); + + assert!(initial_allowed <= 12, "Should limit initial burst to ~10"); + + // Phase 2: Wait for refill + println!(" Phase 2: Waiting 1s for token refill..."); + tokio::time::sleep(Duration::from_secs(1)).await; + + // Phase 3: Check refilled tokens + let mut refill_allowed = 0; + for _ in 0..20 { + if rate_limiter.check_rate_limit(user_id) { + refill_allowed += 1; + } + } + + println!(" Phase 3: After refill"); + println!(" ├─ Allowed: {}", refill_allowed); + + assert!( + refill_allowed >= 8 && refill_allowed <= 12, + "Should allow ~10 requests after refill, got {}", + refill_allowed + ); + + // Phase 4: Gradual increase test + println!(" Phase 4: Gradual increase over 2s"); + let mut gradual_allowed = 0; + let start = Instant::now(); + + while start.elapsed() < Duration::from_millis(2000) { + if rate_limiter.check_rate_limit(user_id) { + gradual_allowed += 1; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + + println!(" ├─ Allowed: {}", gradual_allowed); + println!(" └─ Expected: ~20 (10 req/s * 2s)"); + + assert!( + gradual_allowed >= 18 && gradual_allowed <= 22, + "Gradual increase should allow ~20 requests, got {}", + gradual_allowed + ); + + println!(" ✓ Token bucket algorithm working correctly"); + Ok(()) +} + +#[tokio::test] +async fn stress_test_edge_cases() -> Result<()> { + println!("\n=== STRESS TEST 8: Edge Cases ==="); + + // Test 1: Zero limit + println!(" Test 1: Zero-length user ID"); + let limiter1 = AuthRateLimiter::new(10).map_err(|e| anyhow::anyhow!(e))?; + let mut allowed1 = 0; + for _ in 0..20 { + if limiter1.check_rate_limit("") { + allowed1 += 1; + } + } + println!(" ├─ Empty string: {} allowed", allowed1); + assert!( + allowed1 <= 12, + "Should still enforce limit for empty string" + ); + + // Test 2: Very long user ID + println!(" Test 2: Very long user ID (1KB)"); + let long_id = "x".repeat(1024); + let limiter2 = AuthRateLimiter::new(10).map_err(|e| anyhow::anyhow!(e))?; + let mut allowed2 = 0; + for _ in 0..20 { + if limiter2.check_rate_limit(&long_id) { + allowed2 += 1; + } + } + println!(" ├─ Long ID: {} allowed", allowed2); + assert!(allowed2 <= 12, "Should handle long user IDs"); + + // Test 3: Special characters + println!(" Test 3: Special characters in user ID"); + let special_id = "user@#$%^&*()[]{}"; + let limiter3 = AuthRateLimiter::new(10).map_err(|e| anyhow::anyhow!(e))?; + let mut allowed3 = 0; + for _ in 0..20 { + if limiter3.check_rate_limit(special_id) { + allowed3 += 1; + } + } + println!(" ├─ Special chars: {} allowed", allowed3); + assert!(allowed3 <= 12, "Should handle special characters"); + + println!(" ✓ Edge cases handled correctly"); + Ok(()) +} diff --git a/services/api/tests/rate_limiting_comprehensive.rs b/services/api/tests/rate_limiting_comprehensive.rs new file mode 100644 index 000000000..c63358b58 --- /dev/null +++ b/services/api/tests/rate_limiting_comprehensive.rs @@ -0,0 +1,1133 @@ +//! Comprehensive Rate Limiting Tests - Wave 100 Agent 3 +//! +//! Tests for achieving 95%+ coverage of rate_limiter.rs: +//! 1. Redis backend integration (Lua scripts, connection handling) +//! 2. Cache management (LRU eviction, stats, invalidation) +//! 3. Endpoint configuration (dynamic updates, defaults) +//! 4. Error paths (Redis failures, timeouts) +//! 5. Integration scenarios (multi-endpoint, cross-user) +//! +//! Target: 30+ test cases covering all untested code paths + +use anyhow::Result; +use std::time::{Duration, Instant}; +use uuid::Uuid; + +use api::routing::{RateLimitConfig, RateLimiter}; + +const REDIS_URL: &str = "redis://localhost:6379"; + +// ============================================================================ +// SECTION 1: Redis Backend Integration Tests (10 tests) +// ============================================================================ + +#[tokio::test] +async fn test_redis_backend_basic_check() -> Result<()> { + println!("\n=== Test: Redis Backend Basic Check ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // First request should succeed + let result1 = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + println!(" First request: {}", result1); + assert!(result1, "First request should be allowed"); + + // Subsequent requests should succeed up to capacity + let mut allowed = 0; + for i in 0..150 { + if rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await? + { + allowed += 1; + } else { + println!(" First denial at request #{}", i + 2); + break; + } + } + + println!(" Total allowed: {}", allowed + 1); + + // Should be limited by capacity (100 for trading.submit_order) + assert!( + allowed + 1 <= 110, + "Should not exceed capacity by more than 10%" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_redis_lua_script_execution() -> Result<()> { + println!("\n=== Test: Redis Lua Script Atomic Execution ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Concurrent requests should be handled atomically by Lua script + let mut handles = Vec::new(); + + println!(" Spawning 100 concurrent requests..."); + for _ in 0..100 { + let limiter = rate_limiter.clone(); + let uid = user_id; + let handle = + tokio::spawn(async move { limiter.check_limit(&uid, "trading.submit_order").await }); + handles.push(handle); + } + + // Collect results + let mut allowed = 0; + let mut denied = 0; + for handle in handles { + match handle.await? { + Ok(true) => allowed += 1, + Ok(false) => denied += 1, + Err(_) => {}, + } + } + + println!(" Allowed: {}, Denied: {}", allowed, denied); + + // Lua script should ensure exact limit enforcement + assert_eq!(allowed + denied, 100, "All requests should complete"); + assert!(allowed <= 100, "Should not exceed capacity"); + + Ok(()) +} + +#[tokio::test] +async fn test_redis_token_refill() -> Result<()> { + println!("\n=== Test: Redis Token Bucket Refill ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Exhaust tokens + let mut initial_allowed = 0; + for _ in 0..150 { + if rate_limiter.check_limit(&user_id, "config.update").await? { + initial_allowed += 1; + } + } + + println!(" Initial allowed: {}", initial_allowed); + assert!( + initial_allowed <= 12, + "Should be limited to capacity (10 + tolerance)" + ); + + // Wait for refill (config.update has 10 req/s refill rate) + println!(" Waiting 1s for token refill..."); + tokio::time::sleep(Duration::from_secs(1)).await; + + // Should have refilled tokens + let mut refilled = 0; + for _ in 0..20 { + if rate_limiter.check_limit(&user_id, "config.update").await? { + refilled += 1; + } + } + + println!(" After refill: {}", refilled); + assert!( + refilled >= 8 && refilled <= 12, + "Should refill ~10 tokens, got {}", + refilled + ); + + Ok(()) +} + +#[tokio::test] +async fn test_redis_persistence() -> Result<()> { + println!("\n=== Test: Redis State Persistence ==="); + + let rate_limiter1 = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Make some requests with first limiter instance + let mut count1 = 0; + for _ in 0..5 { + if rate_limiter1 + .check_limit(&user_id, "backtesting.run") + .await? + { + count1 += 1; + } + } + + println!(" Instance 1 allowed: {}", count1); + + // Create new limiter instance (should share Redis state) + let rate_limiter2 = RateLimiter::new(REDIS_URL).await?; + + // Remaining requests should respect previous consumption + let mut count2 = 0; + for _ in 0..5 { + if rate_limiter2 + .check_limit(&user_id, "backtesting.run") + .await? + { + count2 += 1; + } + } + + println!(" Instance 2 allowed: {}", count2); + + // Total should not exceed capacity (5 for backtesting.run) + assert!(count1 + count2 <= 6, "Total should respect shared state"); + + Ok(()) +} + +#[tokio::test] +async fn test_redis_ttl_expiration() -> Result<()> { + println!("\n=== Test: Redis Key TTL Expiration ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Make a request to create Redis key + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + + println!(" Key created with 300s TTL"); + println!(" (TTL validation requires manual Redis inspection)"); + + // Note: Full TTL test would require waiting 300s or manual Redis commands + // This test validates the key is created; TTL is set in Lua script + + Ok(()) +} + +#[tokio::test] +async fn test_redis_connection_pool() -> Result<()> { + println!("\n=== Test: Redis Connection Pool Behavior ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // Make 1000 requests to stress connection pool + let mut handles = Vec::new(); + + println!(" Spawning 1000 concurrent Redis requests..."); + for i in 0..1000 { + let limiter = rate_limiter.clone(); + let user_id = Uuid::new_v4(); + let handle = + tokio::spawn( + async move { limiter.check_limit(&user_id, "trading.submit_order").await }, + ); + handles.push(handle); + } + + // All should succeed without connection pool exhaustion + let mut success = 0; + let mut errors = 0; + for handle in handles { + match handle.await? { + Ok(_) => success += 1, + Err(_) => errors += 1, + } + } + + println!(" Success: {}, Errors: {}", success, errors); + assert!(errors < 10, "Should have minimal connection errors"); + + Ok(()) +} + +#[tokio::test] +async fn test_redis_error_handling() -> Result<()> { + println!("\n=== Test: Redis Connection Error Handling ==="); + + // Attempt connection to invalid Redis URL + let result = RateLimiter::new("redis://invalid-host:9999").await; + + println!(" Invalid Redis URL result: {:?}", result.is_err()); + assert!(result.is_err(), "Should fail for invalid Redis URL"); + + if let Err(e) = result { + println!(" Error message: {}", e); + assert!( + e.to_string().contains("Failed to"), + "Should have descriptive error" + ); + } + + Ok(()) +} + +#[tokio::test] +async fn test_redis_multiple_endpoints() -> Result<()> { + println!("\n=== Test: Redis Multiple Endpoint Tracking ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Same user, different endpoints - should be tracked separately + let mut trading_allowed = 0; + let mut config_allowed = 0; + let mut backtest_allowed = 0; + + for _ in 0..20 { + if rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await? + { + trading_allowed += 1; + } + if rate_limiter.check_limit(&user_id, "config.update").await? { + config_allowed += 1; + } + if rate_limiter + .check_limit(&user_id, "backtesting.run") + .await? + { + backtest_allowed += 1; + } + } + + println!(" Trading: {}", trading_allowed); + println!(" Config: {}", config_allowed); + println!(" Backtest: {}", backtest_allowed); + + // Each endpoint should have independent limits + assert!(trading_allowed >= 15, "Trading should allow most requests"); + assert!(config_allowed >= 8, "Config should allow some requests"); + assert!(backtest_allowed <= 6, "Backtest should be most restrictive"); + + Ok(()) +} + +#[tokio::test] +async fn test_redis_cross_user_isolation() -> Result<()> { + println!("\n=== Test: Redis Cross-User Isolation ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + let user1 = Uuid::new_v4(); + let user2 = Uuid::new_v4(); + + // User 1 exhausts their limit + let mut user1_allowed = 0; + for _ in 0..20 { + if rate_limiter.check_limit(&user1, "config.update").await? { + user1_allowed += 1; + } + } + + // User 2 should still have full quota + let mut user2_allowed = 0; + for _ in 0..20 { + if rate_limiter.check_limit(&user2, "config.update").await? { + user2_allowed += 1; + } + } + + println!(" User 1: {}", user1_allowed); + println!(" User 2: {}", user2_allowed); + + assert!(user1_allowed <= 12, "User 1 should be limited"); + assert!( + user2_allowed <= 12, + "User 2 should be independently limited" + ); + assert_eq!( + user1_allowed, user2_allowed, + "Users should have equal quotas" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_redis_system_time_error() -> Result<()> { + println!("\n=== Test: Redis System Time Error Handling ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Normal requests should succeed (validates system time is working) + let result = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + + println!(" System time operational: {}", result); + assert!(result, "Should work with valid system time"); + + // Note: Testing actual system time errors would require mocking, + // which is outside scope. This validates the happy path. + + Ok(()) +} + +// ============================================================================ +// SECTION 2: Cache Management Tests (8 tests) +// ============================================================================ + +#[tokio::test] +async fn test_cache_basic_operation() -> Result<()> { + println!("\n=== Test: Cache Basic Operation ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // First request (cache miss, Redis hit) + let start1 = Instant::now(); + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + let latency1 = start1.elapsed(); + + // Second request (cache hit) + let start2 = Instant::now(); + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + let latency2 = start2.elapsed(); + + println!(" First request (Redis): {:?}", latency1); + println!(" Second request (cache): {:?}", latency2); + println!( + " Cache speedup: {:.1}x", + latency1.as_nanos() as f64 / latency2.as_nanos() as f64 + ); + + // Cache hit should be significantly faster + assert!(latency2 < latency1, "Cached request should be faster"); + + Ok(()) +} + +#[tokio::test] +async fn test_cache_ttl_expiration() -> Result<()> { + println!("\n=== Test: Cache TTL Expiration ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Request to populate cache + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + + println!(" Cache populated, TTL: 1s"); + + // Wait for cache TTL to expire (1 second) + println!(" Waiting 1.1s for cache expiration..."); + tokio::time::sleep(Duration::from_millis(1100)).await; + + // Next request should be slower (cache miss) + let start = Instant::now(); + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + let latency = start.elapsed(); + + println!(" Post-expiration latency: {:?}", latency); + + // Should go back to Redis (higher latency) + assert!( + latency > Duration::from_micros(1), + "Should bypass expired cache" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_cache_lru_eviction() -> Result<()> { + println!("\n=== Test: Cache LRU Eviction ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // Fill cache beyond max size (10,000 entries) + println!(" Filling cache with 10,500 unique users..."); + for i in 0..10_500 { + let user_id = Uuid::new_v4(); + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + + if i % 1000 == 0 { + let stats = rate_limiter.get_cache_stats().await; + println!(" Progress: {} users, cache size: {}", i, stats.size); + } + } + + // Cache should have evicted oldest 10% (1,000 entries) + let final_stats = rate_limiter.get_cache_stats().await; + println!(" Final cache size: {}", final_stats.size); + + assert!( + final_stats.size <= 10_000, + "Cache should not exceed max size" + ); + assert!( + final_stats.size >= 9_000, + "Cache should retain most recent entries" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_cache_stats() -> Result<()> { + println!("\n=== Test: Cache Statistics ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // Initial stats + let stats1 = rate_limiter.get_cache_stats().await; + println!(" Initial stats:"); + println!(" ├─ Size: {}", stats1.size); + println!(" ├─ Max: {}", stats1.max_size); + println!(" └─ TTL: {}s", stats1.ttl_seconds); + + assert_eq!(stats1.max_size, 10_000, "Max size should be 10,000"); + assert_eq!(stats1.ttl_seconds, 1, "TTL should be 1 second"); + + // Add some entries + for _ in 0..100 { + let user_id = Uuid::new_v4(); + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + } + + let stats2 = rate_limiter.get_cache_stats().await; + println!(" After 100 requests:"); + println!(" └─ Size: {}", stats2.size); + + assert!(stats2.size >= 50, "Should have cached some entries"); + + Ok(()) +} + +#[tokio::test] +async fn test_cache_clear() -> Result<()> { + println!("\n=== Test: Cache Clear Operation ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // Populate cache + for _ in 0..50 { + let user_id = Uuid::new_v4(); + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + } + + let stats_before = rate_limiter.get_cache_stats().await; + println!(" Cache size before clear: {}", stats_before.size); + + // Clear cache + rate_limiter.clear_cache().await; + + let stats_after = rate_limiter.get_cache_stats().await; + println!(" Cache size after clear: {}", stats_after.size); + + assert_eq!(stats_after.size, 0, "Cache should be empty after clear"); + + Ok(()) +} + +#[tokio::test] +async fn test_cache_concurrent_access() -> Result<()> { + println!("\n=== Test: Cache Concurrent Access (DashMap Lock-Free) ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Populate cache for this user + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + + // Concurrent cache hits + let mut handles = Vec::new(); + + println!(" Spawning 1000 concurrent cache hits..."); + for _ in 0..1000 { + let limiter = rate_limiter.clone(); + let uid = user_id; + let handle = tokio::spawn(async move { + let start = Instant::now(); + let _ = limiter.check_limit(&uid, "trading.submit_order").await; + start.elapsed() + }); + handles.push(handle); + } + + // Collect latencies + let mut latencies = Vec::new(); + for handle in handles { + if let Ok(latency) = handle.await { + latencies.push(latency); + } + } + + // Calculate percentiles + latencies.sort(); + let p50 = latencies[499]; + let p99 = latencies[989]; + + println!(" Concurrent cache performance:"); + println!(" ├─ P50: {:?}", p50); + println!(" ├─ P99: {:?}", p99); + println!(" └─ Target: <8ns (DashMap lock-free)"); + + // Note: Actual latency includes network and system overhead + // Target <8ns is for the DashMap operation itself + + Ok(()) +} + +#[tokio::test] +async fn test_cache_invalidation_on_error() -> Result<()> { + println!("\n=== Test: Cache Invalidation on Error ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Normal request to populate cache + let result1 = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + println!(" Initial request: {}", result1); + + // Subsequent requests should use cache + let result2 = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + println!(" Cached request: {}", result2); + + // Cache should remain valid across requests + assert!(result1 || result2, "At least one request should succeed"); + + Ok(()) +} + +#[tokio::test] +async fn test_cache_size_overflow_handling() -> Result<()> { + println!("\n=== Test: Cache Size Overflow Handling ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // Attempt to overflow cache with rapid insertions + println!(" Rapid insertion of 11,000 entries..."); + + let start = Instant::now(); + for i in 0..11_000 { + let user_id = Uuid::new_v4(); + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + + if i % 2000 == 0 { + let stats = rate_limiter.get_cache_stats().await; + println!(" {} entries: cache size = {}", i, stats.size); + } + } + let duration = start.elapsed(); + + let final_stats = rate_limiter.get_cache_stats().await; + println!(" Final: {} entries in {:?}", final_stats.size, duration); + + // Should handle overflow gracefully via LRU eviction + assert!( + final_stats.size <= 10_000, + "Should not exceed max cache size" + ); + + Ok(()) +} + +// ============================================================================ +// SECTION 3: Endpoint Configuration Tests (6 tests) +// ============================================================================ + +#[tokio::test] +async fn test_default_endpoint_configs() -> Result<()> { + println!("\n=== Test: Default Endpoint Configurations ==="); + + let trading = RateLimitConfig::trading_submit_order(); + let config = RateLimitConfig::config_update(); + let backtest = RateLimitConfig::backtesting_run(); + + println!(" Trading config:"); + println!(" ├─ Capacity: {}", trading.capacity); + println!(" ├─ Refill rate: {}/s", trading.refill_rate); + println!(" └─ Burst size: {}", trading.burst_size); + + println!(" Config update:"); + println!(" ├─ Capacity: {}", config.capacity); + println!(" ├─ Refill rate: {}/s", config.refill_rate); + println!(" └─ Burst size: {}", config.burst_size); + + println!(" Backtesting:"); + println!(" ├─ Capacity: {}", backtest.capacity); + println!(" ├─ Refill rate: {}/min", backtest.refill_rate * 60.0); + println!(" └─ Burst size: {}", backtest.burst_size); + + assert_eq!(trading.capacity, 100.0, "Trading capacity"); + assert_eq!(config.capacity, 10.0, "Config capacity"); + assert_eq!(backtest.capacity, 5.0, "Backtest capacity"); + + Ok(()) +} + +#[tokio::test] +async fn test_dynamic_endpoint_config() -> Result<()> { + println!("\n=== Test: Dynamic Endpoint Configuration ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // Add custom endpoint config + let custom_config = RateLimitConfig { + endpoint: "custom.endpoint".to_string(), + capacity: 20.0, + refill_rate: 20.0, + burst_size: 5, + }; + + rate_limiter.set_endpoint_config(custom_config).await; + + println!(" Custom endpoint config added"); + + // Use the custom endpoint + let user_id = Uuid::new_v4(); + let mut allowed = 0; + + for _ in 0..30 { + if rate_limiter + .check_limit(&user_id, "custom.endpoint") + .await? + { + allowed += 1; + } + } + + println!(" Custom endpoint allowed: {}", allowed); + assert!( + allowed >= 18 && allowed <= 22, + "Should respect custom capacity" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_default_for_unknown_endpoint() -> Result<()> { + println!("\n=== Test: Default Config for Unknown Endpoint ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Use unknown endpoint (should get default config) + let mut allowed = 0; + for _ in 0..70 { + if rate_limiter + .check_limit(&user_id, "unknown.endpoint") + .await? + { + allowed += 1; + } + } + + println!(" Unknown endpoint allowed: {}", allowed); + + // Default is 50 req/s capacity + assert!( + allowed >= 45 && allowed <= 55, + "Should use default 50 capacity" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_endpoint_config_update() -> Result<()> { + println!("\n=== Test: Endpoint Configuration Update ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Set initial config + let config1 = RateLimitConfig { + endpoint: "mutable.endpoint".to_string(), + capacity: 10.0, + refill_rate: 10.0, + burst_size: 2, + }; + rate_limiter.set_endpoint_config(config1).await; + + // Test with initial config + let mut count1 = 0; + for _ in 0..20 { + if rate_limiter + .check_limit(&user_id, "mutable.endpoint") + .await? + { + count1 += 1; + } + } + + println!(" With capacity 10: {} allowed", count1); + assert!(count1 <= 12, "Should respect initial capacity"); + + // Wait for refill + tokio::time::sleep(Duration::from_millis(1100)).await; + + // Update config + let config2 = RateLimitConfig { + endpoint: "mutable.endpoint".to_string(), + capacity: 50.0, + refill_rate: 50.0, + burst_size: 10, + }; + rate_limiter.set_endpoint_config(config2).await; + + // Clear cache to use new config + rate_limiter.clear_cache().await; + + // Test with new config + let user_id2 = Uuid::new_v4(); + let mut count2 = 0; + for _ in 0..70 { + if rate_limiter + .check_limit(&user_id2, "mutable.endpoint") + .await? + { + count2 += 1; + } + } + + println!(" With capacity 50: {} allowed", count2); + assert!(count2 >= 45, "Should respect updated capacity"); + + Ok(()) +} + +#[tokio::test] +async fn test_multiple_endpoint_configs() -> Result<()> { + println!("\n=== Test: Multiple Endpoint Configurations ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // Add multiple custom configs + for i in 1..=10 { + let config = RateLimitConfig { + endpoint: format!("endpoint_{}", i), + capacity: (i * 10) as f64, + refill_rate: (i * 10) as f64, + burst_size: i, + }; + rate_limiter.set_endpoint_config(config).await; + } + + println!(" Added 10 endpoint configs"); + + // Verify each endpoint has correct limit + for i in 1..=10 { + let user_id = Uuid::new_v4(); + let mut allowed = 0; + let endpoint = format!("endpoint_{}", i); + + for _ in 0..(i * 20) { + if rate_limiter.check_limit(&user_id, &endpoint).await? { + allowed += 1; + } + } + + let expected = i * 10; + println!( + " Endpoint {}: {} allowed (expected ~{})", + i, allowed, expected + ); + assert!(allowed <= expected + 2, "Should respect individual limits"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_endpoint_config_concurrency() -> Result<()> { + println!("\n=== Test: Concurrent Endpoint Config Updates ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // Concurrent config updates (DashMap should handle safely) + let mut handles = Vec::new(); + + println!(" Spawning 100 concurrent config updates..."); + for i in 0..100 { + let limiter = rate_limiter.clone(); + let handle = tokio::spawn(async move { + let config = RateLimitConfig { + endpoint: format!("concurrent_{}", i % 10), + capacity: ((i % 10 + 1) * 10) as f64, + refill_rate: ((i % 10 + 1) * 10) as f64, + burst_size: i % 10 + 1, + }; + limiter.set_endpoint_config(config).await; + }); + handles.push(handle); + } + + for handle in handles { + handle.await?; + } + + println!(" ✓ All concurrent updates completed"); + + // Verify configs are usable + let user_id = Uuid::new_v4(); + let result = rate_limiter.check_limit(&user_id, "concurrent_5").await?; + println!(" Test request after concurrent updates: {}", result); + + Ok(()) +} + +// ============================================================================ +// SECTION 4: Performance Validation (3 tests) +// ============================================================================ + +#[tokio::test] +async fn test_cache_hit_performance() -> Result<()> { + println!("\n=== Test: Cache Hit Performance (<8ns target) ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + let user_id = Uuid::new_v4(); + + // Warm up cache + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await?; + + // Measure cache hit latency + let mut latencies = Vec::new(); + + for _ in 0..1000 { + let start = Instant::now(); + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await; + latencies.push(start.elapsed()); + } + + latencies.sort(); + let p50 = latencies[499]; + let p95 = latencies[949]; + let p99 = latencies[989]; + + println!(" Cache hit latency:"); + println!(" ├─ P50: {:?}", p50); + println!(" ├─ P95: {:?}", p95); + println!(" ├─ P99: {:?}", p99); + println!(" └─ Target: <8ns (DashMap operation only)"); + + // Note: Includes async/await overhead, actual DashMap is <8ns + + Ok(()) +} + +#[tokio::test] +async fn test_redis_hit_performance() -> Result<()> { + println!("\n=== Test: Redis Hit Performance (<500μs target) ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // Clear cache to force Redis hits + rate_limiter.clear_cache().await; + + let mut latencies = Vec::new(); + + for _ in 0..100 { + let user_id = Uuid::new_v4(); + let start = Instant::now(); + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await; + latencies.push(start.elapsed()); + } + + latencies.sort(); + let p50 = latencies[49]; + let p95 = latencies[94]; + let p99 = latencies[99]; + + println!(" Redis hit latency:"); + println!(" ├─ P50: {:?}", p50); + println!(" ├─ P95: {:?}", p95); + println!(" ├─ P99: {:?}", p99); + println!(" └─ Target: <500μs"); + + assert!( + p99 < Duration::from_millis(1), + "P99 should be under 1ms for local Redis" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_throughput_performance() -> Result<()> { + println!("\n=== Test: Throughput Performance ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + let start = Instant::now(); + let mut requests = 0; + + // Make requests for 1 second + while start.elapsed() < Duration::from_secs(1) { + let user_id = Uuid::new_v4(); + let _ = rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await; + requests += 1; + } + + let duration = start.elapsed(); + let req_per_sec = (requests as f64) / duration.as_secs_f64(); + + println!(" Throughput: {:.0} req/s", req_per_sec); + println!(" Total requests: {}", requests); + + assert!(req_per_sec > 1000.0, "Should handle >1000 req/s"); + + Ok(()) +} + +// ============================================================================ +// SECTION 5: Integration Scenarios (3 tests) +// ============================================================================ + +#[tokio::test] +async fn test_full_workflow_integration() -> Result<()> { + println!("\n=== Test: Full Workflow Integration ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // Simulate realistic trading workflow + let trader = Uuid::new_v4(); + + // 1. Config queries (high frequency) + for _ in 0..5 { + let _ = rate_limiter.check_limit(&trader, "config.get").await; + } + + // 2. Trading submissions (burst) + let mut trades_allowed = 0; + for _ in 0..50 { + if rate_limiter + .check_limit(&trader, "trading.submit_order") + .await? + { + trades_allowed += 1; + } + } + + // 3. Backtest request (rate-limited) + let backtest_allowed = rate_limiter.check_limit(&trader, "backtesting.run").await?; + + println!(" Workflow results:"); + println!(" ├─ Trades allowed: {}/50", trades_allowed); + println!(" └─ Backtest allowed: {}", backtest_allowed); + + assert!(trades_allowed >= 40, "Should allow most trades"); + + Ok(()) +} + +#[tokio::test] +async fn test_multi_user_multi_endpoint() -> Result<()> { + println!("\n=== Test: Multi-User Multi-Endpoint Integration ==="); + + let rate_limiter = RateLimiter::new(REDIS_URL).await?; + + // 10 users, 3 endpoints each + let mut results = Vec::new(); + + for user_idx in 0..10 { + let user_id = Uuid::new_v4(); + let mut user_results = (0, 0, 0); + + for _ in 0..20 { + if rate_limiter + .check_limit(&user_id, "trading.submit_order") + .await? + { + user_results.0 += 1; + } + if rate_limiter.check_limit(&user_id, "config.update").await? { + user_results.1 += 1; + } + if rate_limiter + .check_limit(&user_id, "backtesting.run") + .await? + { + user_results.2 += 1; + } + } + + results.push(user_results); + + if user_idx % 3 == 0 { + println!( + " User {} results: trade={}, config={}, backtest={}", + user_idx, user_results.0, user_results.1, user_results.2 + ); + } + } + + // Verify all users got similar treatment + let avg_trading: usize = results.iter().map(|(t, _, _)| t).sum::() / 10; + println!(" Average trading per user: {}", avg_trading); + + assert!(avg_trading >= 15, "Users should get consistent limits"); + + Ok(()) +} + +#[tokio::test] +async fn test_cache_redis_consistency() -> Result<()> { + println!("\n=== Test: Cache-Redis Consistency ==="); + + let limiter1 = RateLimiter::new(REDIS_URL).await?; + let limiter2 = RateLimiter::new(REDIS_URL).await?; + + let user_id = Uuid::new_v4(); + + // Instance 1 makes requests (populates its cache) + let mut count1 = 0; + for _ in 0..10 { + if limiter1.check_limit(&user_id, "config.update").await? { + count1 += 1; + } + } + + println!(" Instance 1 allowed: {}", count1); + + // Instance 2 makes requests (separate cache, shared Redis) + let mut count2 = 0; + for _ in 0..10 { + if limiter2.check_limit(&user_id, "config.update").await? { + count2 += 1; + } + } + + println!(" Instance 2 allowed: {}", count2); + println!(" Total: {}", count1 + count2); + + // Combined total should respect Redis state (10 capacity) + assert!(count1 + count2 <= 12, "Combined should not exceed capacity"); + + Ok(()) +} diff --git a/services/api/tests/rate_limiting_tests.rs b/services/api/tests/rate_limiting_tests.rs new file mode 100644 index 000000000..867bd2b5f --- /dev/null +++ b/services/api/tests/rate_limiting_tests.rs @@ -0,0 +1,344 @@ +//! Rate Limiting Integration Tests +//! +//! Tests for Layer 6 of the authentication pipeline: +//! - In-memory rate limiting with atomic counters +//! - Per-user rate limits +//! - Concurrent request handling +//! - Rate limit reset behavior + +#[path = "common/mod.rs"] +mod common; + +use anyhow::Result; +use std::time::{Duration, Instant}; + +use api::auth::RateLimiter; + +const REDIS_URL: &str = "redis://localhost:6379"; + +#[tokio::test] +async fn test_rate_limiter_basic() -> Result<()> { + println!("\n=== Test: Rate Limiter Basic Functionality ==="); + + let rate_limiter = RateLimiter::new(10).expect("Failed to create rate limiter"); // 10 requests per second + + let mut allowed_count = 0; + let mut denied_count = 0; + + // Make 15 requests + for i in 1..=15 { + if rate_limiter.check_rate_limit("user_basic") { + allowed_count += 1; + } else { + denied_count += 1; + if denied_count == 1 { + println!(" ✓ First denial at request #{}", i); + } + } + } + + println!(" Allowed: {}, Denied: {}", allowed_count, denied_count); + + assert!(allowed_count <= 10, "Should allow at most 10 requests"); + assert!(denied_count > 0, "Should deny some requests after limit"); + + Ok(()) +} + +#[tokio::test] +async fn test_rate_limiter_per_user() -> Result<()> { + println!("\n=== Test: Per-User Rate Limiting ==="); + + let rate_limiter = RateLimiter::new(5).expect("Failed to create rate limiter"); // 5 requests per second per user + + // User 1 makes 7 requests + let mut user1_allowed = 0; + for _ in 1..=7 { + if rate_limiter.check_rate_limit("user1") { + user1_allowed += 1; + } + } + + // User 2 makes 7 requests + let mut user2_allowed = 0; + for _ in 1..=7 { + if rate_limiter.check_rate_limit("user2") { + user2_allowed += 1; + } + } + + println!(" User 1 allowed: {}", user1_allowed); + println!(" User 2 allowed: {}", user2_allowed); + + // Each user should be rate limited independently + assert!(user1_allowed <= 5, "User 1 should be limited to 5 requests"); + assert!(user2_allowed <= 5, "User 2 should be limited to 5 requests"); + + Ok(()) +} + +#[tokio::test] +async fn test_rate_limiter_concurrent_requests() -> Result<()> { + println!("\n=== Test: Concurrent Rate Limiting ==="); + + let rate_limiter = RateLimiter::new(100).expect("Failed to create rate limiter"); // 100 requests per second + + // Spawn 200 concurrent requests for same user + let mut handles = Vec::new(); + + println!(" Spawning 200 concurrent requests..."); + for _ in 0..200 { + let limiter = rate_limiter.clone(); + let handle = tokio::spawn(async move { limiter.check_rate_limit("concurrent_user") }); + handles.push(handle); + } + + // Collect results + let mut allowed_count = 0; + for handle in handles { + if let Ok(allowed) = handle.await { + if allowed { + allowed_count += 1; + } + } + } + + println!(" ✓ {}/200 concurrent requests allowed", allowed_count); + + // Should allow around 100 requests (may vary slightly due to timing) + assert!(allowed_count >= 90, "Should allow at least 90 requests"); + assert!( + allowed_count <= 110, + "Should not allow more than 110 requests" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_rate_limiter_performance() -> Result<()> { + println!("\n=== Test: Rate Limiter Performance (<50ns target) ==="); + + let rate_limiter = RateLimiter::new(1000000).expect("Failed to create rate limiter"); // Very high limit for perf testing + + let mut latencies = Vec::new(); + + // Perform 1000 rate limit checks + println!(" Running 1000 rate limit checks..."); + for _ in 0..1000 { + let start = Instant::now(); + let _ = rate_limiter.check_rate_limit("perf_user"); + let elapsed = start.elapsed(); + latencies.push(elapsed); + } + + // Calculate percentiles + latencies.sort(); + let p50 = latencies[499]; + let p95 = latencies[949]; + let p99 = latencies[989]; + let p999 = latencies[999]; + + println!("\n Performance Metrics:"); + println!(" ├─ P50: {:?}", p50); + println!(" ├─ P95: {:?}", p95); + println!(" ├─ P99: {:?}", p99); + println!(" └─ P99.9: {:?}", p999); + + println!("\n Target: <50ns per check"); + + if p99 > Duration::from_nanos(50) { + println!(" ⚠ WARNING: P99 latency {:?} exceeds 50ns target", p99); + } else { + println!(" ✓ P99 latency within 50ns target"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_rate_limiter_reset_behavior() -> Result<()> { + println!("\n=== Test: Rate Limiter Reset Behavior ==="); + + let rate_limiter = RateLimiter::new(5).expect("Failed to create rate limiter"); // 5 requests per second + + // Exhaust rate limit + let mut initial_allowed = 0; + for _ in 1..=10 { + if rate_limiter.check_rate_limit("reset_user") { + initial_allowed += 1; + } + } + + println!(" Initial requests allowed: {}", initial_allowed); + assert!(initial_allowed <= 5, "Should be limited to 5 requests"); + + // Wait for rate limiter window to reset (1 second) + println!(" Waiting 1.1s for rate limit window to reset..."); + tokio::time::sleep(Duration::from_millis(1100)).await; + + // Try again after reset + let mut post_reset_allowed = 0; + for _ in 1..=10 { + if rate_limiter.check_rate_limit("reset_user") { + post_reset_allowed += 1; + } + } + + println!(" Post-reset requests allowed: {}", post_reset_allowed); + assert!(post_reset_allowed > 0, "Should allow requests after reset"); + assert!( + post_reset_allowed <= 5, + "Should still enforce limit after reset" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_rate_limiter_multiple_users() -> Result<()> { + println!("\n=== Test: Multiple Users Rate Limiting ==="); + + let rate_limiter = RateLimiter::new(10).expect("Failed to create rate limiter"); // 10 requests per second per user + + // 10 different users make 15 requests each + let mut user_results = Vec::new(); + + for user_id in 1..=10 { + let mut allowed = 0; + for _ in 1..=15 { + if rate_limiter.check_rate_limit(&format!("multi_user_{}", user_id)) { + allowed += 1; + } + } + user_results.push(allowed); + } + + println!(" User results: {:?}", user_results); + + // Each user should be limited independently + for (i, allowed) in user_results.into_iter().enumerate() { + assert!( + allowed <= 10, + "User {} should be limited to 10 requests, got {}", + i + 1, + allowed + ); + } + + println!(" ✓ All 10 users independently rate limited"); + + Ok(()) +} + +#[tokio::test] +async fn test_rate_limiter_burst_handling() -> Result<()> { + println!("\n=== Test: Burst Request Handling ==="); + + let rate_limiter = RateLimiter::new(50).expect("Failed to create rate limiter"); // 50 requests per second + + // Send 100 requests as fast as possible (burst) + let start = Instant::now(); + let mut burst_allowed = 0; + + for _ in 0..100 { + if rate_limiter.check_rate_limit("burst_user") { + burst_allowed += 1; + } + } + + let burst_duration = start.elapsed(); + + println!( + " Burst allowed: {} requests in {:?}", + burst_allowed, burst_duration + ); + + assert!(burst_allowed <= 50, "Should limit burst to 50 requests"); + assert!( + burst_duration < Duration::from_millis(100), + "Burst check should be fast" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_rate_limiter_edge_cases() -> Result<()> { + println!("\n=== Test: Rate Limiter Edge Cases ==="); + + // Test with very low limit + let low_limit = RateLimiter::new(1).expect("Failed to create rate limiter"); + let mut low_allowed = 0; + for _ in 0..5 { + if low_limit.check_rate_limit("low_limit_user") { + low_allowed += 1; + } + } + println!(" Low limit (1/s): {} allowed", low_allowed); + assert!(low_allowed <= 1, "Should enforce limit of 1"); + + // Test with high limit + let high_limit = RateLimiter::new(10000).expect("Failed to create rate limiter"); + let mut high_allowed = 0; + for _ in 0..100 { + if high_limit.check_rate_limit("high_limit_user") { + high_allowed += 1; + } + } + println!(" High limit (10000/s): {} allowed", high_allowed); + assert_eq!(high_allowed, 100, "Should allow all 100 requests"); + + // Test with empty user ID + let empty_limiter = RateLimiter::new(5).expect("Failed to create rate limiter"); + let mut empty_allowed = 0; + for _ in 0..10 { + if empty_limiter.check_rate_limit("") { + empty_allowed += 1; + } + } + println!(" Empty user ID: {} allowed", empty_allowed); + assert!( + empty_allowed <= 5, + "Should still enforce limit for empty user ID" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_rate_limiter_sustained_load() -> Result<()> { + println!("\n=== Test: Sustained Load Rate Limiting ==="); + + let rate_limiter = RateLimiter::new(100).expect("Failed to create rate limiter"); // 100 requests per second + + let mut total_allowed = 0; + let start = Instant::now(); + + // Simulate sustained load for 2 seconds + while start.elapsed() < Duration::from_secs(2) { + if rate_limiter.check_rate_limit("sustained_user") { + total_allowed += 1; + } + // Small delay to prevent tight loop + tokio::time::sleep(Duration::from_micros(100)).await; + } + + let actual_duration = start.elapsed(); + let requests_per_second = (total_allowed as f64) / actual_duration.as_secs_f64(); + + println!( + " Total allowed: {} requests over {:?}", + total_allowed, actual_duration + ); + println!(" Effective rate: {:.2} req/s", requests_per_second); + + // Should be close to 200 requests (100/s * 2s), allowing for some variance + assert!( + (180..=220).contains(&total_allowed), + "Sustained rate should be around 200 requests (got {})", + total_allowed + ); + + Ok(()) +} diff --git a/services/api/tests/real_backend_integration_test.rs b/services/api/tests/real_backend_integration_test.rs new file mode 100644 index 000000000..f4991fb2f --- /dev/null +++ b/services/api/tests/real_backend_integration_test.rs @@ -0,0 +1,637 @@ +//! Real Backend Integration Tests for API Gateway +//! +//! Tests that verify the API Gateway correctly proxies requests to real backend services: +//! - Trading Service (port 50052) +//! - Backtesting Service (port 50053) +//! - ML Training Service (port 50054) +//! +//! All tests use REAL gRPC communication with REAL services (no mocks). + +#[path = "common/mod.rs"] +mod common; + +use anyhow::Result; +use common::{cleanup_redis, generate_test_token, wait_for_redis}; +use std::time::Duration; +use tonic::transport::Channel; +use tonic::Request; + +const REDIS_URL: &str = "redis://localhost:6379"; +const API_GATEWAY_URL: &str = "http://localhost:50051"; +const TRADING_SERVICE_URL: &str = "http://localhost:50052"; +const BACKTESTING_SERVICE_URL: &str = "http://localhost:50053"; +const ML_TRAINING_SERVICE_URL: &str = "http://localhost:50054"; + +// Import proto definitions - we'll use tli's generated proto that includes all service definitions +// API Gateway tests need to use tli's proto definitions since they're testing the client-facing interface +use fxt::proto::health::{health_client::HealthClient, HealthCheckRequest}; + +// We use the standard health client for all service health checks + +// ML Training proto from api +use api::ml_training::{ + ml_training_service_client::MlTrainingServiceClient, HealthCheckRequest as MlHealthRequest, +}; + +/// Helper to wait for a service to be ready +async fn wait_for_service_ready(url: &str, service_name: &str) -> Result<()> { + let max_attempts = 30; + let retry_delay = Duration::from_millis(500); + + for attempt in 1..=max_attempts { + match tokio::net::TcpStream::connect(url.trim_start_matches("http://")).await { + Ok(_) => { + println!("✓ {} is ready (attempt {})", service_name, attempt); + return Ok(()); + }, + Err(_) if attempt < max_attempts => { + tokio::time::sleep(retry_delay).await; + }, + Err(e) => { + return Err(anyhow::anyhow!( + "{} not ready after {} attempts: {}", + service_name, + max_attempts, + e + )); + }, + } + } + + unreachable!() +} + +/// Setup test environment (Redis, wait for services) +async fn setup_test_environment() -> Result<()> { + // Wait for Redis + wait_for_redis(REDIS_URL, 50).await?; + cleanup_redis(REDIS_URL).await?; + + // Wait for all backend services + wait_for_service_ready("localhost:50051", "API Gateway").await?; + wait_for_service_ready("localhost:50052", "Trading Service").await?; + wait_for_service_ready("localhost:50053", "Backtesting Service").await?; + wait_for_service_ready("localhost:50054", "ML Training Service").await?; + + println!("✓ All services are ready for testing"); + Ok(()) +} + +// ============================================================================ +// TRADING SERVICE INTEGRATION TESTS +// ============================================================================ + +#[tokio::test] +async fn test_trading_service_direct_connection() -> Result<()> { + println!("\n=== Test: Trading Service Direct Connection (No Proxy) ==="); + + setup_test_environment().await?; + + // Connect directly to Trading Service (bypass API Gateway) + let channel = Channel::from_static(TRADING_SERVICE_URL) + .connect() + .await + .map_err(|e| anyhow::anyhow!("Failed to connect to Trading Service: {}", e))?; + + let mut client = HealthClient::new(channel); + + // Call health check + let request = Request::new(HealthCheckRequest { + service: "trading".to_string(), + }); + let response = client + .check(request) + .await + .map_err(|e| anyhow::anyhow!("Trading Service health check failed: {}", e))?; + + let health = response.into_inner(); + use fxt::proto::health::ServingStatus; + let status_str = match ServingStatus::try_from(health.status) { + Ok(ServingStatus::Serving) => "serving", + Ok(ServingStatus::NotServing) => "not_serving", + Ok(ServingStatus::Unknown) | Ok(ServingStatus::ServiceUnknown) | Err(_) => "unknown", + }; + println!("✓ Trading Service health: {}", status_str); + assert_eq!( + health.status, ServingStatus::Serving as i32, + "Trading Service should be serving" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_trading_service_via_api_proxy() -> Result<()> { + println!("\n=== Test: Trading Service via API Gateway Proxy ==="); + + setup_test_environment().await?; + + // Generate valid JWT token + let (token, _jti) = generate_test_token( + "test_user", + vec!["trader".to_string()], + vec!["api.access".to_string(), "trading.submit".to_string()], + 3600, + )?; + + // Connect to API Gateway (which proxies to Trading Service) + let channel = Channel::from_static(API_GATEWAY_URL) + .connect() + .await + .map_err(|e| anyhow::anyhow!("Failed to connect to API Gateway: {}", e))?; + + let mut client = HealthClient::new(channel); + + // Call health check through API Gateway proxy + let mut request = Request::new(HealthCheckRequest { + service: "trading".to_string(), + }); + request.metadata_mut().insert( + "authorization", + format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"), + ); + + let start = std::time::Instant::now(); + let response = client + .check(request) + .await + .map_err(|e| anyhow::anyhow!("Trading Service health check via proxy failed: {}", e))?; + let elapsed = start.elapsed(); + + let health = response.into_inner(); + use fxt::proto::health::ServingStatus; + let status_str = match ServingStatus::try_from(health.status) { + Ok(ServingStatus::Serving) => "serving", + Ok(ServingStatus::NotServing) => "not_serving", + Ok(ServingStatus::Unknown) | Ok(ServingStatus::ServiceUnknown) | Err(_) => "unknown", + }; + println!("✓ Trading Service health via proxy: {}", status_str); + println!(" Proxy latency: {:?} (target: <1ms)", elapsed); + + assert_eq!( + health.status, ServingStatus::Serving as i32, + "Trading Service should be serving" + ); + assert!( + elapsed < Duration::from_millis(50), + "Proxy latency should be <50ms, got {:?}", + elapsed + ); + + Ok(()) +} + +#[tokio::test] +async fn test_trading_service_proxy_requires_auth() -> Result<()> { + println!("\n=== Test: Trading Service Proxy Requires Authentication ==="); + + setup_test_environment().await?; + + // Connect to API Gateway WITHOUT auth token + let channel = Channel::from_static(API_GATEWAY_URL) + .connect() + .await + .map_err(|e| anyhow::anyhow!("Failed to connect to API Gateway: {}", e))?; + + let mut client = HealthClient::new(channel); + + // Call health check without auth header + let request = Request::new(HealthCheckRequest { + service: "trading".to_string(), + }); + let result = client.check(request).await; + + // Should fail with UNAUTHENTICATED + assert!( + result.is_err(), + "Request without auth should be rejected by API Gateway" + ); + + let error = result.unwrap_err(); + println!("✓ Correctly rejected: {:?}", error.code()); + assert_eq!( + error.code(), + tonic::Code::Unauthenticated, + "Should return UNAUTHENTICATED" + ); + + Ok(()) +} + +// ============================================================================ +// BACKTESTING SERVICE INTEGRATION TESTS +// ============================================================================ + +#[tokio::test] +async fn test_backtesting_service_direct_connection() -> Result<()> { + println!("\n=== Test: Backtesting Service Direct Connection (No Proxy) ==="); + + setup_test_environment().await?; + + // Connect directly to Backtesting Service (bypass API Gateway) + let channel = Channel::from_static(BACKTESTING_SERVICE_URL) + .connect() + .await + .map_err(|e| anyhow::anyhow!("Failed to connect to Backtesting Service: {}", e))?; + + let mut client = HealthClient::new(channel); + + // Call health check + let request = Request::new(HealthCheckRequest { + service: "backtesting".to_string(), + }); + let response = client + .check(request) + .await + .map_err(|e| anyhow::anyhow!("Backtesting Service health check failed: {}", e))?; + + let health = response.into_inner(); + use fxt::proto::health::ServingStatus; + let status_str = match ServingStatus::try_from(health.status) { + Ok(ServingStatus::Serving) => "serving", + Ok(ServingStatus::NotServing) => "not_serving", + Ok(ServingStatus::Unknown) | Ok(ServingStatus::ServiceUnknown) | Err(_) => "unknown", + }; + println!("✓ Backtesting Service health: {}", status_str); + assert_eq!( + health.status, ServingStatus::Serving as i32, + "Backtesting Service should be serving" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_backtesting_service_via_api_proxy() -> Result<()> { + println!("\n=== Test: Backtesting Service via API Gateway Proxy ==="); + + setup_test_environment().await?; + + // Generate valid JWT token + let (token, _jti) = generate_test_token( + "test_user", + vec!["trader".to_string()], + vec!["api.access".to_string(), "backtesting.run".to_string()], + 3600, + )?; + + // Connect to API Gateway (which proxies to Backtesting Service) + let channel = Channel::from_static(API_GATEWAY_URL) + .connect() + .await + .map_err(|e| anyhow::anyhow!("Failed to connect to API Gateway: {}", e))?; + + let mut client = HealthClient::new(channel); + + // Call health check through API Gateway proxy + let mut request = Request::new(HealthCheckRequest { + service: "backtesting".to_string(), + }); + request.metadata_mut().insert( + "authorization", + format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"), + ); + + let start = std::time::Instant::now(); + let response = client + .check(request) + .await + .map_err(|e| anyhow::anyhow!("Backtesting Service health check via proxy failed: {}", e))?; + let elapsed = start.elapsed(); + + let health = response.into_inner(); + use fxt::proto::health::ServingStatus; + let status_str = match ServingStatus::try_from(health.status) { + Ok(ServingStatus::Serving) => "serving", + Ok(ServingStatus::NotServing) => "not_serving", + Ok(ServingStatus::Unknown) | Ok(ServingStatus::ServiceUnknown) | Err(_) => "unknown", + }; + println!("✓ Backtesting Service health via proxy: {}", status_str); + println!(" Proxy latency: {:?} (target: <1ms)", elapsed); + + assert_eq!( + health.status, ServingStatus::Serving as i32, + "Backtesting Service should be serving" + ); + assert!( + elapsed < Duration::from_millis(50), + "Proxy latency should be <50ms, got {:?}", + elapsed + ); + + Ok(()) +} + +#[tokio::test] +async fn test_backtesting_service_proxy_requires_auth() -> Result<()> { + println!("\n=== Test: Backtesting Service Proxy Requires Authentication ==="); + + setup_test_environment().await?; + + // Connect to API Gateway WITHOUT auth token + let channel = Channel::from_static(API_GATEWAY_URL) + .connect() + .await + .map_err(|e| anyhow::anyhow!("Failed to connect to API Gateway: {}", e))?; + + let mut client = HealthClient::new(channel); + + // Call health check without auth header + let request = Request::new(HealthCheckRequest { + service: "backtesting".to_string(), + }); + let result = client.check(request).await; + + // Should fail with UNAUTHENTICATED + assert!( + result.is_err(), + "Request without auth should be rejected by API Gateway" + ); + + let error = result.unwrap_err(); + println!("✓ Correctly rejected: {:?}", error.code()); + assert_eq!( + error.code(), + tonic::Code::Unauthenticated, + "Should return UNAUTHENTICATED" + ); + + Ok(()) +} + +// ============================================================================ +// ML TRAINING SERVICE INTEGRATION TESTS +// ============================================================================ + +#[tokio::test] +async fn test_ml_training_service_direct_connection() -> Result<()> { + println!("\n=== Test: ML Training Service Direct Connection (No Proxy) ==="); + + setup_test_environment().await?; + + // Connect directly to ML Training Service (bypass API Gateway) + let channel = Channel::from_static(ML_TRAINING_SERVICE_URL) + .connect() + .await + .map_err(|e| anyhow::anyhow!("Failed to connect to ML Training Service: {}", e))?; + + let mut client = MlTrainingServiceClient::new(channel); + + // Call health check + let request = Request::new(MlHealthRequest {}); + let response = client + .health_check(request) + .await + .map_err(|e| anyhow::anyhow!("ML Training Service health check failed: {}", e))?; + + let health = response.into_inner(); + println!("✓ ML Training Service health: {}", if health.healthy { "healthy" } else { "unhealthy" }); + assert!( + health.healthy, + "ML Training Service should be healthy: {}", + health.message + ); + + Ok(()) +} + +#[tokio::test] +async fn test_ml_training_service_via_api_proxy() -> Result<()> { + println!("\n=== Test: ML Training Service via API Gateway Proxy ==="); + + setup_test_environment().await?; + + // Generate valid JWT token + let (token, _jti) = generate_test_token( + "test_user", + vec!["admin".to_string()], + vec!["api.access".to_string(), "ml.train".to_string()], + 3600, + )?; + + // Connect to API Gateway (which proxies to ML Training Service) + let channel = Channel::from_static(API_GATEWAY_URL) + .connect() + .await + .map_err(|e| anyhow::anyhow!("Failed to connect to API Gateway: {}", e))?; + + let mut client = MlTrainingServiceClient::new(channel); + + // Call health check through API Gateway proxy + let mut request = Request::new(MlHealthRequest {}); + request.metadata_mut().insert( + "authorization", + format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"), + ); + + let start = std::time::Instant::now(); + let response = client + .health_check(request) + .await + .map_err(|e| anyhow::anyhow!("ML Training Service health check via proxy failed: {}", e))?; + let elapsed = start.elapsed(); + + let health = response.into_inner(); + println!("✓ ML Training Service health via proxy: {}", if health.healthy { "healthy" } else { "unhealthy" }); + println!(" Proxy latency: {:?} (target: <1ms)", elapsed); + + assert!( + health.healthy, + "ML Training Service should be healthy: {}", + health.message + ); + assert!( + elapsed < Duration::from_millis(50), + "Proxy latency should be <50ms, got {:?}", + elapsed + ); + + Ok(()) +} + +#[tokio::test] +async fn test_ml_training_service_proxy_requires_auth() -> Result<()> { + println!("\n=== Test: ML Training Service Proxy Requires Authentication ==="); + + setup_test_environment().await?; + + // Connect to API Gateway WITHOUT auth token + let channel = Channel::from_static(API_GATEWAY_URL) + .connect() + .await + .map_err(|e| anyhow::anyhow!("Failed to connect to API Gateway: {}", e))?; + + let mut client = MlTrainingServiceClient::new(channel); + + // Call health check without auth header + let request = Request::new(MlHealthRequest {}); + let result = client.health_check(request).await; + + // Should fail with UNAUTHENTICATED + assert!( + result.is_err(), + "Request without auth should be rejected by API Gateway" + ); + + let error = result.unwrap_err(); + println!("✓ Correctly rejected: {:?}", error.code()); + assert_eq!( + error.code(), + tonic::Code::Unauthenticated, + "Should return UNAUTHENTICATED" + ); + + Ok(()) +} + +// ============================================================================ +// MULTI-SERVICE INTEGRATION TESTS +// ============================================================================ + +#[tokio::test] +async fn test_api_routes_to_all_backend_services() -> Result<()> { + println!("\n=== Test: API Gateway Routes to All Backend Services ==="); + + setup_test_environment().await?; + + // Generate valid JWT token + let (token, _jti) = generate_test_token( + "test_user", + vec!["admin".to_string(), "trader".to_string()], + vec![ + "api.access".to_string(), + "trading.submit".to_string(), + "backtesting.run".to_string(), + "ml.train".to_string(), + ], + 3600, + )?; + + // Connect to API Gateway once + let channel = Channel::from_static(API_GATEWAY_URL) + .connect() + .await + .map_err(|e| anyhow::anyhow!("Failed to connect to API Gateway: {}", e))?; + + // Test Trading Service routing + { + let mut client = HealthClient::new(channel.clone()); + let mut request = Request::new(HealthCheckRequest { + service: "trading".to_string(), + }); + request.metadata_mut().insert( + "authorization", + format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"), + ); + + let response = client.check(request).await?; + use fxt::proto::health::ServingStatus; + let health = response.into_inner(); + let status_str = match ServingStatus::try_from(health.status) { + Ok(ServingStatus::Serving) => "serving", + Ok(ServingStatus::NotServing) => "not_serving", + Ok(ServingStatus::Unknown) | Ok(ServingStatus::ServiceUnknown) | Err(_) => "unknown", + }; + println!(" ✓ Trading Service: {}", status_str); + } + + // Test Backtesting Service routing + { + let mut client = HealthClient::new(channel.clone()); + let mut request = Request::new(HealthCheckRequest { + service: "backtesting".to_string(), + }); + request.metadata_mut().insert( + "authorization", + format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"), + ); + + let response = client.check(request).await?; + use fxt::proto::health::ServingStatus; + let health = response.into_inner(); + let status_str = match ServingStatus::try_from(health.status) { + Ok(ServingStatus::Serving) => "serving", + Ok(ServingStatus::NotServing) => "not_serving", + Ok(ServingStatus::Unknown) | Ok(ServingStatus::ServiceUnknown) | Err(_) => "unknown", + }; + println!(" ✓ Backtesting Service: {}", status_str); + } + + // Test ML Training Service routing + { + let mut client = MlTrainingServiceClient::new(channel.clone()); + let mut request = Request::new(MlHealthRequest {}); + request.metadata_mut().insert( + "authorization", + format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"), + ); + + let response = client.health_check(request).await?; + let health = response.into_inner(); + println!(" ✓ ML Training Service: {}", if health.healthy { "healthy" } else { "unhealthy" }); + } + + println!("✓ API Gateway successfully routes to all 3 backend services"); + + Ok(()) +} + +#[tokio::test] +async fn test_api_proxy_latency_across_services() -> Result<()> { + println!("\n=== Test: API Gateway Proxy Latency Across All Services ==="); + + setup_test_environment().await?; + + // Generate valid JWT token + let (token, _jti) = generate_test_token( + "test_user", + vec!["admin".to_string(), "trader".to_string()], + vec![ + "api.access".to_string(), + "trading.submit".to_string(), + "backtesting.run".to_string(), + "ml.train".to_string(), + ], + 3600, + )?; + + let channel = Channel::from_static(API_GATEWAY_URL).connect().await?; + + let mut latencies = Vec::new(); + + // Measure Trading Service latency (10 samples) + for _ in 0..10 { + let mut client = HealthClient::new(channel.clone()); + let mut request = Request::new(HealthCheckRequest { + service: "trading".to_string(), + }); + request.metadata_mut().insert( + "authorization", + format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"), + ); + + let start = std::time::Instant::now(); + let _ = client.check(request).await?; + latencies.push(start.elapsed()); + } + + // Calculate P50, P95, P99 + latencies.sort(); + let p50 = latencies[latencies.len() / 2]; + let p95 = latencies[latencies.len() * 95 / 100]; + let p99 = latencies[latencies.len() * 99 / 100]; + + println!("\n Proxy Latency Statistics:"); + println!(" ├─ P50: {:?}", p50); + println!(" ├─ P95: {:?}", p95); + println!(" └─ P99: {:?}", p99); + + assert!( + p99 < Duration::from_millis(50), + "P99 latency should be <50ms, got {:?}", + p99 + ); + println!("✓ Proxy latency within acceptable bounds"); + + Ok(()) +} diff --git a/services/api/tests/regime_endpoint_tests.rs b/services/api/tests/regime_endpoint_tests.rs new file mode 100644 index 000000000..d10072179 --- /dev/null +++ b/services/api/tests/regime_endpoint_tests.rs @@ -0,0 +1,51 @@ +//! Wave D: Regime Detection API Endpoint Tests +//! +//! Tests for the new regime state and transition endpoints added to the API Gateway. +//! These tests verify proto message translation and endpoint routing. +//! +//! Agent: D35 +//! Date: 2025-10-17 + +#[cfg(test)] +mod tests { + /// Test that GetRegimeStateRequest proto message compiles + #[test] + fn test_get_regime_state_request_proto() { + // This test ensures the proto definition is valid + // Actual RPC testing requires running services + } + + /// Test that GetRegimeStateResponse proto message compiles + #[test] + fn test_get_regime_state_response_proto() { + // This test ensures the proto definition is valid + // Actual RPC testing requires running services + } + + /// Test that GetRegimeTransitionsRequest proto message compiles + #[test] + fn test_get_regime_transitions_request_proto() { + // This test ensures the proto definition is valid + // Actual RPC testing requires running services + } + + /// Test that GetRegimeTransitionsResponse proto message compiles + #[test] + fn test_get_regime_transitions_response_proto() { + // This test ensures the proto definition is valid + // Actual RPC testing requires running services + } + + /// Test that RegimeTransition proto message compiles + #[test] + fn test_regime_transition_proto() { + // This test ensures the proto definition is valid + // Actual RPC testing requires running services + } +} + +// Note: Full integration tests require: +// 1. Running PostgreSQL with regime_states and regime_transitions tables +// 2. Running Trading Service with regime endpoint implementations +// 3. Running API Gateway +// These will be added in Agent D36 (Trading Service Implementation) diff --git a/services/api/tests/regime_routing_integration_test.rs b/services/api/tests/regime_routing_integration_test.rs new file mode 100644 index 000000000..a6e9efb4e --- /dev/null +++ b/services/api/tests/regime_routing_integration_test.rs @@ -0,0 +1,606 @@ +//! Agent F8: Regime Endpoint Routing Integration Tests +//! +//! Comprehensive validation of API Gateway routing, authentication, rate limiting, +//! and performance for regime detection endpoints. +//! +//! Tests: +//! 1. Routing validation (GetRegimeState, GetRegimeTransitions) +//! 2. Authentication enforcement (valid JWT required) +//! 3. Rate limiting operational (requests within quota) +//! 4. Proxy latency (target: < 1ms) +//! 5. Concurrent requests (10 parallel) +//! +//! Requirements: +//! - API Gateway running on port 50051 +//! - Trading Service running on port 50052 +//! - Redis running on port 6379 +//! - PostgreSQL running on port 5432 +//! +//! Run with: +//! ``` +//! cargo test -p api --test regime_routing_integration_test --ignored -- --nocapture +//! ``` + +#[path = "common/mod.rs"] +mod common; + +use anyhow::Result; +use std::time::Instant; +use tonic::{metadata::MetadataValue, Request}; + +use api::foxhunt::tli::{ + trading_service_client::TradingServiceClient, GetRegimeStateRequest, + GetRegimeTransitionsRequest, +}; + +/// Test 1: Basic Routing - GetRegimeState +#[tokio::test] +#[ignore = "Run manually with services running"] +async fn test_get_regime_state_routing() -> Result<()> { + println!("\n=== Test 1: GetRegimeState Routing ==="); + + // Create authenticated request + let (token, _jti) = common::generate_test_token( + "regime-test-user", + vec!["trader".to_string()], + vec!["trading.regime_state".to_string()], + 3600, + )?; + + let mut client = TradingServiceClient::connect("http://localhost:50051").await?; + + let req = GetRegimeStateRequest { + symbol: "ES.FUT".to_string(), + }; + + let mut request = Request::new(req); + let auth_value = MetadataValue::try_from(format!("Bearer {}", token))?; + request.metadata_mut().insert("authorization", auth_value); + + let start = Instant::now(); + let response = client.get_regime_state(request).await; + let elapsed = start.elapsed(); + + println!(" Response time: {:?}", elapsed); + + match response { + Ok(resp) => { + let regime = resp.into_inner(); + println!(" ✓ Routing successful"); + println!(" Symbol: {}", regime.symbol); + println!(" Regime: {}", regime.current_regime); + println!(" Confidence: {:.2}", regime.confidence); + println!(" ADX: {:.2}", regime.adx); + println!(" Latency: {:?}", elapsed); + + // Validate latency target + if elapsed.as_millis() < 1 { + println!(" ✅ PASS: Latency {} μs < 1ms target", elapsed.as_micros()); + } else { + println!( + " ⚠️ WARNING: Latency {} ms >= 1ms target", + elapsed.as_millis() + ); + } + + Ok(()) + }, + Err(e) => { + println!(" ❌ Routing failed: {:?}", e); + println!(" Status code: {:?}", e.code()); + println!(" Message: {}", e.message()); + Err(anyhow::anyhow!("GetRegimeState routing failed: {}", e)) + }, + } +} + +/// Test 2: Basic Routing - GetRegimeTransitions +#[tokio::test] +#[ignore = "Run manually with services running"] +async fn test_get_regime_transitions_routing() -> Result<()> { + println!("\n=== Test 2: GetRegimeTransitions Routing ==="); + + // Create authenticated request + let (token, _jti) = common::generate_test_token( + "regime-test-user", + vec!["trader".to_string()], + vec!["trading.regime_transitions".to_string()], + 3600, + )?; + + let mut client = TradingServiceClient::connect("http://localhost:50051").await?; + + let req = GetRegimeTransitionsRequest { + symbol: "ES.FUT".to_string(), + limit: 10, + }; + + let mut request = Request::new(req); + let auth_value = MetadataValue::try_from(format!("Bearer {}", token))?; + request.metadata_mut().insert("authorization", auth_value); + + let start = Instant::now(); + let response = client.get_regime_transitions(request).await; + let elapsed = start.elapsed(); + + println!(" Response time: {:?}", elapsed); + + match response { + Ok(resp) => { + let transitions = resp.into_inner(); + println!(" ✓ Routing successful"); + println!(" Transitions count: {}", transitions.transitions.len()); + + if !transitions.transitions.is_empty() { + let first = &transitions.transitions[0]; + println!( + " First transition: {} → {}", + first.from_regime, first.to_regime + ); + println!(" Duration: {} bars", first.duration_bars); + println!(" Probability: {:.2}", first.transition_probability); + } + + println!(" Latency: {:?}", elapsed); + + // Validate latency target + if elapsed.as_millis() < 1 { + println!(" ✅ PASS: Latency {} μs < 1ms target", elapsed.as_micros()); + } else { + println!( + " ⚠️ WARNING: Latency {} ms >= 1ms target", + elapsed.as_millis() + ); + } + + Ok(()) + }, + Err(e) => { + println!(" ❌ Routing failed: {:?}", e); + println!(" Status code: {:?}", e.code()); + println!(" Message: {}", e.message()); + Err(anyhow::anyhow!( + "GetRegimeTransitions routing failed: {}", + e + )) + }, + } +} + +/// Test 3: Authentication Enforcement - No Token +#[tokio::test] +#[ignore = "Run manually with services running"] +async fn test_authentication_no_token() -> Result<()> { + println!("\n=== Test 3: Authentication Enforcement (No Token) ==="); + + let mut client = TradingServiceClient::connect("http://localhost:50051").await?; + + let req = GetRegimeStateRequest { + symbol: "ES.FUT".to_string(), + }; + + let request = Request::new(req); // No authorization header + + let response = client.get_regime_state(request).await; + + match response { + Ok(_) => { + println!(" ❌ FAIL: Request succeeded without authentication"); + Err(anyhow::anyhow!("Authentication not enforced")) + }, + Err(e) => { + println!(" ✓ Request rejected (expected)"); + println!(" Status code: {:?}", e.code()); + println!(" Message: {}", e.message()); + + if e.code() == tonic::Code::Unauthenticated { + println!(" ✅ PASS: Correct error code (Unauthenticated)"); + Ok(()) + } else { + println!( + " ⚠️ WARNING: Expected Unauthenticated, got {:?}", + e.code() + ); + Ok(()) + } + }, + } +} + +/// Test 4: Authentication Enforcement - Invalid Token +#[tokio::test] +#[ignore = "Run manually with services running"] +async fn test_authentication_invalid_token() -> Result<()> { + println!("\n=== Test 4: Authentication Enforcement (Invalid Token) ==="); + + let mut client = TradingServiceClient::connect("http://localhost:50051").await?; + + let req = GetRegimeStateRequest { + symbol: "ES.FUT".to_string(), + }; + + let mut request = Request::new(req); + let auth_value = MetadataValue::try_from("Bearer invalid.token.signature")?; + request.metadata_mut().insert("authorization", auth_value); + + let response = client.get_regime_state(request).await; + + match response { + Ok(_) => { + println!(" ❌ FAIL: Request succeeded with invalid token"); + Err(anyhow::anyhow!("Invalid token not rejected")) + }, + Err(e) => { + println!(" ✓ Request rejected (expected)"); + println!(" Status code: {:?}", e.code()); + println!(" Message: {}", e.message()); + + if e.code() == tonic::Code::Unauthenticated { + println!(" ✅ PASS: Correct error code (Unauthenticated)"); + Ok(()) + } else { + println!( + " ⚠️ WARNING: Expected Unauthenticated, got {:?}", + e.code() + ); + Ok(()) + } + }, + } +} + +/// Test 5: Authentication Enforcement - Expired Token +#[tokio::test] +#[ignore = "Run manually with services running"] +async fn test_authentication_expired_token() -> Result<()> { + println!("\n=== Test 5: Authentication Enforcement (Expired Token) ==="); + + let token = common::generate_expired_token("expired-user")?; + + let mut client = TradingServiceClient::connect("http://localhost:50051").await?; + + let req = GetRegimeStateRequest { + symbol: "ES.FUT".to_string(), + }; + + let mut request = Request::new(req); + let auth_value = MetadataValue::try_from(format!("Bearer {}", token))?; + request.metadata_mut().insert("authorization", auth_value); + + let response = client.get_regime_state(request).await; + + match response { + Ok(_) => { + println!(" ❌ FAIL: Request succeeded with expired token"); + Err(anyhow::anyhow!("Expired token not rejected")) + }, + Err(e) => { + println!(" ✓ Request rejected (expected)"); + println!(" Status code: {:?}", e.code()); + println!(" Message: {}", e.message()); + + if e.code() == tonic::Code::Unauthenticated { + println!(" ✅ PASS: Correct error code (Unauthenticated)"); + Ok(()) + } else { + println!( + " ⚠️ WARNING: Expected Unauthenticated, got {:?}", + e.code() + ); + Ok(()) + } + }, + } +} + +/// Test 6: Rate Limiting - Within Quota +#[tokio::test] +#[ignore = "Run manually with services running"] +async fn test_rate_limiting_within_quota() -> Result<()> { + println!("\n=== Test 6: Rate Limiting (Within Quota) ==="); + + // Create authenticated request + let (token, _jti) = common::generate_test_token( + "rate-limit-test-user", + vec!["trader".to_string()], + vec!["trading.regime_state".to_string()], + 3600, + )?; + + let mut client = TradingServiceClient::connect("http://localhost:50051").await?; + + // Send 10 requests (default rate limit is 100 req/s) + let mut success_count = 0; + let mut failed_count = 0; + + for i in 0..10 { + let req = GetRegimeStateRequest { + symbol: "ES.FUT".to_string(), + }; + + let mut request = Request::new(req); + let auth_value = MetadataValue::try_from(format!("Bearer {}", token))?; + request.metadata_mut().insert("authorization", auth_value); + + let response = client.get_regime_state(request).await; + + match response { + Ok(_) => success_count += 1, + Err(e) => { + if e.code() == tonic::Code::ResourceExhausted { + failed_count += 1; + println!(" ⚠️ Request {} rate limited (unexpected)", i + 1); + } + }, + } + } + + println!(" Successful requests: {}/10", success_count); + println!(" Rate limited: {}/10", failed_count); + + if success_count == 10 { + println!(" ✅ PASS: All requests within quota succeeded"); + Ok(()) + } else { + println!(" ⚠️ WARNING: {} requests rate limited", failed_count); + Ok(()) + } +} + +/// Test 7: Proxy Latency Measurement +#[tokio::test] +#[ignore = "Run manually with services running"] +async fn test_proxy_latency_measurement() -> Result<()> { + println!("\n=== Test 7: Proxy Latency Measurement ==="); + + // Create authenticated request + let (token, _jti) = common::generate_test_token( + "latency-test-user", + vec!["trader".to_string()], + vec!["trading.regime_state".to_string()], + 3600, + )?; + + let mut client = TradingServiceClient::connect("http://localhost:50051").await?; + + // Warmup: 100 requests + println!(" Warming up with 100 requests..."); + for _ in 0..100 { + let req = GetRegimeStateRequest { + symbol: "ES.FUT".to_string(), + }; + + let mut request = Request::new(req); + let auth_value = MetadataValue::try_from(format!("Bearer {}", token))?; + request.metadata_mut().insert("authorization", auth_value); + + let _ = client.get_regime_state(request).await; + } + + // Measure 1000 warm requests + let mut latencies = Vec::new(); + println!(" Measuring 1000 warm requests..."); + + for _ in 0..1000 { + let req = GetRegimeStateRequest { + symbol: "ES.FUT".to_string(), + }; + + let mut request = Request::new(req); + let auth_value = MetadataValue::try_from(format!("Bearer {}", token))?; + request.metadata_mut().insert("authorization", auth_value); + + let start = Instant::now(); + let _ = client.get_regime_state(request).await; + latencies.push(start.elapsed()); + } + + latencies.sort(); + let p50 = latencies[latencies.len() / 2]; + let p95 = latencies[(latencies.len() as f64 * 0.95) as usize]; + let p99 = latencies[(latencies.len() as f64 * 0.99) as usize]; + let min = latencies[0]; + let max = latencies[latencies.len() - 1]; + + println!("\n 📊 Proxy Latency Statistics:"); + println!(" Min: {:>8} μs", min.as_micros()); + println!(" P50: {:>8} μs", p50.as_micros()); + println!(" P95: {:>8} μs", p95.as_micros()); + println!(" P99: {:>8} μs", p99.as_micros()); + println!(" Max: {:>8} μs", max.as_micros()); + println!(" Target: < 1,000 μs (1ms)"); + + if p99.as_micros() < 1000 { + println!(" ✅ PASS: P99 {} μs < 1ms target", p99.as_micros()); + Ok(()) + } else { + println!(" ⚠️ WARNING: P99 {} μs >= 1ms target", p99.as_micros()); + Ok(()) + } +} + +/// Test 8: Concurrent Requests (10 parallel) +#[tokio::test] +#[ignore = "Run manually with services running"] +async fn test_concurrent_requests() -> Result<()> { + println!("\n=== Test 8: Concurrent Requests (10 parallel) ==="); + + // Create authenticated request + let (token, _jti) = common::generate_test_token( + "concurrent-test-user", + vec!["trader".to_string()], + vec![ + "trading.regime_state".to_string(), + "trading.regime_transitions".to_string(), + ], + 3600, + )?; + + let start = Instant::now(); + + let mut handles = vec![]; + for i in 0..10 { + let token_clone = token.clone(); + let handle = tokio::spawn(async move { + let mut client = TradingServiceClient::connect("http://localhost:50051") + .await + .unwrap(); + + // Alternate between GetRegimeState and GetRegimeTransitions + let result = if i % 2 == 0 { + let req = GetRegimeStateRequest { + symbol: "ES.FUT".to_string(), + }; + + let mut request = Request::new(req); + let auth_value = + MetadataValue::try_from(format!("Bearer {}", token_clone)).unwrap(); + request.metadata_mut().insert("authorization", auth_value); + + client.get_regime_state(request).await.map(|_| ()) + } else { + let req = GetRegimeTransitionsRequest { + symbol: "ES.FUT".to_string(), + limit: 10, + }; + + let mut request = Request::new(req); + let auth_value = + MetadataValue::try_from(format!("Bearer {}", token_clone)).unwrap(); + request.metadata_mut().insert("authorization", auth_value); + + client.get_regime_transitions(request).await.map(|_| ()) + }; + result + }); + handles.push(handle); + } + + let mut success_count = 0; + let mut error_count = 0; + + for handle in handles { + match handle.await { + Ok(Ok(_)) => success_count += 1, + Ok(Err(e)) => { + error_count += 1; + println!(" ⚠️ Request failed: {:?}", e); + }, + Err(e) => { + error_count += 1; + println!(" ⚠️ Task panicked: {:?}", e); + }, + } + } + + let elapsed = start.elapsed(); + let avg_per_request = elapsed / 10; + + println!(" Total time: {:?}", elapsed); + println!(" Avg/request: {:?}", avg_per_request); + println!(" Successful: {}/10", success_count); + println!(" Failed: {}/10", error_count); + + if success_count == 10 { + println!(" ✅ PASS: All concurrent requests succeeded"); + Ok(()) + } else { + println!(" ⚠️ WARNING: {}/10 requests failed", error_count); + Ok(()) + } +} + +/// Test 9: Metadata Forwarding +#[tokio::test] +#[ignore = "Run manually with services running"] +async fn test_metadata_forwarding() -> Result<()> { + println!("\n=== Test 9: Metadata Forwarding ==="); + + // Create authenticated request with custom metadata + let (token, _jti) = common::generate_test_token( + "metadata-test-user", + vec!["trader".to_string()], + vec!["trading.regime_state".to_string()], + 3600, + )?; + + let mut client = TradingServiceClient::connect("http://localhost:50051").await?; + + let req = GetRegimeStateRequest { + symbol: "ES.FUT".to_string(), + }; + + let mut request = Request::new(req); + let auth_value = MetadataValue::try_from(format!("Bearer {}", token))?; + request.metadata_mut().insert("authorization", auth_value); + + // Add custom metadata + let user_id_value = MetadataValue::try_from("metadata-test-user")?; + request.metadata_mut().insert("x-user-id", user_id_value); + + let response = client.get_regime_state(request).await; + + match response { + Ok(resp) => { + println!(" ✓ Request succeeded with custom metadata"); + let regime = resp.into_inner(); + println!(" Symbol: {}", regime.symbol); + println!(" Regime: {}", regime.current_regime); + println!(" ✅ PASS: Metadata forwarding works"); + Ok(()) + }, + Err(e) => { + println!(" ❌ Request failed: {:?}", e); + Err(anyhow::anyhow!("Metadata forwarding test failed: {}", e)) + }, + } +} + +/// Test 10: Circuit Breaker Behavior (Simulated Backend Failure) +#[tokio::test] +#[ignore = "Run manually with services running - requires stopping Trading Service"] +async fn test_circuit_breaker_backend_failure() -> Result<()> { + println!("\n=== Test 10: Circuit Breaker (Backend Failure) ==="); + println!(" NOTE: This test requires stopping the Trading Service to simulate failure"); + + // Create authenticated request + let (token, _jti) = common::generate_test_token( + "circuit-breaker-test-user", + vec!["trader".to_string()], + vec!["trading.regime_state".to_string()], + 3600, + )?; + + let mut client = TradingServiceClient::connect("http://localhost:50051").await?; + + let req = GetRegimeStateRequest { + symbol: "ES.FUT".to_string(), + }; + + let mut request = Request::new(req); + let auth_value = MetadataValue::try_from(format!("Bearer {}", token))?; + request.metadata_mut().insert("authorization", auth_value); + + let response = client.get_regime_state(request).await; + + match response { + Ok(_) => { + println!(" ✓ Backend is available (test requires backend to be down)"); + println!(" ⚠️ SKIPPED: Stop Trading Service to test circuit breaker"); + Ok(()) + }, + Err(e) => { + println!(" ✓ Request failed (expected when backend is down)"); + println!(" Status code: {:?}", e.code()); + println!(" Message: {}", e.message()); + + if e.code() == tonic::Code::Unavailable { + println!(" ✅ PASS: Circuit breaker opened (Unavailable)"); + Ok(()) + } else { + println!(" ⚠️ WARNING: Expected Unavailable, got {:?}", e.code()); + Ok(()) + } + }, + } +} diff --git a/services/api/tests/routing_edge_cases.rs b/services/api/tests/routing_edge_cases.rs new file mode 100644 index 000000000..c9dc6d36d --- /dev/null +++ b/services/api/tests/routing_edge_cases.rs @@ -0,0 +1,595 @@ +//! Request Routing and Backend Failure Edge Case Tests +//! +//! This test suite focuses on request routing scenarios including: +//! +//! 1. Backend Service Failures: +//! - Connection refused +//! - Service timeout +//! - Network errors +//! - Circuit breaker activation +//! +//! 2. Load Balancing: +//! - Round-robin distribution +//! - Failover to healthy backends +//! - Sticky sessions +//! +//! 3. Service Discovery: +//! - Backend registration +//! - Health check integration +//! - Dynamic endpoint updates +//! +//! 4. Timeout Handling: +//! - Request timeout +//! - Connection timeout +//! - Streaming timeout + +use anyhow::Result; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::time::timeout; +use tonic::transport::Endpoint; +use tonic::Status; + +// ============================================================================ +// Backend Connection Tests +// ============================================================================ + +#[tokio::test] +async fn test_backend_connection_refused() -> Result<()> { + println!("\n=== Test: Backend Connection Refused ==="); + + // Try to connect to non-existent backend + let endpoint = Endpoint::from_static("http://localhost:59999") // Non-existent port + .connect_timeout(Duration::from_millis(100)) + .timeout(Duration::from_millis(100)); + + let result = endpoint.connect().await; + + assert!( + result.is_err(), + "Connection to non-existent backend should fail" + ); + + if let Err(e) = result { + println!("✓ Connection refused as expected: {}", e); + } + + Ok(()) +} + +#[tokio::test] +async fn test_backend_connection_timeout() -> Result<()> { + println!("\n=== Test: Backend Connection Timeout ==="); + + // Use a non-routable IP (192.0.2.0 is TEST-NET-1 from RFC 5737) + let endpoint = Endpoint::from_static("http://192.0.2.1:50000") + .connect_timeout(Duration::from_millis(50)) + .timeout(Duration::from_millis(50)); + + let start = Instant::now(); + let result = endpoint.connect().await; + let elapsed = start.elapsed(); + + assert!(result.is_err(), "Connection should timeout"); + assert!( + elapsed < Duration::from_millis(200), + "Timeout should be enforced" + ); + + println!("✓ Connection timeout after {:?}", elapsed); + + Ok(()) +} + +#[tokio::test] +async fn test_invalid_endpoint_url() -> Result<()> { + println!("\n=== Test: Invalid Endpoint URL ==="); + + // Try invalid URL format + let result = Endpoint::from_shared("not-a-valid-url"); + + assert!(result.is_err(), "Invalid URL should be rejected"); + + if let Err(e) = result { + println!("✓ Invalid URL rejected: {}", e); + } + + Ok(()) +} + +#[tokio::test] +async fn test_missing_scheme_in_url() -> Result<()> { + println!("\n=== Test: Missing Scheme in URL ==="); + + // URL without http:// or https:// + let result = Endpoint::from_shared("localhost:50000"); + + assert!(result.is_err(), "URL without scheme should be rejected"); + + if let Err(e) = result { + println!("✓ URL without scheme rejected: {}", e); + } + + Ok(()) +} + +// ============================================================================ +// Request Timeout Tests +// ============================================================================ + +#[tokio::test] +async fn test_request_timeout_enforced() -> Result<()> { + println!("\n=== Test: Request Timeout Enforced ==="); + + // Simulate long-running operation + let operation = async { + tokio::time::sleep(Duration::from_millis(500)).await; + Ok::<_, anyhow::Error>("completed") + }; + + // Apply 100ms timeout + let start = Instant::now(); + let result = timeout(Duration::from_millis(100), operation).await; + let elapsed = start.elapsed(); + + assert!(result.is_err(), "Request should timeout"); + assert!( + elapsed < Duration::from_millis(200), + "Timeout should be enforced quickly" + ); + + println!("✓ Request timeout enforced after {:?}", elapsed); + + Ok(()) +} + +#[tokio::test] +async fn test_streaming_timeout() -> Result<()> { + println!("\n=== Test: Streaming Timeout ==="); + + // Simulate slow streaming response + let stream_operation = async { + tokio::time::sleep(Duration::from_millis(500)).await; + Ok::<_, anyhow::Error>("stream chunk") + }; + + // Apply timeout + let start = Instant::now(); + let result = timeout(Duration::from_millis(100), stream_operation).await; + let elapsed = start.elapsed(); + + assert!(result.is_err(), "Streaming should timeout"); + println!("✓ Streaming timeout after {:?}", elapsed); + + Ok(()) +} + +// ============================================================================ +// Circuit Breaker Tests +// ============================================================================ + +#[tokio::test] +async fn test_circuit_breaker_opens_after_failures() -> Result<()> { + println!("\n=== Test: Circuit Breaker Opens After Failures ==="); + + let failure_threshold = 5; + let failure_count = Arc::new(AtomicUsize::new(0)); + let circuit_open = Arc::new(AtomicUsize::new(0)); // 0=closed, 1=open + + // Simulate consecutive failures + for i in 1..=10 { + // Simulate backend call failure + let count = failure_count.fetch_add(1, Ordering::SeqCst) + 1; + + if count >= failure_threshold && circuit_open.load(Ordering::SeqCst) == 0 { + circuit_open.store(1, Ordering::SeqCst); + println!(" ✓ Circuit breaker opened after {} failures", count); + } + + if circuit_open.load(Ordering::SeqCst) == 1 { + println!(" Request #{}: Circuit OPEN - fail fast", i); + } else { + println!(" Request #{}: Circuit CLOSED - attempting", i); + } + } + + let final_count = failure_count.load(Ordering::SeqCst); + let is_open = circuit_open.load(Ordering::SeqCst) == 1; + + assert!(is_open, "Circuit breaker should be open"); + assert!( + final_count >= failure_threshold, + "Should have recorded all failures" + ); + + println!( + "✓ Circuit breaker correctly opened after {} failures", + final_count + ); + + Ok(()) +} + +#[tokio::test] +async fn test_circuit_breaker_half_open_state() -> Result<()> { + println!("\n=== Test: Circuit Breaker Half-Open State ==="); + + let circuit_state = Arc::new(AtomicUsize::new(1)); // 0=closed, 1=open, 2=half-open + + // Simulate circuit opening + println!(" Circuit state: OPEN"); + + // Wait for timeout (simulate) + tokio::time::sleep(Duration::from_millis(50)).await; + + // Transition to half-open + circuit_state.store(2, Ordering::SeqCst); + println!(" Circuit state: HALF-OPEN (testing with probe request)"); + + // Simulate successful probe request + tokio::time::sleep(Duration::from_millis(10)).await; + let success = true; // Simulated success + + if success { + circuit_state.store(0, Ordering::SeqCst); // Close circuit + println!(" ✓ Probe succeeded - Circuit state: CLOSED"); + } else { + circuit_state.store(1, Ordering::SeqCst); // Reopen circuit + println!(" Probe failed - Circuit state: OPEN"); + } + + assert_eq!( + circuit_state.load(Ordering::SeqCst), + 0, + "Circuit should be closed after successful probe" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_circuit_breaker_reset_after_success() -> Result<()> { + println!("\n=== Test: Circuit Breaker Reset After Success ==="); + + let failure_count = Arc::new(AtomicUsize::new(0)); + let success_count = Arc::new(AtomicUsize::new(0)); + + // Simulate failures + for _ in 0..3 { + failure_count.fetch_add(1, Ordering::SeqCst); + } + + println!(" Failures: {}", failure_count.load(Ordering::SeqCst)); + + // Simulate success + success_count.fetch_add(1, Ordering::SeqCst); + failure_count.store(0, Ordering::SeqCst); // Reset on success + + println!( + " Success - failure count reset: {}", + failure_count.load(Ordering::SeqCst) + ); + + assert_eq!( + failure_count.load(Ordering::SeqCst), + 0, + "Failure count should reset" + ); + assert_eq!( + success_count.load(Ordering::SeqCst), + 1, + "Success should be recorded" + ); + + println!("✓ Circuit breaker reset successfully"); + + Ok(()) +} + +// ============================================================================ +// Load Balancing Tests +// ============================================================================ + +#[tokio::test] +async fn test_round_robin_distribution() -> Result<()> { + println!("\n=== Test: Round-Robin Load Distribution ==="); + + let backends = vec!["backend-1", "backend-2", "backend-3"]; + let current_index = Arc::new(AtomicUsize::new(0)); + let request_counts = Arc::new([ + AtomicUsize::new(0), + AtomicUsize::new(0), + AtomicUsize::new(0), + ]); + + // Simulate 15 requests with round-robin + for _ in 0..15 { + let index = current_index.fetch_add(1, Ordering::SeqCst) % backends.len(); + request_counts[index].fetch_add(1, Ordering::SeqCst); + } + + // Check distribution + let counts: Vec = request_counts + .iter() + .map(|c| c.load(Ordering::SeqCst)) + .collect(); + + println!("\n Request Distribution:"); + for (i, count) in counts.iter().enumerate() { + println!(" {} -> {} requests", backends[i], count); + } + + // Each backend should get 5 requests + for count in &counts { + assert_eq!(*count, 5, "Each backend should get equal requests"); + } + + println!("✓ Round-robin distribution working correctly"); + + Ok(()) +} + +#[tokio::test] +async fn test_failover_to_healthy_backend() -> Result<()> { + println!("\n=== Test: Failover to Healthy Backend ==="); + + struct Backend { + name: String, + healthy: bool, + } + + let backends = vec![ + Backend { + name: "backend-1".to_string(), + healthy: false, + }, // Unhealthy + Backend { + name: "backend-2".to_string(), + healthy: true, + }, // Healthy + Backend { + name: "backend-3".to_string(), + healthy: false, + }, // Unhealthy + ]; + + // Select first healthy backend + let selected = backends.iter().find(|b| b.healthy); + + assert!(selected.is_some(), "Should find healthy backend"); + + if let Some(backend) = selected { + println!("✓ Failover selected: {}", backend.name); + assert_eq!(backend.name, "backend-2"); + } + + Ok(()) +} + +#[tokio::test] +async fn test_all_backends_unhealthy() -> Result<()> { + println!("\n=== Test: All Backends Unhealthy ==="); + + struct Backend { + name: String, + healthy: bool, + } + + let backends = vec![ + Backend { + name: "backend-1".to_string(), + healthy: false, + }, + Backend { + name: "backend-2".to_string(), + healthy: false, + }, + Backend { + name: "backend-3".to_string(), + healthy: false, + }, + ]; + + // Try to find healthy backend + let selected = backends.iter().find(|b| b.healthy); + + assert!(selected.is_none(), "Should not find any healthy backend"); + println!("✓ Correctly detected no healthy backends"); + + Ok(()) +} + +// ============================================================================ +// Health Check Tests +// ============================================================================ + +#[tokio::test] +async fn test_health_check_marks_unhealthy_on_failure() -> Result<()> { + println!("\n=== Test: Health Check Marks Unhealthy on Failure ==="); + + let is_healthy = Arc::new(AtomicUsize::new(1)); // 1=healthy, 0=unhealthy + + // Simulate health check failure + let health_check_result = Err::<(), _>("Connection failed"); + + if health_check_result.is_err() { + is_healthy.store(0, Ordering::SeqCst); + println!(" ✓ Backend marked unhealthy"); + } + + assert_eq!( + is_healthy.load(Ordering::SeqCst), + 0, + "Backend should be unhealthy" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_health_check_recovers_on_success() -> Result<()> { + println!("\n=== Test: Health Check Recovers on Success ==="); + + let is_healthy = Arc::new(AtomicUsize::new(0)); // Start unhealthy + + println!(" Backend initially: UNHEALTHY"); + + // Simulate successful health check + let health_check_result = Ok::<(), &str>(()); + + if health_check_result.is_ok() { + is_healthy.store(1, Ordering::SeqCst); + println!(" ✓ Backend marked healthy"); + } + + assert_eq!( + is_healthy.load(Ordering::SeqCst), + 1, + "Backend should be healthy" + ); + + Ok(()) +} + +#[tokio::test] +async fn test_health_check_interval_respected() -> Result<()> { + println!("\n=== Test: Health Check Interval Respected ==="); + + let last_check = Arc::new(AtomicUsize::new(0)); + let check_interval_ms = 100; + + // First check + let now = Instant::now(); + last_check.store(now.elapsed().as_millis() as usize, Ordering::SeqCst); + println!(" First health check"); + + // Try immediate second check - should be skipped + let elapsed_since_last = now.elapsed().as_millis() as usize - last_check.load(Ordering::SeqCst); + + if elapsed_since_last < check_interval_ms { + println!(" Second check skipped (interval not elapsed)"); + } + + // Wait for interval + tokio::time::sleep(Duration::from_millis(check_interval_ms as u64 + 10)).await; + + // Now check should proceed + let elapsed_since_last = now.elapsed().as_millis() as usize - last_check.load(Ordering::SeqCst); + + if elapsed_since_last >= check_interval_ms { + println!(" Third check executed (interval elapsed)"); + last_check.store(now.elapsed().as_millis() as usize, Ordering::SeqCst); + } + + println!("✓ Health check interval correctly enforced"); + + Ok(()) +} + +// ============================================================================ +// Endpoint Configuration Tests +// ============================================================================ + +#[tokio::test] +async fn test_tcp_keepalive_configuration() -> Result<()> { + println!("\n=== Test: TCP Keepalive Configuration ==="); + + let endpoint = Endpoint::from_static("http://localhost:50000") + .tcp_keepalive(Some(Duration::from_secs(60))); + + println!("✓ TCP keepalive configured: 60s"); + + // Endpoint configured successfully + assert!(true); + + Ok(()) +} + +#[tokio::test] +async fn test_http2_keepalive_configuration() -> Result<()> { + println!("\n=== Test: HTTP/2 Keepalive Configuration ==="); + + let endpoint = Endpoint::from_static("http://localhost:50000") + .http2_keep_alive_interval(Duration::from_secs(30)); + + println!("✓ HTTP/2 keepalive configured: 30s"); + + assert!(true); + + Ok(()) +} + +#[tokio::test] +async fn test_multiple_endpoint_configurations() -> Result<()> { + println!("\n=== Test: Multiple Endpoint Configurations ==="); + + let endpoint = Endpoint::from_static("http://localhost:50000") + .connect_timeout(Duration::from_millis(5000)) + .timeout(Duration::from_millis(30000)) + .tcp_keepalive(Some(Duration::from_secs(60))) + .http2_keep_alive_interval(Duration::from_secs(30)); + + println!("✓ Endpoint configured with:"); + println!(" - Connect timeout: 5000ms"); + println!(" - Request timeout: 30000ms"); + println!(" - TCP keepalive: 60s"); + println!(" - HTTP/2 keepalive: 30s"); + + assert!(true); + + Ok(()) +} + +// ============================================================================ +// Error Handling Tests +// ============================================================================ + +#[tokio::test] +async fn test_status_code_mapping() -> Result<()> { + println!("\n=== Test: Status Code Mapping ==="); + + // Test various error scenarios + let errors = vec![ + (tonic::Code::Unavailable, "Service unavailable"), + (tonic::Code::DeadlineExceeded, "Request timeout"), + (tonic::Code::Internal, "Internal error"), + (tonic::Code::Unauthenticated, "Authentication failed"), + (tonic::Code::PermissionDenied, "Permission denied"), + ]; + + for (code, message) in errors { + let status = Status::new(code, message); + println!(" {} -> {}", code, status.message()); + assert_eq!(status.code(), code); + } + + println!("✓ Status code mapping correct"); + + Ok(()) +} + +#[tokio::test] +async fn test_metadata_propagation() -> Result<()> { + println!("\n=== Test: Metadata Propagation ==="); + + use tonic::metadata::{MetadataMap, MetadataValue}; + + let mut metadata = MetadataMap::new(); + metadata.insert("x-request-id", MetadataValue::try_from("req-123")?); + metadata.insert("x-user-id", MetadataValue::try_from("user-456")?); + + // Verify metadata + assert_eq!( + metadata.get("x-request-id").expect("INVARIANT: Key should exist in map"), + &MetadataValue::try_from("req-123")? + ); + assert_eq!( + metadata.get("x-user-id").expect("INVARIANT: Key should exist in map"), + &MetadataValue::try_from("user-456")? + ); + + println!("✓ Metadata propagation working"); + + Ok(()) +} diff --git a/services/api/tests/service_proxy_tests.rs b/services/api/tests/service_proxy_tests.rs new file mode 100644 index 000000000..b2178cf77 --- /dev/null +++ b/services/api/tests/service_proxy_tests.rs @@ -0,0 +1,317 @@ +//! Service Proxy Integration Tests +//! +//! Tests for backend service proxying with circuit breakers: +//! - Connection pooling +//! - Circuit breaker activation +//! - Request forwarding +//! - Health checking + +#[path = "common/mod.rs"] +mod common; + +use anyhow::Result; +use std::time::Duration; + +#[tokio::test] +async fn test_ml_training_proxy_config() -> Result<()> { + println!("\n=== Test: ML Training Proxy Configuration ==="); + + use api::grpc::server::MlTrainingBackendConfig; + + let config = MlTrainingBackendConfig::default(); + + println!(" Default configuration:"); + println!(" ├─ Address: {}", config.address); + println!(" ├─ Connect timeout: {}ms", config.connect_timeout_ms); + println!(" ├─ Request timeout: {}ms", config.request_timeout_ms); + println!(" ├─ CB failures: {}", config.circuit_breaker_failures); + println!(" └─ CB reset: {}s", config.circuit_breaker_reset_secs); + + assert_eq!(config.address, "http://localhost:50053"); + assert_eq!(config.connect_timeout_ms, 5000); + assert_eq!(config.request_timeout_ms, 30000); + assert_eq!(config.circuit_breaker_failures, 5); + assert_eq!(config.circuit_breaker_reset_secs, 30); + + Ok(()) +} + +#[tokio::test] +async fn test_ml_training_proxy_custom_config() -> Result<()> { + println!("\n=== Test: ML Training Proxy Custom Configuration ==="); + + use api::grpc::server::MlTrainingBackendConfig; + + let config = MlTrainingBackendConfig { + address: "http://custom-service:9999".to_string(), + connect_timeout_ms: 1000, + request_timeout_ms: 5000, + circuit_breaker_failures: 3, + circuit_breaker_reset_secs: 60, + tls_ca_cert_path: None, + tls_client_cert_path: None, + tls_client_key_path: None, + }; + + println!(" Custom configuration:"); + println!(" ├─ Address: {}", config.address); + println!(" ├─ Connect timeout: {}ms", config.connect_timeout_ms); + println!(" ├─ Request timeout: {}ms", config.request_timeout_ms); + println!(" ├─ CB failures: {}", config.circuit_breaker_failures); + println!(" └─ CB reset: {}s", config.circuit_breaker_reset_secs); + + assert_eq!(config.address, "http://custom-service:9999"); + assert_eq!(config.connect_timeout_ms, 1000); + assert_eq!(config.circuit_breaker_failures, 3); + + Ok(()) +} + +#[tokio::test] +async fn test_circuit_breaker_config_validation() -> Result<()> { + println!("\n=== Test: Circuit Breaker Configuration Validation ==="); + + use api::grpc::server::MlTrainingBackendConfig; + + let configs = vec![ + (1, 5, "Minimal failure threshold"), + (5, 10, "Moderate failure threshold"), + (10, 30, "High failure threshold"), + ]; + + for (failures, reset_secs, description) in configs { + let config = MlTrainingBackendConfig { + circuit_breaker_failures: failures, + circuit_breaker_reset_secs: reset_secs, + ..Default::default() + }; + + println!( + " ✓ Valid config: {} (failures={}, reset={}s)", + description, config.circuit_breaker_failures, config.circuit_breaker_reset_secs + ); + + assert!(config.circuit_breaker_failures > 0); + assert!(config.circuit_breaker_reset_secs > 0); + } + + Ok(()) +} + +#[tokio::test] +async fn test_connection_timeout_behavior() -> Result<()> { + println!("\n=== Test: Connection Timeout Behavior ==="); + + use api::grpc::server::{setup_ml_training_client, MlTrainingBackendConfig}; + + // Test with invalid address (should timeout) + let config = MlTrainingBackendConfig { + address: "http://non-existent-service:9999".to_string(), + connect_timeout_ms: 100, // Very short timeout + ..Default::default() + }; + + println!(" Attempting connection to non-existent service..."); + let start = std::time::Instant::now(); + let result = setup_ml_training_client(config).await; + let elapsed = start.elapsed(); + + println!(" Connection attempt took: {:?}", elapsed); + + assert!( + result.is_err(), + "Connection to non-existent service should fail" + ); + assert!( + elapsed < Duration::from_millis(500), + "Should timeout quickly (within 500ms)" + ); + + println!(" ✓ Connection timeout worked correctly"); + + Ok(()) +} + +#[tokio::test] +async fn test_service_proxy_error_handling() -> Result<()> { + println!("\n=== Test: Service Proxy Error Handling ==="); + + use api::grpc::server::{setup_ml_training_client, MlTrainingBackendConfig}; + + let test_cases = vec![ + ("http://localhost:1", "Connection refused (port 1)"), + ("http://192.0.2.1:50053", "Network unreachable (TEST-NET-1)"), + ( + "http://10.255.255.1:50053", + "Connection timeout (non-routable)", + ), + ]; + + for (address, description) in test_cases { + let config = MlTrainingBackendConfig { + address: address.to_string(), + connect_timeout_ms: 100, + ..Default::default() + }; + + let result = setup_ml_training_client(config).await; + + assert!(result.is_err(), "{} should fail", description); + println!(" ✓ Handled: {}", description); + } + + Ok(()) +} + +#[tokio::test] +async fn test_backend_config_serialization() -> Result<()> { + println!("\n=== Test: Backend Config Serialization ==="); + + use api::grpc::server::MlTrainingBackendConfig; + + let config = MlTrainingBackendConfig { + address: "http://ml-service:50053".to_string(), + connect_timeout_ms: 2000, + request_timeout_ms: 10000, + circuit_breaker_failures: 3, + circuit_breaker_reset_secs: 45, + tls_ca_cert_path: None, + tls_client_cert_path: None, + tls_client_key_path: None, + }; + + // Test Debug formatting + let debug_str = format!("{:?}", config); + assert!(debug_str.contains("ml-service:50053")); + assert!(debug_str.contains("2000")); + println!(" ✓ Debug format: {}", debug_str); + + // Test Clone + let cloned = config.clone(); + assert_eq!(cloned.address, config.address); + assert_eq!(cloned.connect_timeout_ms, config.connect_timeout_ms); + println!(" ✓ Clone works correctly"); + + Ok(()) +} + +#[tokio::test] +async fn test_multiple_backend_configs() -> Result<()> { + println!("\n=== Test: Multiple Backend Service Configurations ==="); + + use api::grpc::server::MlTrainingBackendConfig; + + // Simulate configurations for different environments + let dev_config = MlTrainingBackendConfig { + address: "http://localhost:50053".to_string(), + connect_timeout_ms: 5000, + request_timeout_ms: 30000, + circuit_breaker_failures: 5, + circuit_breaker_reset_secs: 30, + tls_ca_cert_path: None, + tls_client_cert_path: None, + tls_client_key_path: None, + }; + + let staging_config = MlTrainingBackendConfig { + address: "http://ml-training-staging:50053".to_string(), + connect_timeout_ms: 3000, + request_timeout_ms: 20000, + circuit_breaker_failures: 3, + circuit_breaker_reset_secs: 60, + tls_ca_cert_path: None, + tls_client_cert_path: None, + tls_client_key_path: None, + }; + + let prod_config = MlTrainingBackendConfig { + address: "http://ml-training-prod:50053".to_string(), + connect_timeout_ms: 2000, + request_timeout_ms: 15000, + circuit_breaker_failures: 3, + circuit_breaker_reset_secs: 120, + tls_ca_cert_path: None, + tls_client_cert_path: None, + tls_client_key_path: None, + }; + + println!(" Development: {}", dev_config.address); + println!(" Staging: {}", staging_config.address); + println!(" Production: {}", prod_config.address); + + // Verify configurations are independent + assert_ne!( + dev_config.connect_timeout_ms, + prod_config.connect_timeout_ms + ); + assert_ne!( + staging_config.circuit_breaker_reset_secs, + prod_config.circuit_breaker_reset_secs + ); + + println!(" ✓ Multiple environment configurations validated"); + + Ok(()) +} + +#[tokio::test] +async fn test_proxy_performance_overhead() -> Result<()> { + println!("\n=== Test: Proxy Configuration Performance ==="); + + use api::grpc::server::MlTrainingBackendConfig; + + let mut config_creation_times = Vec::new(); + + // Measure config creation overhead + for _ in 0..1000 { + let start = std::time::Instant::now(); + let _config = MlTrainingBackendConfig::default(); + let elapsed = start.elapsed(); + config_creation_times.push(elapsed); + } + + config_creation_times.sort(); + let p50 = config_creation_times[499]; + let p99 = config_creation_times[989]; + + println!("\n Config Creation Performance:"); + println!(" ├─ P50: {:?}", p50); + println!(" └─ P99: {:?}", p99); + + assert!( + p99 < Duration::from_micros(10), + "Config creation should be <10μs" + ); + println!(" ✓ Config creation overhead is minimal"); + + Ok(()) +} + +#[tokio::test] +async fn test_circuit_breaker_threshold_edge_cases() -> Result<()> { + println!("\n=== Test: Circuit Breaker Threshold Edge Cases ==="); + + use api::grpc::server::MlTrainingBackendConfig; + + // Test with threshold of 1 (opens after single failure) + let sensitive_config = MlTrainingBackendConfig { + circuit_breaker_failures: 1, + circuit_breaker_reset_secs: 5, + ..Default::default() + }; + + println!(" ✓ Sensitive CB (failures=1): Valid"); + assert_eq!(sensitive_config.circuit_breaker_failures, 1); + + // Test with high threshold (tolerates many failures) + let tolerant_config = MlTrainingBackendConfig { + circuit_breaker_failures: 100, + circuit_breaker_reset_secs: 300, + ..Default::default() + }; + + println!(" ✓ Tolerant CB (failures=100): Valid"); + assert_eq!(tolerant_config.circuit_breaker_failures, 100); + + Ok(()) +}