diff --git a/proto/broker_gateway.proto b/proto/broker_gateway.proto new file mode 100644 index 000000000..5677dfa76 --- /dev/null +++ b/proto/broker_gateway.proto @@ -0,0 +1,212 @@ +// Broker Gateway Service - FIX Order Routing Protocol +// +// This service handles all broker communication via FIX 4.2/4.4 protocol +// for order routing, execution management, and account state synchronization. + +syntax = "proto3"; + +package broker_gateway; + +// ============================================================================ +// Broker Gateway Service +// ============================================================================ + +service BrokerGatewayService { + // Route order to broker via FIX protocol + rpc RouteOrder(RouteOrderRequest) returns (RouteOrderResponse); + + // Cancel existing order + rpc CancelOrder(CancelOrderRequest) returns (CancelOrderResponse); + + // Get current account state (balance, margin, positions) + rpc GetAccountState(GetAccountStateRequest) returns (GetAccountStateResponse); + + // Get all positions for account + rpc GetPositions(GetPositionsRequest) returns (GetPositionsResponse); + + // Get FIX session status + rpc GetSessionStatus(GetSessionStatusRequest) returns (GetSessionStatusResponse); + + // Stream real-time executions from broker + rpc StreamExecutions(StreamExecutionsRequest) returns (stream ExecutionEvent); + + // Health check + rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse); +} + +// ============================================================================ +// Order Routing +// ============================================================================ + +message RouteOrderRequest { + string symbol = 1; // ES, NQ, etc. + OrderSide side = 2; // BUY, SELL + double quantity = 3; // Number of contracts + OrderType order_type = 4; // MARKET, LIMIT, STOP, STOP_LIMIT + optional double price = 5; // Limit price (required for LIMIT orders) + optional double stop_price = 6; // Stop price (required for STOP orders) + string account_id = 7; // AMP account identifier + map metadata = 8; // Strategy, model_name, etc. +} + +message RouteOrderResponse { + string broker_order_id = 1; // Broker-assigned OrderID (Tag 37, filled after ack) + string client_order_id = 2; // Our ClOrdID (Tag 11) + OrderStatus status = 3; // PENDING_SUBMIT, SUBMITTED, etc. + int64 submitted_at = 4; // Timestamp (nanoseconds) + string message = 5; // Success/error message +} + +message CancelOrderRequest { + string client_order_id = 1; // Order to cancel + string account_id = 2; // Account verification +} + +message CancelOrderResponse { + bool success = 1; + string message = 2; + OrderStatus new_status = 3; // CANCEL_PENDING, CANCELLED, etc. +} + +// ============================================================================ +// Account & Position Management +// ============================================================================ + +message GetAccountStateRequest { + string account_id = 1; +} + +message GetAccountStateResponse { + string account_id = 1; + double cash_balance = 2; + double equity = 3; + double margin_used = 4; + double margin_available = 5; + double buying_power = 6; + double unrealized_pnl = 7; + double realized_pnl = 8; + int64 last_updated = 9; // Timestamp (nanoseconds) +} + +message GetPositionsRequest { + string account_id = 1; + optional string symbol = 2; // Filter by symbol (optional) +} + +message GetPositionsResponse { + repeated Position positions = 1; + double total_equity = 2; + double total_exposure = 3; + double leverage_ratio = 4; + int64 timestamp = 5; +} + +message Position { + string symbol = 1; + double quantity = 2; // Positive = long, negative = short + double average_price = 3; + double market_value = 4; + double unrealized_pnl = 5; +} + +// ============================================================================ +// Session Management +// ============================================================================ + +message GetSessionStatusRequest { + optional string session_id = 1; // Optional: default to active session +} + +message GetSessionStatusResponse { + string session_id = 1; + SessionState state = 2; + int64 sender_seq_num = 3; // Current outgoing sequence + int64 target_seq_num = 4; // Expected incoming sequence + int64 last_heartbeat_sent = 5; // Timestamp (nanoseconds) + int64 last_heartbeat_received = 6; // Timestamp (nanoseconds) + double heartbeat_rtt_ms = 7; // Round-trip time in milliseconds + int64 connected_at = 8; // Timestamp (nanoseconds) + map details = 9; // Additional session info +} + +// ============================================================================ +// Execution Streaming +// ============================================================================ + +message StreamExecutionsRequest { + optional string account_id = 1; // Filter by account + optional string symbol = 2; // Filter by symbol +} + +message ExecutionEvent { + string execution_id = 1; // ExecID (Tag 17) + string broker_order_id = 2; // OrderID (Tag 37) + string client_order_id = 3; // ClOrdID (Tag 11) + string symbol = 4; + OrderSide side = 5; + ExecutionType exec_type = 6; // NEW, TRADE, CANCELED, REJECTED + OrderStatus order_status = 7; // Order status after this execution + double last_qty = 8; // Quantity filled (Tag 32) + double last_price = 9; // Fill price (Tag 31) + double cum_qty = 10; // Total filled (Tag 14) + double avg_price = 11; // Average fill price (Tag 6) + int64 transact_time = 12; // Execution timestamp + optional string text = 13; // Reject reason (if applicable) +} + +// ============================================================================ +// Health Check +// ============================================================================ + +message HealthCheckRequest {} + +message HealthCheckResponse { + bool healthy = 1; + string message = 2; + map details = 3; +} + +// ============================================================================ +// Enums +// ============================================================================ + +enum OrderSide { + ORDER_SIDE_UNSPECIFIED = 0; + ORDER_SIDE_BUY = 1; + ORDER_SIDE_SELL = 2; +} + +enum OrderType { + ORDER_TYPE_UNSPECIFIED = 0; + ORDER_TYPE_MARKET = 1; + ORDER_TYPE_LIMIT = 2; + ORDER_TYPE_STOP = 3; + ORDER_TYPE_STOP_LIMIT = 4; +} + +enum OrderStatus { + ORDER_STATUS_UNSPECIFIED = 0; + ORDER_STATUS_PENDING_SUBMIT = 1; + ORDER_STATUS_SUBMITTED = 2; + ORDER_STATUS_PARTIALLY_FILLED = 3; + ORDER_STATUS_FILLED = 4; + ORDER_STATUS_CANCEL_PENDING = 5; + ORDER_STATUS_CANCELLED = 6; + ORDER_STATUS_REJECTED = 7; +} + +enum ExecutionType { + EXECUTION_TYPE_UNSPECIFIED = 0; + EXECUTION_TYPE_NEW = 1; // Order accepted + EXECUTION_TYPE_TRADE = 2; // Partial or full fill + EXECUTION_TYPE_CANCELED = 3; // Order canceled + EXECUTION_TYPE_REJECTED = 4; // Order rejected +} + +enum SessionState { + SESSION_STATE_DISCONNECTED = 0; + SESSION_STATE_CONNECTED = 1; + SESSION_STATE_LOGGING_IN = 2; + SESSION_STATE_ACTIVE = 3; + SESSION_STATE_LOGGING_OUT = 4; +} diff --git a/proto/config.proto b/proto/config.proto new file mode 100644 index 000000000..aca45685c --- /dev/null +++ b/proto/config.proto @@ -0,0 +1,326 @@ +syntax = "proto3"; + +package config; + +// Configuration Service provides centralized, PostgreSQL-based configuration management with hot-reload capabilities. +// This service supports real-time configuration updates, validation, history tracking, and import/export functionality +// for all trading system components with comprehensive audit trails and rollback capabilities. +service ConfigService { + // Configuration CRUD Operations + // Get configuration settings by category, key, or environment + rpc GetConfiguration(GetConfigurationRequest) returns (GetConfigurationResponse); + + // Update configuration value with validation and audit logging + rpc UpdateConfiguration(UpdateConfigurationRequest) returns (UpdateConfigurationResponse); + + // Delete configuration setting with audit trail + rpc DeleteConfiguration(DeleteConfigurationRequest) returns (DeleteConfigurationResponse); + + // List available configuration categories + rpc ListCategories(ListCategoriesRequest) returns (ListCategoriesResponse); + + // Real-time Configuration Streaming + // Stream real-time configuration changes with hot-reload support + rpc StreamConfigChanges(StreamConfigChangesRequest) returns (stream ConfigChangeEvent); + + // Configuration Management Operations + // Validate configuration value against schema and rules + rpc ValidateConfiguration(ValidateConfigurationRequest) returns (ValidateConfigurationResponse); + + // Get configuration change history with audit details + rpc GetConfigurationHistory(GetConfigurationHistoryRequest) returns (GetConfigurationHistoryResponse); + + // Rollback configuration to previous value + rpc RollbackConfiguration(RollbackConfigurationRequest) returns (RollbackConfigurationResponse); + + // Export configuration data in various formats + rpc ExportConfiguration(ExportConfigurationRequest) returns (ExportConfigurationResponse); + + // Import configuration data with validation + rpc ImportConfiguration(ImportConfigurationRequest) returns (ImportConfigurationResponse); + + // Schema Management Operations + // Get configuration schema definitions + rpc GetConfigSchema(GetConfigSchemaRequest) returns (GetConfigSchemaResponse); + + // Update configuration schema with validation rules + rpc UpdateConfigSchema(UpdateConfigSchemaRequest) returns (UpdateConfigSchemaResponse); +} + +// Configuration CRUD Messages +message GetConfigurationRequest { + optional string category = 1; + optional string key = 2; + optional string environment = 3; +} + +message GetConfigurationResponse { + repeated ConfigurationSetting settings = 1; +} + +message UpdateConfigurationRequest { + string category = 1; + string key = 2; + string value = 3; + string changed_by = 4; + optional string change_reason = 5; + optional string environment = 6; +} + +message UpdateConfigurationResponse { + bool success = 1; + string message = 2; + optional ValidationResult validation_result = 3; + int64 timestamp = 4; +} + +message DeleteConfigurationRequest { + string category = 1; + string key = 2; + string deleted_by = 3; + optional string delete_reason = 4; +} + +message DeleteConfigurationResponse { + bool success = 1; + string message = 2; + int64 timestamp = 3; +} + +message ListCategoriesRequest { + optional string parent_category = 1; +} + +message ListCategoriesResponse { + repeated ConfigurationCategory categories = 1; +} + +// Streaming Messages +message StreamConfigChangesRequest { + repeated string categories = 1; + repeated string keys = 2; +} + +// Validation Messages +message ValidateConfigurationRequest { + string category = 1; + string key = 2; + string value = 3; +} + +message ValidateConfigurationResponse { + bool is_valid = 1; + ValidationResult validation_result = 2; +} + +// History Messages +message GetConfigurationHistoryRequest { + optional string category = 1; + optional string key = 2; + optional int64 start_time = 3; + optional int64 end_time = 4; + optional int32 limit = 5; +} + +message GetConfigurationHistoryResponse { + repeated ConfigurationHistoryEntry history = 1; +} + +message RollbackConfigurationRequest { + string category = 1; + string key = 2; + int64 rollback_to_timestamp = 3; + string rolled_back_by = 4; + optional string rollback_reason = 5; +} + +message RollbackConfigurationResponse { + bool success = 1; + string message = 2; + ConfigurationSetting restored_setting = 3; + int64 timestamp = 4; +} + +// Import/Export Messages +message ExportConfigurationRequest { + repeated string categories = 1; + optional string environment = 2; + ExportFormat format = 3; +} + +message ExportConfigurationResponse { + string exported_data = 1; + ExportFormat format = 2; + int32 settings_count = 3; + int64 exported_at = 4; +} + +message ImportConfigurationRequest { + string imported_data = 1; + ExportFormat format = 2; + string imported_by = 3; + bool dry_run = 4; + bool overwrite_existing = 5; +} + +message ImportConfigurationResponse { + bool success = 1; + string message = 2; + repeated ImportResult import_results = 3; + int32 imported_count = 4; + int32 skipped_count = 5; + int32 error_count = 6; +} + +// Schema Messages +message GetConfigSchemaRequest { + optional string category = 1; +} + +message GetConfigSchemaResponse { + repeated ConfigurationSchema schemas = 1; +} + +message UpdateConfigSchemaRequest { + string schema_name = 1; + string schema_definition = 2; + string updated_by = 3; +} + +message UpdateConfigSchemaResponse { + bool success = 1; + string message = 2; + int64 timestamp = 3; +} + +// Core Data Types + +// Complete configuration setting with metadata and validation rules +message ConfigurationSetting { + int64 id = 1; // Unique setting identifier + string category = 2; // Configuration category (e.g., "trading.limits") + string key = 3; // Configuration key (e.g., "max_position_size") + string value = 4; // Current configuration value + ConfigDataType data_type = 5; // Data type (string, number, boolean, etc.) + bool hot_reload = 6; // Whether changes trigger hot-reload + string description = 7; // Human-readable description + optional string default_value = 8; // Default value if not set + bool required = 9; // Whether this setting is required + bool sensitive = 10; // Whether value contains sensitive data + optional string validation_rule = 11; // Validation rule expression + optional string environment_override = 12; // Environment-specific override + optional double min_value = 13; // Minimum numeric value + optional double max_value = 14; // Maximum numeric value + optional string enum_values = 15; // Allowed enum values (comma-separated) + repeated string depends_on = 16; // Dependencies on other settings + repeated string tags = 17; // Tags for categorization + int32 display_order = 18; // Display order in UI + int64 created_at = 19; // Creation timestamp + int64 modified_at = 20; // Last modification timestamp +} + +message ConfigurationCategory { + int64 id = 1; + string name = 2; + string description = 3; + optional int64 parent_id = 4; + int32 display_order = 5; + optional string icon = 6; + int64 created_at = 7; + repeated ConfigurationCategory children = 8; +} + +message ConfigurationHistoryEntry { + int64 id = 1; + int64 setting_id = 2; + optional string old_value = 3; + string new_value = 4; + optional string change_reason = 5; + string changed_by = 6; + int64 changed_at = 7; + string change_source = 8; + optional ValidationResult validation_result = 9; + optional int64 rollback_id = 10; +} + +message ConfigurationSchema { + int64 id = 1; + string name = 2; + string schema_definition = 3; + string description = 4; + int64 created_at = 5; +} + +message ValidationResult { + bool is_valid = 1; + repeated ValidationError errors = 2; + repeated ValidationWarning warnings = 3; +} + +message ValidationError { + string field = 1; + string message = 2; + string error_code = 3; +} + +message ValidationWarning { + string field = 1; + string message = 2; + string warning_code = 3; +} + +message ImportResult { + string category = 1; + string key = 2; + ImportStatus status = 3; + optional string error_message = 4; +} + +// Event Messages +message ConfigChangeEvent { + int64 setting_id = 1; + string category = 2; + string key = 3; + string old_value = 4; + string new_value = 5; + string changed_by = 6; + int64 timestamp = 7; + ConfigChangeType change_type = 8; + bool hot_reload = 9; +} + +// Enums + +// Data types for configuration values +enum ConfigDataType { + CONFIG_DATA_TYPE_UNSPECIFIED = 0; // Default/unknown type + CONFIG_DATA_TYPE_STRING = 1; // Text string value + CONFIG_DATA_TYPE_NUMBER = 2; // Numeric value (int or float) + CONFIG_DATA_TYPE_BOOLEAN = 3; // Boolean true/false value + CONFIG_DATA_TYPE_JSON = 4; // JSON object or array + CONFIG_DATA_TYPE_ENCRYPTED = 5; // Encrypted sensitive value +} + +enum ExportFormat { + EXPORT_FORMAT_UNSPECIFIED = 0; + EXPORT_FORMAT_JSON = 1; + EXPORT_FORMAT_YAML = 2; + EXPORT_FORMAT_TOML = 3; + EXPORT_FORMAT_ENV = 4; +} + +enum ImportStatus { + IMPORT_STATUS_UNSPECIFIED = 0; + IMPORT_STATUS_SUCCESS = 1; + IMPORT_STATUS_SKIPPED = 2; + IMPORT_STATUS_ERROR = 3; + IMPORT_STATUS_VALIDATION_FAILED = 4; +} + +enum ConfigChangeType { + CONFIG_CHANGE_TYPE_UNSPECIFIED = 0; + CONFIG_CHANGE_TYPE_CREATED = 1; + CONFIG_CHANGE_TYPE_UPDATED = 2; + CONFIG_CHANGE_TYPE_DELETED = 3; + CONFIG_CHANGE_TYPE_ROLLBACK = 4; +} \ No newline at end of file diff --git a/proto/config_service.proto b/proto/config_service.proto new file mode 100644 index 000000000..de96bb785 --- /dev/null +++ b/proto/config_service.proto @@ -0,0 +1,81 @@ +syntax = "proto3"; + +package foxhunt.config; + +// Configuration Management Service +service ConfigurationService { + // Get a single configuration value + rpc GetConfig(GetConfigRequest) returns (GetConfigResponse); + + // Update a configuration value + rpc UpdateConfig(UpdateConfigRequest) returns (UpdateConfigResponse); + + // List all configurations for a service scope + rpc ListConfigs(ListConfigsRequest) returns (ListConfigsResponse); + + // Trigger configuration reload + rpc ReloadConfig(ReloadConfigRequest) returns (ReloadConfigResponse); +} + +// Get configuration request +message GetConfigRequest { + string service_scope = 1; + string config_key = 2; +} + +// Get configuration response +message GetConfigResponse { + string config_value = 1; // JSON-serialized value + string data_type = 2; + string description = 3; + int64 updated_at = 4; // Unix timestamp + string updated_by = 5; +} + +// Update configuration request +message UpdateConfigRequest { + string service_scope = 1; + string config_key = 2; + string new_value = 3; // JSON-serialized value + string updated_by = 4; +} + +// Update configuration response +message UpdateConfigResponse { + bool success = 1; + string message = 2; +} + +// List configurations request +message ListConfigsRequest { + optional string service_scope = 1; // If not provided, lists all scopes +} + +// Configuration item +message ConfigItem { + string service_scope = 1; + string config_key = 2; + string config_value = 3; // JSON-serialized value + string data_type = 4; + string description = 5; + int64 created_at = 6; + int64 updated_at = 7; + string updated_by = 8; +} + +// List configurations response +message ListConfigsResponse { + repeated ConfigItem configs = 1; +} + +// Reload configuration request +message ReloadConfigRequest { + optional string service_scope = 1; + optional string config_key = 2; +} + +// Reload configuration response +message ReloadConfigResponse { + bool success = 1; + string message = 2; +} diff --git a/proto/data_acquisition.proto b/proto/data_acquisition.proto new file mode 100644 index 000000000..67b484d96 --- /dev/null +++ b/proto/data_acquisition.proto @@ -0,0 +1,177 @@ +syntax = "proto3"; + +package data_acquisition; + +// Data Acquisition Service provides automated Databento data downloading and storage. +// This service handles scheduled downloads, cost tracking, quality validation, and automatic MinIO upload. +service DataAcquisitionService { + // Download Management + // Schedule a new data download from Databento + rpc ScheduleDownload(ScheduleDownloadRequest) returns (ScheduleDownloadResponse); + + // Get status of a download job + rpc GetDownloadStatus(GetDownloadStatusRequest) returns (GetDownloadStatusResponse); + + // Cancel a running or pending download job + rpc CancelDownload(CancelDownloadRequest) returns (CancelDownloadResponse); + + // List all download jobs with optional filters + rpc ListDownloadJobs(ListDownloadJobsRequest) returns (ListDownloadJobsResponse); + + // Service Health and Status + // Check service health and resource availability + rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse); +} + +// --- Core Request/Response Messages --- + +// Request to schedule a new data download +message ScheduleDownloadRequest { + // Databento dataset (e.g., "GLBX.MDP3" for CME futures) + string dataset = 1; + + // List of symbols to download (e.g., ["ES.FUT", "NQ.FUT"]) + repeated string symbols = 2; + + // Start date in YYYY-MM-DD format + string start_date = 3; + + // End date in YYYY-MM-DD format + string end_date = 4; + + // Schema type (e.g., "ohlcv-1m", "mbp-10", "trades") + string schema = 5; + + // Optional description for this download + string description = 6; + + // Optional tags for categorization + map tags = 7; + + // Priority level (1=low, 5=high) + uint32 priority = 8; +} + +message ScheduleDownloadResponse { + // Unique job identifier + string job_id = 1; + + // Initial job status + DownloadStatus status = 2; + + // Estimated cost in USD + double estimated_cost_usd = 3; + + // Human-readable message + string message = 4; +} + +message GetDownloadStatusRequest { + string job_id = 1; +} + +message GetDownloadStatusResponse { + DownloadJobDetails job_details = 1; +} + +message CancelDownloadRequest { + string job_id = 1; + string reason = 2; // Optional cancellation reason +} + +message CancelDownloadResponse { + bool success = 1; + string message = 2; +} + +message ListDownloadJobsRequest { + uint32 page = 1; + uint32 page_size = 2; + DownloadStatus status_filter = 3; + int64 start_time = 4; // Unix timestamp + int64 end_time = 5; // Unix timestamp +} + +message ListDownloadJobsResponse { + repeated DownloadJobSummary jobs = 1; + uint32 total_count = 2; + uint32 page = 3; + uint32 page_size = 4; +} + +message HealthCheckRequest {} + +message HealthCheckResponse { + bool healthy = 1; + string message = 2; + map details = 3; +} + +// --- Status and Details Messages --- + +enum DownloadStatus { + DOWNLOAD_STATUS_UNKNOWN = 0; + PENDING = 1; // Queued, waiting to start + DOWNLOADING = 2; // Actively downloading from Databento + VALIDATING = 3; // Validating data quality + UPLOADING = 4; // Uploading to MinIO + COMPLETED = 5; // Successfully completed + FAILED = 6; // Failed with errors + CANCELLED = 7; // Cancelled by user +} + +message DownloadJobDetails { + string job_id = 1; + DownloadStatus status = 2; + string dataset = 3; + repeated string symbols = 4; + string start_date = 5; + string end_date = 6; + string schema = 7; + string description = 8; + map tags = 9; + uint32 priority = 10; + + // Progress tracking + float progress_percentage = 11; // 0.0 to 100.0 + uint64 bytes_downloaded = 12; + uint64 total_bytes = 13; + + // Cost tracking + double estimated_cost_usd = 14; + double actual_cost_usd = 15; + + // Quality metrics + uint64 records_count = 16; + uint64 invalid_records = 17; + double data_quality_score = 18; // 0.0 to 1.0 + + // Storage paths + string local_path = 19; + string minio_path = 20; + + // Timestamps + int64 created_at = 21; // Unix timestamp + int64 started_at = 22; + int64 completed_at = 23; + + // Error information + string error_message = 24; + uint32 retry_count = 25; + + // Metadata + string created_by = 26; +} + +message DownloadJobSummary { + string job_id = 1; + DownloadStatus status = 2; + string dataset = 3; + repeated string symbols = 4; + string start_date = 5; + string end_date = 6; + float progress_percentage = 7; + double actual_cost_usd = 8; + int64 created_at = 9; + int64 completed_at = 10; +} diff --git a/proto/health.proto b/proto/health.proto new file mode 100644 index 000000000..71add527a --- /dev/null +++ b/proto/health.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package grpc.health.v1; + +// Health check service definition +service Health { + // Check the health status of the service + rpc Check(HealthCheckRequest) returns (HealthCheckResponse); + + // Watch for health status changes + rpc Watch(HealthCheckRequest) returns (stream HealthCheckResponse); +} + +// Health check request +message HealthCheckRequest { + // Service name to check (empty for overall service health) + string service = 1; +} + +// Health check response +message HealthCheckResponse { + // Health status + ServingStatus status = 1; +} + +// Serving status enum +enum ServingStatus { + UNKNOWN = 0; + SERVING = 1; + NOT_SERVING = 2; + SERVICE_UNKNOWN = 3; // Used when the requested service is unknown +} \ No newline at end of file diff --git a/proto/ml.proto b/proto/ml.proto new file mode 100644 index 000000000..830bc07fa --- /dev/null +++ b/proto/ml.proto @@ -0,0 +1,344 @@ +syntax = "proto3"; + +package ml; + +// ML Service provides machine learning model management, predictions, and insights for trading decisions. +// This service integrates multiple ML models including MAMBA-2, TLOB transformers, DQN, and PPO models +// to provide real-time predictions, ensemble voting, and model performance monitoring. +service MLService { + // Model Predictions and Inference + // Get single prediction from a specific model + rpc GetPrediction(GetPredictionRequest) returns (GetPredictionResponse); + + // Stream real-time predictions from multiple models + rpc StreamPredictions(StreamPredictionsRequest) returns (stream PredictionEvent); + + // Get ensemble voting results from multiple models + rpc GetEnsembleVote(GetEnsembleVoteRequest) returns (GetEnsembleVoteResponse); + + // Model Lifecycle Management + // Get current status of ML models (health, performance, etc.) + rpc GetModelStatus(GetModelStatusRequest) returns (GetModelStatusResponse); + + // List all available models and their capabilities + rpc GetAvailableModels(GetAvailableModelsRequest) returns (GetAvailableModelsResponse); + + // Trigger model retraining with new data + rpc RetrainModel(RetrainModelRequest) returns (RetrainModelResponse); + + // Model Performance and Analytics + // Get comprehensive performance metrics for a model + rpc GetModelPerformance(GetModelPerformanceRequest) returns (GetModelPerformanceResponse); + + // Stream real-time model performance metrics + rpc StreamModelMetrics(StreamModelMetricsRequest) returns (stream ModelMetricsEvent); + + // Feature Analysis and Signal Intelligence + // Get feature importance analysis for model interpretation + rpc GetFeatureImportance(GetFeatureImportanceRequest) returns (GetFeatureImportanceResponse); + + // Stream real-time signal strength indicators across models + rpc StreamSignalStrength(StreamSignalStrengthRequest) returns (stream SignalStrengthEvent); +} + +// Prediction Messages + +// Request for model prediction +message GetPredictionRequest { + string model_name = 1; // Model to use (e.g., "mamba2", "tlob-transformer") + string symbol = 2; // Trading symbol to predict + optional int32 horizon_minutes = 3; // Prediction horizon in minutes + map features = 4; // Input features for prediction +} + +// Response containing model prediction +message GetPredictionResponse { + Prediction prediction = 1; // Model prediction with details + double confidence = 2; // Prediction confidence (0.0 to 1.0) + int64 timestamp = 3; // Prediction timestamp (nanoseconds) +} + +message StreamPredictionsRequest { + repeated string model_names = 1; + repeated string symbols = 2; + optional int32 update_frequency_seconds = 3; +} + +message GetEnsembleVoteRequest { + string symbol = 1; + optional int32 horizon_minutes = 2; + repeated string model_names = 3; +} + +message GetEnsembleVoteResponse { + EnsembleVote ensemble_vote = 1; + repeated ModelVote individual_votes = 2; + double overall_confidence = 3; + int64 timestamp = 4; +} + +// Model Management Messages +message GetModelStatusRequest { + optional string model_name = 1; +} + +message GetModelStatusResponse { + repeated ModelStatus model_statuses = 1; +} + +message GetAvailableModelsRequest {} + +message GetAvailableModelsResponse { + repeated ModelInfo available_models = 1; +} + +message RetrainModelRequest { + string model_name = 1; + optional int64 start_time = 2; + optional int64 end_time = 3; + map parameters = 4; +} + +message RetrainModelResponse { + bool success = 1; + string message = 2; + optional string job_id = 3; + int64 started_at = 4; +} + +// Performance Messages +message GetModelPerformanceRequest { + string model_name = 1; + optional int64 start_time = 2; + optional int64 end_time = 3; +} + +message GetModelPerformanceResponse { + ModelPerformance performance = 1; +} + +message StreamModelMetricsRequest { + repeated string model_names = 1; + optional int32 update_frequency_seconds = 2; +} + +// Feature Analysis Messages +message GetFeatureImportanceRequest { + string model_name = 1; + optional string symbol = 2; +} + +message GetFeatureImportanceResponse { + repeated FeatureImportance feature_importances = 1; + string model_name = 2; + int64 calculated_at = 3; +} + +message StreamSignalStrengthRequest { + repeated string symbols = 1; + optional int32 update_frequency_seconds = 2; +} + +// Core ML Data Types + +// Complete prediction information from a model +message Prediction { + string model_name = 1; // Model that generated prediction + string symbol = 2; // Trading symbol + PredictionType prediction_type = 3; // Type of prediction (buy/sell/hold/price direction) + double value = 4; // Predicted value (price change, probability, etc.) + double confidence = 5; // Model confidence in prediction (0.0 to 1.0) + int32 horizon_minutes = 6; // Prediction time horizon + repeated Feature features = 7; // Input features used for prediction + int64 timestamp = 8; // Prediction generation timestamp (nanoseconds) +} + +message EnsembleVote { + string symbol = 1; + PredictionType consensus_prediction = 2; + double consensus_confidence = 3; + int32 votes_buy = 4; + int32 votes_sell = 5; + int32 votes_hold = 6; + int32 total_models = 7; + SignalStrength signal_strength = 8; +} + +message ModelVote { + string model_name = 1; + PredictionType prediction = 2; + double confidence = 3; + double weight = 4; +} + +message ModelStatus { + string model_name = 1; + ModelState state = 2; + optional string error_message = 3; + int64 last_updated = 4; + int64 last_prediction = 5; + ModelHealth health = 6; + map metadata = 7; +} + +message ModelInfo { + string model_name = 1; + string model_type = 2; + string description = 3; + repeated string supported_symbols = 4; + repeated int32 supported_horizons = 5; + ModelCapabilities capabilities = 6; + map parameters = 7; +} + +message ModelPerformance { + string model_name = 1; + double accuracy = 2; + double precision = 3; + double recall = 4; + double f1_score = 5; + double sharpe_ratio = 6; + double win_rate = 7; + double avg_return = 8; + double max_drawdown = 9; + int32 total_predictions = 10; + int64 performance_period_start = 11; + int64 performance_period_end = 12; + repeated DailyPerformance daily_performance = 13; +} + +message DailyPerformance { + string date = 1; + double accuracy = 2; + double return_pct = 3; + int32 predictions_count = 4; + double sharpe_ratio = 5; +} + +message FeatureImportance { + string feature_name = 1; + double importance_score = 2; + FeatureType feature_type = 3; + double contribution_pct = 4; +} + +message Feature { + string name = 1; + double value = 2; + FeatureType feature_type = 3; + double normalized_value = 4; +} + +message ModelCapabilities { + bool supports_streaming = 1; + bool supports_retraining = 2; + bool supports_feature_importance = 3; + bool supports_confidence_intervals = 4; + repeated string supported_asset_classes = 5; +} + +// Event Messages +message PredictionEvent { + string model_name = 1; + string symbol = 2; + Prediction prediction = 3; + PredictionEventType event_type = 4; + int64 timestamp = 5; +} + +message ModelMetricsEvent { + string model_name = 1; + ModelMetrics metrics = 2; + int64 timestamp = 3; +} + +message SignalStrengthEvent { + string symbol = 1; + SignalStrength signal_strength = 2; + repeated ModelSignal model_signals = 3; + int64 timestamp = 4; +} + +message ModelMetrics { + string model_name = 1; + double cpu_usage = 2; + double memory_usage_mb = 3; + double gpu_usage = 4; + double predictions_per_second = 5; + double avg_inference_time_ms = 6; + int32 queue_size = 7; + ModelHealth health = 8; +} + +message ModelSignal { + string model_name = 1; + double signal_strength = 2; + PredictionType direction = 3; + double confidence = 4; +} + +// Enums + +// Types of predictions that ML models can generate +enum PredictionType { + PREDICTION_TYPE_UNSPECIFIED = 0; // Default/unknown prediction type + PREDICTION_TYPE_BUY = 1; // Recommendation to buy (go long) + PREDICTION_TYPE_SELL = 2; // Recommendation to sell (go short) + PREDICTION_TYPE_HOLD = 3; // Recommendation to hold position + PREDICTION_TYPE_PRICE_UP = 4; // Price expected to increase + PREDICTION_TYPE_PRICE_DOWN = 5; // Price expected to decrease + PREDICTION_TYPE_VOLATILITY_HIGH = 6; // High volatility expected + PREDICTION_TYPE_VOLATILITY_LOW = 7; // Low volatility expected +} + +// Current operational state of ML models +enum ModelState { + MODEL_STATE_UNSPECIFIED = 0; // Default/unknown state + MODEL_STATE_LOADING = 1; // Model is loading from storage + MODEL_STATE_READY = 2; // Model loaded and ready for predictions + MODEL_STATE_PREDICTING = 3; // Model actively making predictions + MODEL_STATE_TRAINING = 4; // Model is being retrained + MODEL_STATE_ERROR = 5; // Model encountered an error + MODEL_STATE_OFFLINE = 6; // Model is offline/disabled +} + +// Health status of ML models +enum ModelHealth { + MODEL_HEALTH_UNSPECIFIED = 0; // Default/unknown health + MODEL_HEALTH_HEALTHY = 1; // Model operating normally + MODEL_HEALTH_DEGRADED = 2; // Model performance degraded + MODEL_HEALTH_UNHEALTHY = 3; // Model not performing well + MODEL_HEALTH_CRITICAL = 4; // Model in critical state +} + +// Types of features used in ML models +enum FeatureType { + FEATURE_TYPE_UNSPECIFIED = 0; // Default/unknown feature type + FEATURE_TYPE_PRICE = 1; // Price-based features (OHLC, etc.) + FEATURE_TYPE_VOLUME = 2; // Volume-based features + FEATURE_TYPE_TECHNICAL = 3; // Technical indicators (RSI, MACD, etc.) + FEATURE_TYPE_FUNDAMENTAL = 4; // Fundamental analysis features + FEATURE_TYPE_SENTIMENT = 5; // Market sentiment features + FEATURE_TYPE_MACRO = 6; // Macroeconomic features + FEATURE_TYPE_TIME = 7; // Time-based features + FEATURE_TYPE_ORDERBOOK = 8; // Order book depth and microstructure features + FEATURE_TYPE_MICROSTRUCTURE = 9; // Market microstructure and flow features +} + +// Signal strength levels for predictions +enum SignalStrength { + SIGNAL_STRENGTH_UNSPECIFIED = 0; // Default/unknown strength + SIGNAL_STRENGTH_VERY_WEAK = 1; // Very weak signal confidence + SIGNAL_STRENGTH_WEAK = 2; // Weak signal confidence + SIGNAL_STRENGTH_MODERATE = 3; // Moderate signal confidence + SIGNAL_STRENGTH_STRONG = 4; // Strong signal confidence + SIGNAL_STRENGTH_VERY_STRONG = 5; // Very strong signal confidence +} + +enum PredictionEventType { + PREDICTION_EVENT_TYPE_UNSPECIFIED = 0; + PREDICTION_EVENT_TYPE_NEW = 1; + PREDICTION_EVENT_TYPE_UPDATED = 2; + PREDICTION_EVENT_TYPE_EXPIRED = 3; + PREDICTION_EVENT_TYPE_CONFIRMED = 4; +} diff --git a/proto/ml_training.proto b/proto/ml_training.proto new file mode 100644 index 000000000..fd83c4cde --- /dev/null +++ b/proto/ml_training.proto @@ -0,0 +1,612 @@ +syntax = "proto3"; + +package ml_training; + +// ML Training Service provides comprehensive machine learning model training capabilities for HFT systems. +// This service manages training jobs for MAMBA-2, TLOB transformers, DQN, PPO, Liquid Networks, and TFT models +// with real-time progress monitoring, resource management, and performance tracking. +service MLTrainingService { + // Training Job Management + // Initiates a new training job and returns job ID immediately + rpc StartTraining(StartTrainingRequest) returns (StartTrainingResponse); + + // Subscribe to real-time training progress and status updates + rpc SubscribeToTrainingStatus(SubscribeToTrainingStatusRequest) returns (stream TrainingStatusUpdate); + + // Stop a running training job (idempotent operation) + rpc StopTraining(StopTrainingRequest) returns (StopTrainingResponse); + + // Model and Job Discovery + // List available ML models with their training parameters + rpc ListAvailableModels(ListAvailableModelsRequest) returns (ListAvailableModelsResponse); + + // Get paginated list of training job history + rpc ListTrainingJobs(ListTrainingJobsRequest) returns (ListTrainingJobsResponse); + + // Get comprehensive details for a specific training job + rpc GetTrainingJobDetails(GetTrainingJobDetailsRequest) returns (GetTrainingJobDetailsResponse); + + // Service Health and Status + // Check service health and resource availability + rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse); + + // Hyperparameter Tuning Management + // Start a new hyperparameter tuning job using Optuna + rpc StartTuningJob(StartTuningJobRequest) returns (StartTuningJobResponse); + + // Get current status and best parameters from a tuning job + rpc GetTuningJobStatus(GetTuningJobStatusRequest) returns (GetTuningJobStatusResponse); + + // Stop a running hyperparameter tuning job + rpc StopTuningJob(StopTuningJobRequest) returns (StopTuningJobResponse); + + // INTERNAL: Train a single model instance with specific hyperparameters (called by Optuna subprocess) + rpc TrainModel(TrainModelRequest) returns (TrainModelResponse); + + // Stream real-time tuning progress updates (trial completion events) + rpc StreamTuningProgress(StreamProgressRequest) returns (stream ProgressUpdate); + + // Batch Tuning Management + // Start batch tuning job for multiple models with automatic dependency resolution + rpc BatchStartTuningJobs(BatchStartTuningJobsRequest) returns (BatchStartTuningJobsResponse); + + // Get batch tuning job status with per-model results + rpc GetBatchTuningStatus(GetBatchTuningStatusRequest) returns (GetBatchTuningStatusResponse); + + // Stop a running batch tuning job + rpc StopBatchTuningJob(StopBatchTuningJobRequest) returns (StopBatchTuningJobResponse); + + // Job Completion Callback (called by training-uploader sidecar) + rpc ReportJobCompletion(JobCompletionReport) returns (JobCompletionAck); + + // Model Promotion Management + rpc ListPendingPromotions(ListPendingPromotionsRequest) returns (ListPendingPromotionsResponse); + rpc ApprovePromotion(ApprovePromotionRequest) returns (ApprovePromotionResponse); + rpc RejectPromotion(RejectPromotionRequest) returns (RejectPromotionResponse); + + // Model approval / rejection (dashboard-facing, delegates to promotion pipeline) + rpc ApproveModel(ApproveModelRequest) returns (ApproveModelResponse); + rpc RejectModel(RejectModelRequest) returns (RejectModelResponse); +} + +// --- Core Request/Response Messages --- + +// Training mode selection +enum TrainingMode { + TRAINING_MODE_FULL = 0; // Full training from scratch (default, backward-compatible) + TRAINING_MODE_FINE_TUNE = 1; // Fine-tune from existing checkpoint +} + +// Request to start a new model training job +message StartTrainingRequest { + string model_type = 1; // Model type ("TLOB", "MAMBA_2", "DQN", "PPO", "LIQUID", "TFT") + DataSource data_source = 2; // Training data source configuration + Hyperparameters hyperparameters = 3; // Model-specific training parameters + bool use_gpu = 4; // Whether to use GPU acceleration + string description = 5; // Optional job description + map tags = 6; // Optional categorization tags + TrainingMode mode = 7; // FULL (default) or FINE_TUNE + string resume_checkpoint_path = 8; // Path to checkpoint for fine-tune + uint32 max_epochs = 9; // Override epoch count (0 = use default) +} + +message StartTrainingResponse { + string job_id = 1; + TrainingStatus status = 2; + string message = 3; +} + +message SubscribeToTrainingStatusRequest { + string job_id = 1; +} + +// Real-time training progress update streamed from server +message TrainingStatusUpdate { + string job_id = 1; // Training job identifier + TrainingStatus status = 2; // Current job status + float progress_percentage = 3; // Training progress (0.0 to 100.0) + uint32 current_epoch = 4; // Current training epoch + uint32 total_epochs = 5; // Total epochs planned + map metrics = 6; // Training metrics (loss, accuracy, sharpe_ratio, etc.) + string message = 7; // Human-readable status message + int64 timestamp = 8; // Update timestamp (Unix seconds) + FinancialMetrics financial_metrics = 9; // Financial performance metrics + ResourceUsage resource_usage = 10; // Current resource utilization +} + +message StopTrainingRequest { + string job_id = 1; + string reason = 2; // Optional reason for stopping +} + +message StopTrainingResponse { + bool success = 1; + string message = 2; +} + +message ListAvailableModelsRequest {} + +message ListAvailableModelsResponse { + repeated ModelDefinition models = 1; +} + +message ListTrainingJobsRequest { + uint32 page = 1; + uint32 page_size = 2; + TrainingStatus status_filter = 3; + string model_type_filter = 4; + int64 start_time = 5; // Unix timestamp in seconds + int64 end_time = 6; // Unix timestamp in seconds +} + +message ListTrainingJobsResponse { + repeated TrainingJobSummary jobs = 1; + uint32 total_count = 2; + uint32 page = 3; + uint32 page_size = 4; +} + +message GetTrainingJobDetailsRequest { + string job_id = 1; +} + +message GetTrainingJobDetailsResponse { + TrainingJobDetails job_details = 1; +} + +message HealthCheckRequest {} + +message HealthCheckResponse { + bool healthy = 1; + string message = 2; + map details = 3; +} + +// Request to start hyperparameter tuning job +message StartTuningJobRequest { + string model_type = 1; // Model type to tune ("TLOB", "MAMBA_2", "DQN", "PPO", "LIQUID", "TFT") + uint32 num_trials = 2; // Number of tuning trials to run + string config_path = 3; // Path to tuning configuration file (search space, objectives) + DataSource data_source = 4; // Training data source for all trials + bool use_gpu = 5; // Whether to use GPU acceleration + string description = 6; // Optional job description + map tags = 7; // Optional categorization tags +} + +message StartTuningJobResponse { + string job_id = 1; // Unique tuning job identifier + TuningJobStatus status = 2; // Initial job status + string message = 3; // Human-readable status message +} + +// Request to query tuning job status +message GetTuningJobStatusRequest { + string job_id = 1; // Tuning job identifier +} + +message GetTuningJobStatusResponse { + string job_id = 1; // Tuning job identifier + TuningJobStatus status = 2; // Current job status + uint32 current_trial = 3; // Current trial number (0-indexed) + uint32 total_trials = 4; // Total number of trials + map best_params = 5; // Best hyperparameters found so far + map best_metrics = 6; // Metrics for best parameters (sharpe_ratio, training_loss, etc.) + repeated TrialResult trial_history = 7; // Complete trial history + string message = 8; // Human-readable status message + int64 started_at = 9; // Job start time (Unix timestamp in seconds) + int64 updated_at = 10; // Last update time (Unix timestamp in seconds) +} + +// Request to stop a tuning job +message StopTuningJobRequest { + string job_id = 1; // Tuning job identifier + string reason = 2; // Optional reason for stopping +} + +message StopTuningJobResponse { + bool success = 1; // Whether stop was successful + string message = 2; // Human-readable status message + TuningJobStatus final_status = 3; // Final job status after stopping +} + +// INTERNAL: Request to train a model with specific hyperparameters (called by Optuna) +message TrainModelRequest { + string model_type = 1; // Model type ("TLOB", "MAMBA_2", "DQN", "PPO", "LIQUID", "TFT") + map hyperparameters = 2; // Hyperparameters to use for this trial + DataSource data_source = 3; // Training data source + bool use_gpu = 4; // Whether to use GPU acceleration + string trial_id = 5; // Optuna trial identifier for tracking +} + +message TrainModelResponse { + bool success = 1; // Whether training succeeded + float sharpe_ratio = 2; // Primary optimization objective (Sharpe ratio) + float training_loss = 3; // Final training loss + map validation_metrics = 4; // Additional validation metrics + string error_message = 5; // Error message if training failed + int64 training_duration_seconds = 6; // Total training time +} + +// Individual trial result for tuning job history +message TrialResult { + uint32 trial_number = 1; // Trial index + map params = 2; // Hyperparameters tested + float objective_value = 3; // Objective metric (e.g., Sharpe ratio) + map metrics = 4; // Additional metrics + TrialState state = 5; // Trial outcome state + int64 started_at = 6; // Trial start time (Unix timestamp in seconds) + int64 completed_at = 7; // Trial completion time (Unix timestamp in seconds) +} + +// Request to stream tuning progress updates +message StreamProgressRequest { + string job_id = 1; // Tuning job identifier to subscribe to +} + +// Real-time progress update streamed after each trial completes +message ProgressUpdate { + string job_id = 1; // Tuning job identifier + uint32 current_trial = 2; // Current trial number (0-indexed) + uint32 total_trials = 3; // Total number of trials + map trial_params = 4; // Current trial hyperparameters (as strings for display) + float trial_sharpe = 5; // Current trial's Sharpe ratio (objective value) + float best_sharpe_so_far = 6; // Best Sharpe ratio achieved so far + uint32 estimated_time_remaining = 7; // Estimated seconds until completion + TuningJobStatus status = 8; // Current job status + string message = 9; // Human-readable status message + int64 timestamp = 10; // Update timestamp (Unix seconds) + UpdateType update_type = 11; // Type of update (trial completion, heartbeat, job complete) +} + +// Type of progress update +enum UpdateType { + UPDATE_UNKNOWN = 0; // Unknown/unspecified + UPDATE_TRIAL_COMPLETE = 1; // Trial completed + UPDATE_HEARTBEAT = 2; // Keepalive heartbeat (no trial change) + UPDATE_JOB_COMPLETE = 3; // Job completed/stopped/failed +} + +// --- Enums --- + +// Current status of a training job +enum TrainingStatus { + UNKNOWN = 0; // Default/unknown status + PENDING = 1; // Job queued, waiting to start + RUNNING = 2; // Job currently executing + COMPLETED = 3; // Job finished successfully + FAILED = 4; // Job failed with error + STOPPED = 5; // Job manually stopped + PAUSED = 6; // Job temporarily paused +} + +// Status of a hyperparameter tuning job +enum TuningJobStatus { + TUNING_UNKNOWN = 0; // Default/unknown status + TUNING_PENDING = 1; // Job queued, waiting to start + TUNING_RUNNING = 2; // Job currently executing trials + TUNING_COMPLETED = 3; // Job finished all trials successfully + TUNING_FAILED = 4; // Job failed with error + TUNING_STOPPED = 5; // Job manually stopped before completion +} + +// Outcome state of an individual trial +enum TrialState { + TRIAL_UNKNOWN = 0; // Default/unknown state + TRIAL_RUNNING = 1; // Trial currently executing + TRIAL_COMPLETE = 2; // Trial completed successfully + TRIAL_PRUNED = 3; // Trial pruned by Optuna (early stopping) + TRIAL_FAILED = 4; // Trial failed with error +} + +// --- Data Structures --- + +message DataSource { + oneof source { + string historical_db_query = 1; + string real_time_stream_topic = 2; + string file_path = 3; + } + int64 start_time = 4; // Unix timestamp in seconds + int64 end_time = 5; // Unix timestamp in seconds +} + +// Provides type-safe hyperparameter configuration. +message Hyperparameters { + oneof model_params { + TlobParams tlob_params = 1; + MambaParams mamba_params = 2; + DqnParams dqn_params = 3; + PpoParams ppo_params = 4; + LiquidParams liquid_params = 5; + TftParams tft_params = 6; + } +} + +// TLOB (Time-Limit Order Book) Transformer parameters +message TlobParams { + uint32 epochs = 1; + float learning_rate = 2; + uint32 batch_size = 3; + uint32 sequence_length = 4; + uint32 hidden_dim = 5; + uint32 num_heads = 6; + uint32 num_layers = 7; + float dropout_rate = 8; + bool use_positional_encoding = 9; +} + +// MAMBA-2 State Space Model parameters +message MambaParams { + uint32 epochs = 1; + float learning_rate = 2; + uint32 batch_size = 3; + uint32 state_dim = 4; + uint32 hidden_dim = 5; + uint32 num_layers = 6; + float dt_min = 7; + float dt_max = 8; + bool use_cuda_kernels = 9; +} + +// DQN (Deep Q-Network) parameters +message DqnParams { + uint32 epochs = 1; + float learning_rate = 2; + uint32 batch_size = 3; + uint32 replay_buffer_size = 4; + float epsilon_start = 5; + float epsilon_end = 6; + uint32 epsilon_decay_steps = 7; + float gamma = 8; + uint32 target_update_frequency = 9; + bool use_double_dqn = 10; + bool use_dueling = 11; + bool use_prioritized_replay = 12; +} + +// PPO (Proximal Policy Optimization) parameters +message PpoParams { + uint32 epochs = 1; + float learning_rate = 2; + uint32 batch_size = 3; + float clip_ratio = 4; + float value_loss_coef = 5; + float entropy_coef = 6; + uint32 rollout_steps = 7; + uint32 minibatch_size = 8; + float gae_lambda = 9; +} + +// Liquid Network parameters +message LiquidParams { + uint32 epochs = 1; + float learning_rate = 2; + uint32 batch_size = 3; + uint32 num_neurons = 4; + float tau = 5; + float sigma = 6; + bool use_adaptive_tau = 7; +} + +// Temporal Fusion Transformer parameters +message TftParams { + uint32 epochs = 1; + float learning_rate = 2; + uint32 batch_size = 3; + uint32 hidden_dim = 4; + uint32 num_heads = 5; + uint32 num_layers = 6; + uint32 lookback_window = 7; + uint32 forecast_horizon = 8; + float dropout_rate = 9; +} + +message ModelDefinition { + string model_type = 1; + string description = 2; + Hyperparameters default_hyperparameters = 3; + repeated string required_features = 4; + uint32 estimated_training_time_minutes = 5; + bool requires_gpu = 6; +} + +message TrainingJobSummary { + string job_id = 1; + string model_type = 2; + TrainingStatus status = 3; + int64 created_at = 4; // Unix timestamp in seconds + int64 started_at = 5; // Unix timestamp in seconds + int64 completed_at = 6; // Unix timestamp in seconds + string description = 7; + float final_loss = 8; + float best_validation_score = 9; + map tags = 10; +} + +message TrainingJobDetails { + string job_id = 1; + string model_type = 2; + TrainingStatus status = 3; + int64 created_at = 4; // Unix timestamp in seconds + int64 started_at = 5; // Unix timestamp in seconds + int64 completed_at = 6; // Unix timestamp in seconds + string description = 7; + Hyperparameters hyperparameters = 8; + DataSource data_source = 9; + repeated TrainingStatusUpdate status_history = 10; + FinancialMetrics final_financial_metrics = 11; + string model_artifact_path = 12; + map tags = 13; + string error_message = 14; +} + +message FinancialMetrics { + float simulated_return = 1; + float sharpe_ratio = 2; + float max_drawdown = 3; + float hit_rate = 4; + float avg_prediction_error_bps = 5; + float risk_adjusted_return = 6; + float var_5pct = 7; + float expected_shortfall = 8; +} + +message ResourceUsage { + float cpu_usage_percent = 1; + float memory_usage_gb = 2; + float gpu_usage_percent = 3; + float gpu_memory_usage_gb = 4; + uint32 active_workers = 5; +} + +// --- Batch Tuning Messages --- + +// Request to start batch tuning for multiple models +message BatchStartTuningJobsRequest { + repeated string model_types = 1; // List of models to tune (DQN, PPO, MAMBA_2, TFT, etc.) + uint32 trials_per_model = 2; // Number of trials for each model + string config_path = 3; // Path to tuning configuration file + DataSource data_source = 4; // Training data source for all models + bool use_gpu = 5; // Whether to use GPU acceleration + bool auto_export_yaml = 6; // Automatically export best params to YAML (default: true) + string yaml_export_path = 7; // Custom YAML export path (default: ml/config/best_hyperparameters.yaml) + string description = 8; // Optional batch job description + map tags = 9; // Optional categorization tags +} + +message BatchStartTuningJobsResponse { + string batch_id = 1; // Unique batch job identifier + repeated string execution_order = 2; // Model execution order (after dependency resolution) + string message = 3; // Human-readable status message + BatchTuningStatus status = 4; // Initial batch status +} + +// Request to get batch tuning job status +message GetBatchTuningStatusRequest { + string batch_id = 1; // Batch job identifier +} + +message GetBatchTuningStatusResponse { + string batch_id = 1; // Batch job identifier + BatchTuningStatus status = 2; // Current batch status + uint32 current_model_index = 3; // Index of currently executing model (0-based) + uint32 total_models = 4; // Total number of models in batch + repeated ModelTuningResult results = 5; // Results for completed models + string current_model = 6; // Currently tuning model type + int64 started_at = 7; // Batch start time (Unix timestamp) + int64 updated_at = 8; // Last update time (Unix timestamp) + int64 estimated_completion_time = 9; // Estimated completion time (Unix timestamp) + string yaml_export_path = 10; // Path where YAML will be exported +} + +// Individual model tuning result within batch +message ModelTuningResult { + string model_type = 1; // Model type (DQN, PPO, etc.) + string job_id = 2; // Individual tuning job ID + TuningJobStatus status = 3; // Model tuning status + map best_params = 4; // Best hyperparameters found + map best_metrics = 5; // Best metrics achieved + uint32 trials_completed = 6; // Number of trials completed + int64 started_at = 7; // Model tuning start time + int64 completed_at = 8; // Model tuning completion time + string error_message = 9; // Error message if failed +} + +// Request to stop batch tuning job +message StopBatchTuningJobRequest { + string batch_id = 1; // Batch job identifier + string reason = 2; // Optional reason for stopping +} + +message StopBatchTuningJobResponse { + bool success = 1; // Whether stop was successful + string message = 2; // Human-readable status message + BatchTuningStatus final_status = 3; // Final batch status + repeated ModelTuningResult completed_results = 4; // Results for completed models +} + +// Batch tuning job status +enum BatchTuningStatus { + BATCH_UNKNOWN = 0; // Default/unknown status + BATCH_PENDING = 1; // Batch queued, waiting to start + BATCH_RUNNING = 2; // Batch currently executing models + BATCH_COMPLETED = 3; // All models completed successfully + BATCH_PARTIALLY_COMPLETED = 4; // Some models succeeded, some failed + BATCH_FAILED = 5; // Batch failed (all models failed or critical error) + BATCH_STOPPED = 6; // Batch manually stopped +} + +// --- Job Completion Callback (from training-uploader sidecar) --- + +message JobCompletionReport { + string job_id = 1; + string s3_path = 2; + bool success = 3; + string error_message = 4; + map metrics = 5; +} + +message JobCompletionAck { + bool accepted = 1; + string promotion_status = 2; +} + +// --- Model Promotion --- + +message ListPendingPromotionsRequest {} + +message ListPendingPromotionsResponse { + repeated PendingPromotion promotions = 1; +} + +message PendingPromotion { + string model_id = 1; + string model_type = 2; + string symbol = 3; + string s3_path = 4; + map new_metrics = 5; + map current_metrics = 6; + int64 trained_at = 7; + string job_id = 8; +} + +message ApprovePromotionRequest { + string model_id = 1; +} + +message ApprovePromotionResponse { + bool success = 1; + string message = 2; +} + +message RejectPromotionRequest { + string model_id = 1; + string reason = 2; +} + +message RejectPromotionResponse { + bool success = 1; + string message = 2; +} + +// --- Model Approval / Rejection (dashboard-facing) --- + +message ApproveModelRequest { + string model_id = 1; + string promoted_to = 2; // e.g. "production", "staging", "canary" +} + +message ApproveModelResponse { + bool success = 1; + string message = 2; +} + +message RejectModelRequest { + string model_id = 1; + string reason = 2; +} + +message RejectModelResponse { + bool success = 1; + string message = 2; +} \ No newline at end of file diff --git a/proto/monitoring.proto b/proto/monitoring.proto new file mode 100644 index 000000000..fe83f8010 --- /dev/null +++ b/proto/monitoring.proto @@ -0,0 +1,538 @@ +syntax = "proto3"; + +package monitoring; + +// Monitoring Service provides comprehensive system health monitoring, performance metrics collection, +// alerting capabilities, and training metrics for the HFT trading system. This service tracks latency, +// throughput, resource utilization, service health, and ML training progress across all components +// with real-time alerting and streaming. +service MonitoringService { + // Health and Status Monitoring + // Get overall system status and individual service health + rpc GetSystemStatus(GetSystemStatusRequest) returns (GetSystemStatusResponse); + + // Stream real-time system status changes + rpc StreamSystemStatus(StreamSystemStatusRequest) returns (stream SystemStatusEvent); + + // Perform detailed health checks on services + rpc GetHealthCheck(GetHealthCheckRequest) returns (GetHealthCheckResponse); + + // Performance Metrics Collection + // Get system and application metrics + rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse); + + // Stream real-time performance metrics + rpc StreamMetrics(StreamMetricsRequest) returns (stream MetricsEvent); + + // Get detailed latency performance metrics + rpc GetLatencyMetrics(GetLatencyMetricsRequest) returns (GetLatencyMetricsResponse); + + // Get throughput and capacity metrics + rpc GetThroughputMetrics(GetThroughputMetricsRequest) returns (GetThroughputMetricsResponse); + + // Alerting and Notification System + // Stream real-time system alerts and notifications + rpc StreamAlerts(StreamAlertsRequest) returns (stream AlertEvent); + + // Acknowledge an active alert + rpc AcknowledgeAlert(AcknowledgeAlertRequest) returns (AcknowledgeAlertResponse); + + // Get all currently active alerts + rpc GetActiveAlerts(GetActiveAlertsRequest) returns (GetActiveAlertsResponse); + + // Training Metrics (from monitoring_service) + // Single snapshot of all active training sessions + rpc GetLiveTrainingMetrics(GetLiveTrainingMetricsRequest) + returns (GetLiveTrainingMetricsResponse); + + // Server-streaming: pushes updates every N seconds + rpc StreamTrainingMetrics(StreamTrainingMetricsRequest) + returns (stream GetLiveTrainingMetricsResponse); + + // Epoch history for a specific session (ring buffer, max 50 epochs) + rpc GetEpochHistory(GetEpochHistoryRequest) + returns (GetEpochHistoryResponse); +} + +// ============================================================================ +// Health and Status Messages +// ============================================================================ + +message GetSystemStatusRequest { + repeated string service_names = 1; +} + +message GetSystemStatusResponse { + SystemStatus overall_status = 1; + repeated ServiceStatus service_statuses = 2; + int64 timestamp = 3; +} + +message StreamSystemStatusRequest { + repeated string service_names = 1; + optional int32 update_frequency_seconds = 2; +} + +message GetHealthCheckRequest { + optional string service_name = 1; +} + +message GetHealthCheckResponse { + HealthStatus health_status = 1; + repeated HealthCheck health_checks = 2; + int64 timestamp = 3; +} + +// ============================================================================ +// Performance Metrics Messages +// ============================================================================ + +message GetMetricsRequest { + repeated string metric_names = 1; + optional int64 start_time = 2; + optional int64 end_time = 3; + optional MetricAggregation aggregation = 4; +} + +message GetMetricsResponse { + repeated Metric metrics = 1; + int64 timestamp = 2; +} + +message StreamMetricsRequest { + repeated string metric_names = 1; + optional int32 update_frequency_seconds = 2; +} + +message GetLatencyMetricsRequest { + optional string service_name = 1; + optional string operation_name = 2; + optional int64 start_time = 3; + optional int64 end_time = 4; +} + +message GetLatencyMetricsResponse { + repeated LatencyMetric latency_metrics = 1; +} + +message GetThroughputMetricsRequest { + optional string service_name = 1; + optional string operation_name = 2; + optional int64 start_time = 3; + optional int64 end_time = 4; +} + +message GetThroughputMetricsResponse { + repeated ThroughputMetric throughput_metrics = 1; +} + +// ============================================================================ +// Alert Messages +// ============================================================================ + +message StreamAlertsRequest { + optional AlertSeverity min_severity = 1; + repeated string service_names = 2; + repeated AlertType alert_types = 3; +} + +message AcknowledgeAlertRequest { + string alert_id = 1; + string acknowledged_by = 2; + optional string note = 3; +} + +message AcknowledgeAlertResponse { + bool success = 1; + string message = 2; + int64 timestamp = 3; +} + +message GetActiveAlertsRequest { + optional AlertSeverity min_severity = 1; + repeated string service_names = 2; +} + +message GetActiveAlertsResponse { + repeated Alert active_alerts = 1; + int32 total_count = 2; +} + +// ============================================================================ +// Core Data Types (System Health) +// ============================================================================ + +message ServiceStatus { + string service_name = 1; + ServiceHealth health = 2; + ServiceState state = 3; + optional string version = 4; + optional string error_message = 5; + int64 uptime_seconds = 6; + int64 last_health_check = 7; + map metadata = 8; + repeated Dependency dependencies = 9; +} + +message SystemStatus { + SystemHealth overall_health = 1; + int32 healthy_services = 2; + int32 total_services = 3; + repeated string critical_issues = 4; + int64 system_uptime_seconds = 5; + SystemMetrics system_metrics = 6; +} + +message HealthCheck { + string check_name = 1; + HealthStatus status = 2; + optional string message = 3; + optional double response_time_ms = 4; + int64 last_checked = 5; + map details = 6; +} + +message Dependency { + string name = 1; + DependencyType dependency_type = 2; + HealthStatus status = 3; + optional string endpoint = 4; + optional double response_time_ms = 5; + int64 last_checked = 6; +} + +message SystemMetrics { + double cpu_usage_percent = 1; + double memory_usage_percent = 2; + double disk_usage_percent = 3; + double network_io_mbps = 4; + int32 active_connections = 5; + int32 total_requests = 6; + double avg_response_time_ms = 7; + double error_rate_percent = 8; +} + +message Metric { + string name = 1; + MetricType metric_type = 2; + double value = 3; + string unit = 4; + map labels = 5; + int64 timestamp = 6; + optional MetricStatistics statistics = 7; +} + +message MetricStatistics { + double min = 1; + double max = 2; + double avg = 3; + double percentile_95 = 4; + double percentile_99 = 5; + double std_dev = 6; + int32 sample_count = 7; +} + +message LatencyMetric { + string service_name = 1; + string operation_name = 2; + double avg_latency_ms = 3; + double p50_latency_ms = 4; + double p95_latency_ms = 5; + double p99_latency_ms = 6; + double max_latency_ms = 7; + int32 request_count = 8; + int64 time_window_start = 9; + int64 time_window_end = 10; +} + +message ThroughputMetric { + string service_name = 1; + string operation_name = 2; + double requests_per_second = 3; + double bytes_per_second = 4; + int32 total_requests = 5; + int64 total_bytes = 6; + int64 time_window_start = 7; + int64 time_window_end = 8; +} + +message Alert { + string alert_id = 1; + AlertType alert_type = 2; + AlertSeverity severity = 3; + string title = 4; + string description = 5; + string service_name = 6; + map labels = 7; + int64 triggered_at = 8; + optional int64 acknowledged_at = 9; + optional string acknowledged_by = 10; + optional int64 resolved_at = 11; + AlertStatus status = 12; + optional string resolution_note = 13; +} + +// ============================================================================ +// Event Messages (System Health) +// ============================================================================ + +message SystemStatusEvent { + SystemStatus system_status = 1; + SystemStatusChangeType change_type = 2; + int64 timestamp = 3; +} + +message MetricsEvent { + repeated Metric metrics = 1; + int64 timestamp = 2; +} + +message AlertEvent { + Alert alert = 1; + AlertEventType event_type = 2; + int64 timestamp = 3; +} + +// ============================================================================ +// Training Metrics Messages (from monitoring_service) +// ============================================================================ + +message GetLiveTrainingMetricsRequest { + string model_filter = 1; // optional: "dqn", "ppo", etc. +} + +message StreamTrainingMetricsRequest { + string model_filter = 1; + uint32 interval_seconds = 2; // 0 = server default (3s) +} + +message GetLiveTrainingMetricsResponse { + repeated TrainingSession sessions = 1; + GpuSnapshot gpu = 2; + uint32 active_k8s_jobs = 3; + int64 timestamp = 4; + float cpu_percent = 5; + float memory_used_mb = 6; + float memory_total_mb = 7; +} + +message TrainingSession { + string model = 1; + string fold = 2; + bool is_hyperopt = 3; + + // Epoch/progress + float current_epoch = 4; + float epoch_loss = 5; + float validation_loss = 6; + + // Throughput + float batches_per_second = 7; + float batches_processed = 8; + float iteration_seconds = 9; + + // Eval metrics + float eval_accuracy = 10; + float eval_precision = 11; + float eval_recall = 12; + float eval_f1 = 13; + + // Checkpoint + float checkpoint_size_bytes = 14; + uint32 checkpoint_saves = 15; + uint32 checkpoint_failures = 16; + + // Health counters + uint32 nan_detected = 17; + uint32 gradient_explosions = 18; + uint32 feature_errors = 19; + + // Hyperopt fields (populated when is_hyperopt=true) + uint32 hyperopt_trial_current = 20; + uint32 hyperopt_trial_total = 21; + float hyperopt_best_objective = 22; + uint32 hyperopt_trials_failed = 23; + + // RL diagnostics + float q_value_mean = 24; + float q_value_max = 25; + float policy_entropy = 26; + float kl_divergence = 27; + float advantage_mean = 28; + uint32 replay_buffer_size = 29; + + // Gradient & training health + float gradient_norm = 30; + float learning_rate = 31; + float epoch_duration_seconds = 32; + + // Hyperopt intra-trial + uint32 hyperopt_trial_epoch = 33; + float hyperopt_trial_best_loss = 34; + float hyperopt_elapsed_seconds = 35; + + // Epoch-level financial metrics + float epoch_sharpe = 36; + float epoch_sortino = 37; + float epoch_win_rate = 38; + float epoch_max_drawdown = 39; + float epoch_profit_factor = 40; + float epoch_total_return = 41; + float epoch_avg_return = 42; + uint32 epoch_total_trades = 43; + // Action distribution + float action_buy_pct = 44; + float action_sell_pct = 45; + float action_hold_pct = 46; +} + +message GpuSnapshot { + float utilization_percent = 1; + float memory_used_mb = 2; + float memory_total_mb = 3; + float temperature_celsius = 4; + float power_watts = 5; +} + +message GetEpochHistoryRequest { + string model = 1; + string fold = 2; + uint32 max_epochs = 3; // 0 = all (up to 50) +} + +message EpochFinancialSnapshot { + uint32 epoch = 1; + float sharpe = 2; + float sortino = 3; + float win_rate = 4; + float max_drawdown = 5; + float profit_factor = 6; + float total_return = 7; + float avg_return = 8; + uint32 total_trades = 9; + float loss = 10; + float val_loss = 11; + float learning_rate = 12; + float action_buy_pct = 13; + float action_sell_pct = 14; + float action_hold_pct = 15; +} + +message GetEpochHistoryResponse { + string model = 1; + string fold = 2; + repeated EpochFinancialSnapshot epochs = 3; +} + +// ============================================================================ +// Enums +// ============================================================================ + +// Health status levels for services +enum ServiceHealth { + SERVICE_HEALTH_UNSPECIFIED = 0; // Default/unknown health + SERVICE_HEALTH_HEALTHY = 1; // Service operating normally + SERVICE_HEALTH_DEGRADED = 2; // Service performance degraded + SERVICE_HEALTH_UNHEALTHY = 3; // Service not functioning properly + SERVICE_HEALTH_CRITICAL = 4; // Service in critical failure state +} + +// Operational states of services +enum ServiceState { + SERVICE_STATE_UNSPECIFIED = 0; // Default/unknown state + SERVICE_STATE_STARTING = 1; // Service is starting up + SERVICE_STATE_RUNNING = 2; // Service is running normally + SERVICE_STATE_STOPPING = 3; // Service is shutting down + SERVICE_STATE_STOPPED = 4; // Service is stopped + SERVICE_STATE_ERROR = 5; // Service encountered an error +} + +enum SystemHealth { + SYSTEM_HEALTH_UNSPECIFIED = 0; + SYSTEM_HEALTH_HEALTHY = 1; + SYSTEM_HEALTH_DEGRADED = 2; + SYSTEM_HEALTH_UNHEALTHY = 3; + SYSTEM_HEALTH_CRITICAL = 4; +} + +enum HealthStatus { + HEALTH_STATUS_UNSPECIFIED = 0; + HEALTH_STATUS_HEALTHY = 1; + HEALTH_STATUS_DEGRADED = 2; + HEALTH_STATUS_UNHEALTHY = 3; + HEALTH_STATUS_CRITICAL = 4; +} + +enum DependencyType { + DEPENDENCY_TYPE_UNSPECIFIED = 0; + DEPENDENCY_TYPE_DATABASE = 1; + DEPENDENCY_TYPE_MESSAGE_QUEUE = 2; + DEPENDENCY_TYPE_CACHE = 3; + DEPENDENCY_TYPE_EXTERNAL_API = 4; + DEPENDENCY_TYPE_FILE_SYSTEM = 5; + DEPENDENCY_TYPE_NETWORK = 6; +} + +enum MetricType { + METRIC_TYPE_UNSPECIFIED = 0; + METRIC_TYPE_COUNTER = 1; + METRIC_TYPE_GAUGE = 2; + METRIC_TYPE_HISTOGRAM = 3; + METRIC_TYPE_TIMER = 4; +} + +enum MetricAggregation { + METRIC_AGGREGATION_UNSPECIFIED = 0; + METRIC_AGGREGATION_SUM = 1; + METRIC_AGGREGATION_AVG = 2; + METRIC_AGGREGATION_MIN = 3; + METRIC_AGGREGATION_MAX = 4; + METRIC_AGGREGATION_COUNT = 5; +} + +enum AlertType { + ALERT_TYPE_UNSPECIFIED = 0; + ALERT_TYPE_HEALTH_CHECK = 1; + ALERT_TYPE_PERFORMANCE = 2; + ALERT_TYPE_ERROR_RATE = 3; + ALERT_TYPE_LATENCY = 4; + ALERT_TYPE_THROUGHPUT = 5; + ALERT_TYPE_RESOURCE_USAGE = 6; + ALERT_TYPE_DEPENDENCY = 7; + ALERT_TYPE_SECURITY = 8; +} + +// Alert severity levels +enum AlertSeverity { + ALERT_SEVERITY_UNSPECIFIED = 0; // Default/unknown severity + ALERT_SEVERITY_INFO = 1; // Informational alert + ALERT_SEVERITY_WARNING = 2; // Warning requiring attention + ALERT_SEVERITY_CRITICAL = 3; // Critical issue requiring immediate action + ALERT_SEVERITY_EMERGENCY = 4; // Emergency requiring immediate response +} + +// Current status of alerts +enum AlertStatus { + ALERT_STATUS_UNSPECIFIED = 0; // Default/unknown status + ALERT_STATUS_ACTIVE = 1; // Alert is currently active + ALERT_STATUS_ACKNOWLEDGED = 2; // Alert has been acknowledged + ALERT_STATUS_RESOLVED = 3; // Alert has been resolved + ALERT_STATUS_SUPPRESSED = 4; // Alert is temporarily suppressed +} + +enum SystemStatusChangeType { + SYSTEM_STATUS_CHANGE_TYPE_UNSPECIFIED = 0; + SYSTEM_STATUS_CHANGE_TYPE_HEALTH_IMPROVED = 1; + SYSTEM_STATUS_CHANGE_TYPE_HEALTH_DEGRADED = 2; + SYSTEM_STATUS_CHANGE_TYPE_SERVICE_STARTED = 3; + SYSTEM_STATUS_CHANGE_TYPE_SERVICE_STOPPED = 4; + SYSTEM_STATUS_CHANGE_TYPE_SERVICE_ERROR = 5; +} + +enum AlertEventType { + ALERT_EVENT_TYPE_UNSPECIFIED = 0; + ALERT_EVENT_TYPE_TRIGGERED = 1; + ALERT_EVENT_TYPE_ACKNOWLEDGED = 2; + ALERT_EVENT_TYPE_RESOLVED = 3; + ALERT_EVENT_TYPE_ESCALATED = 4; +} diff --git a/proto/risk.proto b/proto/risk.proto new file mode 100644 index 000000000..f5a7cc14d --- /dev/null +++ b/proto/risk.proto @@ -0,0 +1,301 @@ +syntax = "proto3"; + +package risk; + +// Risk Management Service provides comprehensive risk assessment, monitoring, and control capabilities +// for high-frequency trading operations. This service integrates real-time VaR calculations, +// position risk analysis, compliance monitoring, and emergency controls. +service RiskService { + // Value at Risk (VaR) Calculations + // Calculate current portfolio VaR using specified method and parameters + rpc GetVaR(GetVaRRequest) returns (GetVaRResponse); + + // Stream real-time VaR updates as market conditions change + rpc StreamVaRUpdates(StreamVaRRequest) returns (stream VaREvent); + + // Position Risk Analysis + // Get comprehensive risk analysis for current positions + rpc GetPositionRisk(GetPositionRiskRequest) returns (GetPositionRiskResponse); + + // Validate order against risk limits before execution + rpc ValidateOrder(ValidateOrderRequest) returns (ValidateOrderResponse); + + // Risk Metrics and Monitoring + // Get comprehensive portfolio risk metrics and statistics + rpc GetRiskMetrics(GetRiskMetricsRequest) returns (GetRiskMetricsResponse); + + // Stream real-time risk alerts and violations + rpc StreamRiskAlerts(StreamRiskAlertsRequest) returns (stream RiskAlertEvent); + + // Emergency Controls and Circuit Breakers + // Trigger emergency stop to halt trading activities + rpc EmergencyStop(EmergencyStopRequest) returns (EmergencyStopResponse); + + // Get status of all circuit breakers and safety mechanisms + rpc GetCircuitBreakerStatus(GetCircuitBreakerStatusRequest) returns (GetCircuitBreakerStatusResponse); +} + +// VaR (Value at Risk) Messages + +// Request to calculate portfolio VaR +message GetVaRRequest { + repeated string symbols = 1; // Symbols to include in VaR calculation (empty = all positions) + double confidence_level = 2; // Confidence level (e.g., 0.95 for 95% VaR) + int32 lookback_days = 3; // Historical data period for calculation + VaRMethod method = 4; // VaR calculation method (historical, parametric, Monte Carlo) +} + +// Response containing VaR calculation results +message GetVaRResponse { + double portfolio_var = 1; // Total portfolio VaR value + repeated SymbolVaR symbol_vars = 2; // Individual symbol VaR contributions + double confidence_level = 3; // Confidence level used in calculation + int32 lookback_days = 4; // Historical period used + VaRMethod method = 5; // Calculation method used + int64 calculated_at = 6; // Calculation timestamp (nanoseconds) +} + +// Request to stream real-time VaR updates +message StreamVaRRequest { + double confidence_level = 1; // Confidence level for VaR calculation + int32 update_frequency_seconds = 2; // How often to send updates +} + +// VaR contribution for a specific symbol +message SymbolVaR { + string symbol = 1; // Trading symbol + double var_value = 2; // VaR value for this symbol + double position_size = 3; // Current position size + double contribution_pct = 4; // Percentage contribution to total portfolio VaR +} + +// Position Risk Analysis Messages + +// Request for position risk analysis +message GetPositionRiskRequest { + optional string symbol = 1; // Filter by symbol (all symbols if not specified) + optional string account_id = 2; // Filter by account (all accounts if not specified) +} + +// Response containing position risk analysis +message GetPositionRiskResponse { + repeated PositionRisk position_risks = 1; // Risk analysis for each position + double portfolio_risk_score = 2; // Overall portfolio risk score (0-100) +} + +// Request to validate order against risk limits +message ValidateOrderRequest { + string symbol = 1; // Trading symbol + double quantity = 2; // Order quantity + double price = 3; // Order price + string side = 4; // Buy or sell + string account_id = 5; // Trading account +} + +// Response containing order validation results +message ValidateOrderResponse { + bool is_valid = 1; // True if order passes all risk checks + repeated RiskViolation violations = 2; // List of risk violations (if any) + RiskScore risk_score = 3; // Risk assessment for this order + string message = 4; // Human-readable validation message +} + +// Risk Metrics and Monitoring Messages + +// Request for comprehensive risk metrics +message GetRiskMetricsRequest { + optional string portfolio_id = 1; // Portfolio identifier (default portfolio if not specified) +} + +// Response containing comprehensive risk metrics +message GetRiskMetricsResponse { + RiskMetrics metrics = 1; // Complete risk metrics and statistics + int64 calculated_at = 2; // Metrics calculation timestamp (nanoseconds) +} + +// Request to stream real-time risk alerts +message StreamRiskAlertsRequest { + RiskAlertSeverity min_severity = 1; // Minimum alert severity to receive + repeated RiskAlertType alert_types = 2; // Types of alerts to receive (empty = all types) +} + +// Emergency Control Messages + +// Request to trigger emergency stop +message EmergencyStopRequest { + EmergencyStopType stop_type = 1; // Type of emergency stop (all trading, symbol, account, etc.) + string reason = 2; // Reason for emergency stop + optional string symbol = 3; // Symbol to stop (for symbol-specific stops) + optional string account_id = 4; // Account to stop (for account-specific stops) +} + +// Response after emergency stop execution +message EmergencyStopResponse { + bool success = 1; // True if emergency stop was successful + string message = 2; // Status message or error description + int64 timestamp = 3; // Emergency stop timestamp (nanoseconds) + repeated string affected_orders = 4; // List of order IDs affected by the stop +} + +// Request for circuit breaker status +message GetCircuitBreakerStatusRequest { + optional string symbol = 1; // Filter by symbol (all symbols if not specified) +} + +// Response containing circuit breaker status +message GetCircuitBreakerStatusResponse { + repeated CircuitBreakerStatus circuit_breakers = 1; // Status of all circuit breakers +} + +// Core Risk Data Types + +// Risk analysis for a specific position +message PositionRisk { + string symbol = 1; // Trading symbol + double position_size = 2; // Current position size + double market_value = 3; // Market value of position + double var_contribution = 4; // Contribution to portfolio VaR + double concentration_risk = 5; // Position concentration risk (0-100) + double liquidity_risk = 6; // Liquidity risk score (0-100) + RiskScore overall_score = 7; // Overall risk assessment + repeated RiskMetric metrics = 8; // Additional risk metrics +} + +message RiskViolation { + RiskViolationType violation_type = 1; + string description = 2; + double current_value = 3; + double limit_value = 4; + RiskAlertSeverity severity = 5; +} + +message RiskScore { + double overall_score = 1; + double concentration_score = 2; + double liquidity_score = 3; + double volatility_score = 4; + double correlation_score = 5; + RiskLevel risk_level = 6; +} + +message RiskMetrics { + double portfolio_var_1d = 1; + double portfolio_var_5d = 2; + double portfolio_var_30d = 3; + double max_drawdown = 4; + double current_drawdown = 5; + double sharpe_ratio = 6; + double sortino_ratio = 7; + double beta = 8; + double alpha = 9; + double volatility = 10; + repeated PositionRisk position_risks = 11; +} + +message RiskMetric { + string name = 1; + double value = 2; + string unit = 3; + RiskLevel risk_level = 4; +} + +message CircuitBreakerStatus { + string name = 1; + bool is_triggered = 2; + optional string trigger_reason = 3; + optional int64 triggered_at = 4; + optional int64 reset_at = 5; + CircuitBreakerType breaker_type = 6; +} + +// Event Messages +message VaREvent { + double portfolio_var = 1; + repeated SymbolVaR symbol_vars = 2; + VaRChangeType change_type = 3; + int64 timestamp = 4; +} + +message RiskAlertEvent { + string alert_id = 1; + RiskAlertType alert_type = 2; + RiskAlertSeverity severity = 3; + string message = 4; + optional string symbol = 5; + optional string account_id = 6; + map metadata = 7; + int64 timestamp = 8; +} + +// Enums + +// VaR calculation methodology +enum VaRMethod { + VAR_METHOD_UNSPECIFIED = 0; // Default/unknown method + VAR_METHOD_HISTORICAL = 1; // Historical simulation method + VAR_METHOD_PARAMETRIC = 2; // Parametric (variance-covariance) method + VAR_METHOD_MONTE_CARLO = 3; // Monte Carlo simulation method +} + +enum RiskViolationType { + RISK_VIOLATION_TYPE_UNSPECIFIED = 0; + RISK_VIOLATION_TYPE_POSITION_LIMIT = 1; + RISK_VIOLATION_TYPE_CONCENTRATION = 2; + RISK_VIOLATION_TYPE_VAR_LIMIT = 3; + RISK_VIOLATION_TYPE_DRAWDOWN = 4; + RISK_VIOLATION_TYPE_LIQUIDITY = 5; + RISK_VIOLATION_TYPE_CORRELATION = 6; +} + +// Risk assessment levels +enum RiskLevel { + RISK_LEVEL_UNSPECIFIED = 0; // Default/unknown level + RISK_LEVEL_LOW = 1; // Low risk (green) + RISK_LEVEL_MEDIUM = 2; // Medium risk (yellow) + RISK_LEVEL_HIGH = 3; // High risk (orange) + RISK_LEVEL_CRITICAL = 4; // Critical risk (red) +} + +// Severity levels for risk alerts +enum RiskAlertSeverity { + RISK_ALERT_SEVERITY_UNSPECIFIED = 0; // Default/unknown severity + RISK_ALERT_SEVERITY_INFO = 1; // Informational alert + RISK_ALERT_SEVERITY_WARNING = 2; // Warning alert + RISK_ALERT_SEVERITY_CRITICAL = 3; // Critical alert requiring attention + RISK_ALERT_SEVERITY_EMERGENCY = 4; // Emergency alert requiring immediate action +} + +// Types of risk alerts +enum RiskAlertType { + RISK_ALERT_TYPE_UNSPECIFIED = 0; // Default/unknown type + RISK_ALERT_TYPE_VAR_BREACH = 1; // VaR limit breach + RISK_ALERT_TYPE_POSITION_LIMIT = 2; // Position size limit breach + RISK_ALERT_TYPE_DRAWDOWN = 3; // Drawdown limit breach + RISK_ALERT_TYPE_CONCENTRATION = 4; // Portfolio concentration risk + RISK_ALERT_TYPE_LIQUIDITY = 5; // Liquidity risk alert + RISK_ALERT_TYPE_CORRELATION = 6; // Correlation risk alert +} + +// Types of emergency stops +enum EmergencyStopType { + EMERGENCY_STOP_TYPE_UNSPECIFIED = 0; // Default/unknown type + EMERGENCY_STOP_TYPE_ALL_TRADING = 1; // Stop all trading activity + EMERGENCY_STOP_TYPE_SYMBOL = 2; // Stop trading for specific symbol + EMERGENCY_STOP_TYPE_ACCOUNT = 3; // Stop trading for specific account + EMERGENCY_STOP_TYPE_STRATEGY = 4; // Stop specific trading strategy +} + +enum CircuitBreakerType { + CIRCUIT_BREAKER_TYPE_UNSPECIFIED = 0; + CIRCUIT_BREAKER_TYPE_PORTFOLIO_LOSS = 1; + CIRCUIT_BREAKER_TYPE_SYMBOL_VOLATILITY = 2; + CIRCUIT_BREAKER_TYPE_POSITION_SIZE = 3; + CIRCUIT_BREAKER_TYPE_DRAWDOWN = 4; +} + +enum VaRChangeType { + VAR_CHANGE_TYPE_UNSPECIFIED = 0; + VAR_CHANGE_TYPE_INCREASED = 1; + VAR_CHANGE_TYPE_DECREASED = 2; + VAR_CHANGE_TYPE_BREACH = 3; +} \ No newline at end of file diff --git a/proto/trading.proto b/proto/trading.proto new file mode 100644 index 000000000..53bafc934 --- /dev/null +++ b/proto/trading.proto @@ -0,0 +1,481 @@ +syntax = "proto3"; + +package trading; + +// Trading Service provides comprehensive real-time trading operations for high-frequency trading. +// This service handles order management, position tracking, market data streaming, and execution monitoring. +// All operations are designed for ultra-low latency with microsecond precision timing. +service TradingService { + // Order Management Operations + // Submit a new trading order with validation and risk checks + rpc SubmitOrder(SubmitOrderRequest) returns (SubmitOrderResponse); + + // Cancel an existing order by order ID + rpc CancelOrder(CancelOrderRequest) returns (CancelOrderResponse); + + // Get current status of a specific order + rpc GetOrderStatus(GetOrderStatusRequest) returns (GetOrderStatusResponse); + + // Stream real-time order events for monitoring order lifecycle + rpc StreamOrders(StreamOrdersRequest) returns (stream OrderEvent); + + // Position Management Operations + // Get current positions for account and/or symbol + rpc GetPositions(GetPositionsRequest) returns (GetPositionsResponse); + + // Stream real-time position updates as trades execute + rpc StreamPositions(StreamPositionsRequest) returns (stream PositionEvent); + + // Get comprehensive portfolio summary with P&L and risk metrics + rpc GetPortfolioSummary(GetPortfolioSummaryRequest) returns (GetPortfolioSummaryResponse); + + // Market Data Operations + // Stream real-time market data (trades, quotes, order book) + rpc StreamMarketData(StreamMarketDataRequest) returns (stream MarketDataEvent); + + // Get current order book snapshot for a symbol + rpc GetOrderBook(GetOrderBookRequest) returns (GetOrderBookResponse); + + // Execution Operations + // Stream real-time trade executions as they occur + rpc StreamExecutions(StreamExecutionsRequest) returns (stream ExecutionEvent); + + // Get historical execution data with filtering options + rpc GetExecutionHistory(GetExecutionHistoryRequest) returns (GetExecutionHistoryResponse); + + // ML-specific Trading Operations + // Submit ML-generated trading order with ensemble predictions + rpc SubmitMLOrder(MLOrderRequest) returns (MLOrderResponse); + + // Get ML prediction history with outcomes + rpc GetMLPredictions(MLPredictionsRequest) returns (MLPredictionsResponse); + + // Get ML model performance metrics + rpc GetMLPerformance(MLPerformanceRequest) returns (MLPerformanceResponse); + + // Wave D: Regime Detection Operations + // Get current regime state for a symbol + rpc GetRegimeState(GetRegimeStateRequest) returns (GetRegimeStateResponse); + + // Get regime transition history for a symbol + rpc GetRegimeTransitions(GetRegimeTransitionsRequest) returns (GetRegimeTransitionsResponse); +} + +// Order Management Messages + +// Request to submit a new trading order +message SubmitOrderRequest { + string symbol = 1; // Trading symbol (e.g., "AAPL", "BTC-USD") + OrderSide side = 2; // Buy or sell direction + double quantity = 3; // Number of shares/units to trade + OrderType order_type = 4; // Market, limit, stop, or stop-limit + optional double price = 5; // Limit price (required for limit orders) + optional double stop_price = 6; // Stop price (required for stop orders) + string account_id = 7; // Trading account identifier + map metadata = 8; // Additional order metadata (strategy, tags, etc.) +} + +// Response after submitting an order +message SubmitOrderResponse { + string order_id = 1; // Unique order identifier assigned by system + OrderStatus status = 2; // Current order status (pending, submitted, etc.) + string message = 3; // Status message or error description + int64 timestamp = 4; // Order submission timestamp (nanoseconds) +} + +// Request to cancel an existing order +message CancelOrderRequest { + string order_id = 1; // Order ID to cancel + string account_id = 2; // Account ID for verification +} + +// Response after attempting to cancel an order +message CancelOrderResponse { + bool success = 1; // True if cancellation was successful + string message = 2; // Success confirmation or error message + int64 timestamp = 3; // Cancellation timestamp (nanoseconds) +} + +// Request to get current status of an order +message GetOrderStatusRequest { + string order_id = 1; // Order ID to query +} + +// Response containing order status information +message GetOrderStatusResponse { + Order order = 1; // Complete order details with current status +} + +// Request to stream real-time order events +message StreamOrdersRequest { + optional string account_id = 1; // Filter by account (all accounts if not specified) + optional string symbol = 2; // Filter by symbol (all symbols if not specified) +} + +// Position Management Messages + +// Request to get current positions +message GetPositionsRequest { + optional string account_id = 1; // Filter by account (all accounts if not specified) + optional string symbol = 2; // Filter by symbol (all symbols if not specified) +} + +// Response containing position information +message GetPositionsResponse { + repeated Position positions = 1; // List of current positions +} + +// Request to stream real-time position updates +message StreamPositionsRequest { + optional string account_id = 1; // Filter by account (all accounts if not specified) +} + +// Request for portfolio summary +message GetPortfolioSummaryRequest { + string account_id = 1; // Account ID for portfolio summary +} + +// Response containing comprehensive portfolio information +message GetPortfolioSummaryResponse { + double total_value = 1; // Total portfolio value in USD + double unrealized_pnl = 2; // Unrealized profit/loss + double realized_pnl = 3; // Realized profit/loss for the day + double day_pnl = 4; // Total P&L for the current trading day + double buying_power = 5; // Available buying power + double margin_used = 6; // Amount of margin currently used + repeated Position positions = 7; // Detailed position information +} + +// Market Data Messages + +// Request to stream real-time market data +message StreamMarketDataRequest { + repeated string symbols = 1; // List of symbols to subscribe to + repeated MarketDataType data_types = 2; // Types of data to stream (trades, quotes, order book) +} + +// Request for order book snapshot +message GetOrderBookRequest { + string symbol = 1; // Symbol to get order book for + optional int32 depth = 2; // Number of price levels (default: full book) +} + +// Response containing order book data +message GetOrderBookResponse { + OrderBook order_book = 1; // Current order book snapshot +} + +// Execution Messages + +// Request to stream real-time executions +message StreamExecutionsRequest { + optional string account_id = 1; // Filter by account (all accounts if not specified) + optional string symbol = 2; // Filter by symbol (all symbols if not specified) +} + +// Request for historical execution data +message GetExecutionHistoryRequest { + optional string account_id = 1; // Filter by account (all accounts if not specified) + optional string symbol = 2; // Filter by symbol (all symbols if not specified) + optional int64 start_time = 3; // Start time for query (nanoseconds) + optional int64 end_time = 4; // End time for query (nanoseconds) + optional int32 limit = 5; // Maximum number of executions to return +} + +// Response containing execution history +message GetExecutionHistoryResponse { + repeated Execution executions = 1; // List of historical executions +} + +// ML Trading Messages + +// Request to submit ML-generated order +message MLOrderRequest { + string symbol = 1; // Trading symbol (e.g., "ES.FUT") + string account_id = 2; // Trading account identifier + bool use_ensemble = 3; // Use ensemble voting or specific model + optional string model_name = 4; // Specific model name if not using ensemble + repeated double features = 5; // Feature vector for ML prediction (26 features: OHLCV + technicals) +} + +// Response after submitting ML order +message MLOrderResponse { + string order_id = 1; // Order ID if executed + string prediction_id = 2; // Prediction ID from ensemble_predictions table + string action = 3; // Action taken: BUY, SELL, HOLD + double confidence = 4; // Prediction confidence (0.0-1.0) + string message = 5; // Status message + bool executed = 6; // True if order was executed +} + +// Request to get ML prediction history +message MLPredictionsRequest { + string symbol = 1; // Trading symbol to filter by + optional string model_name = 2; // Filter by specific model + int32 limit = 3; // Maximum predictions to return (default: 100) + optional int64 start_time = 4; // Start time filter (nanoseconds) + optional int64 end_time = 5; // End time filter (nanoseconds) +} + +// Response containing ML prediction history +message MLPredictionsResponse { + repeated MLPrediction predictions = 1; // List of predictions with outcomes +} + +// Single ML prediction with outcome +message MLPrediction { + string id = 1; // Prediction ID (UUID) + string symbol = 2; // Trading symbol + string ensemble_action = 3; // Predicted action: BUY, SELL, HOLD + double ensemble_signal = 4; // Signal strength (-1.0 to 1.0) + double ensemble_confidence = 5; // Confidence level (0.0-1.0) + int64 timestamp = 6; // Prediction timestamp (nanoseconds) + optional string order_id = 7; // Order ID if executed + optional double actual_pnl = 8; // Actual P&L if order filled + repeated ModelPrediction model_predictions = 9; // Individual model predictions +} + +// Individual model prediction within ensemble +message ModelPrediction { + string model_name = 1; // Model name (DQN, MAMBA2, PPO, TFT) + double signal = 2; // Model signal strength + double confidence = 3; // Model confidence +} + +// Request to get ML model performance metrics +message MLPerformanceRequest { + optional string model_name = 1; // Filter by specific model (or all if not specified) + optional int64 start_time = 2; // Start time for metrics (nanoseconds) + optional int64 end_time = 3; // End time for metrics (nanoseconds) +} + +// Response containing ML model performance +message MLPerformanceResponse { + repeated ModelPerformance models = 1; // Performance metrics per model +} + +// Performance metrics for a single model +message ModelPerformance { + string model_name = 1; // Model name + int64 total_predictions = 2; // Total predictions made + int64 correct_predictions = 3; // Correct predictions (profitable) + double accuracy = 4; // Accuracy rate (0.0-1.0) + double sharpe_ratio = 5; // Risk-adjusted return + double avg_pnl = 6; // Average P&L per prediction +} + +// Wave D: Regime Detection Messages + +// Request to get current regime state +message GetRegimeStateRequest { + string symbol = 1; // Trading symbol to query +} + +// Response containing current regime state +message GetRegimeStateResponse { + string symbol = 1; // Trading symbol + string current_regime = 2; // Current regime: TRENDING, RANGING, VOLATILE, CRISIS + double confidence = 3; // Regime confidence (0.0-1.0) + double cusum_s_plus = 4; // CUSUM S+ statistic + double cusum_s_minus = 5; // CUSUM S- statistic + double adx = 6; // Average Directional Index + double stability = 7; // Regime stability score (0.0-1.0) + double entropy = 8; // Transition entropy (0.0-1.0) + int64 updated_at = 9; // Last update timestamp (nanoseconds) +} + +// Request to get regime transition history +message GetRegimeTransitionsRequest { + string symbol = 1; // Trading symbol to query + int32 limit = 2; // Maximum transitions to return (default: 100) +} + +// Response containing regime transition history +message GetRegimeTransitionsResponse { + repeated RegimeTransition transitions = 1; // List of regime transitions +} + +// Single regime transition record +message RegimeTransition { + string from_regime = 1; // Previous regime + string to_regime = 2; // New regime + int32 duration_bars = 3; // Duration in previous regime (bars) + double transition_probability = 4; // Transition probability from matrix + int64 timestamp = 5; // Transition timestamp (nanoseconds) +} + +// Core Data Types + +// Complete order information with all lifecycle details +message Order { + string order_id = 1; // Unique order identifier + string symbol = 2; // Trading symbol (e.g., "AAPL", "BTC-USD") + OrderSide side = 3; // Buy or sell direction + double quantity = 4; // Total quantity ordered + double filled_quantity = 5; // Quantity already filled + OrderType order_type = 6; // Market, limit, stop, or stop-limit + optional double price = 7; // Limit price (for limit orders) + optional double stop_price = 8; // Stop price (for stop orders) + OrderStatus status = 9; // Current order status + int64 created_at = 10; // Order creation timestamp (nanoseconds) + optional int64 updated_at = 11; // Last update timestamp (nanoseconds) + string account_id = 12; // Associated trading account + map metadata = 13; // Additional order metadata +} + +// Current position information for a symbol +message Position { + string symbol = 1; // Trading symbol + double quantity = 2; // Current position size (positive for long, negative for short) + double average_price = 3; // Average cost basis per share + double market_value = 4; // Current market value of position + double unrealized_pnl = 5; // Unrealized profit/loss + double realized_pnl = 6; // Realized profit/loss for the day + string account_id = 7; // Associated trading account + int64 updated_at = 8; // Last update timestamp (nanoseconds) +} + +// Trade execution details +message Execution { + string execution_id = 1; // Unique execution identifier + string order_id = 2; // Associated order ID + string symbol = 3; // Trading symbol + OrderSide side = 4; // Buy or sell direction + double quantity = 5; // Quantity executed + double price = 6; // Execution price + int64 timestamp = 7; // Execution timestamp (nanoseconds) + string account_id = 8; // Associated trading account + map metadata = 9; // Additional execution metadata +} + +// Order book snapshot for a symbol +message OrderBook { + string symbol = 1; // Trading symbol + repeated OrderBookLevel bids = 2; // Bid levels (buyers) sorted by price descending + repeated OrderBookLevel asks = 3; // Ask levels (sellers) sorted by price ascending + int64 timestamp = 4; // Order book timestamp (nanoseconds) +} + +// Single price level in the order book +message OrderBookLevel { + double price = 1; // Price level + double quantity = 2; // Total quantity at this price level + int32 order_count = 3; // Number of orders at this price level +} + +// Event Messages + +// Real-time order event notification +message OrderEvent { + string order_id = 1; // Order identifier + Order order = 2; // Complete order details + OrderEventType event_type = 3; // Type of event (created, updated, filled, etc.) + int64 timestamp = 4; // Event timestamp (nanoseconds) + string message = 5; // Event message or additional details +} + +// Real-time position change notification +message PositionEvent { + string symbol = 1; // Trading symbol + Position position = 2; // Updated position details + PositionEventType event_type = 3; // Type of event (opened, updated, closed) + int64 timestamp = 4; // Event timestamp (nanoseconds) + double quantity = 5; // Position quantity (quick access) + double average_price = 6; // Average entry price (quick access) + double unrealized_pnl = 7; // Unrealized P&L (quick access) +} + +// Real-time execution notification +message ExecutionEvent { + string execution_id = 1; // Execution identifier + Execution execution = 2; // Execution details + int64 timestamp = 3; // Event timestamp (nanoseconds) + string order_id = 4; // Associated order ID (quick access) + string symbol = 5; // Trading symbol (quick access) + double quantity = 6; // Executed quantity (quick access) + double price = 7; // Execution price (quick access) +} + +// Real-time market data update +message MarketDataEvent { + string symbol = 1; // Trading symbol + MarketDataType data_type = 2; // Type of market data + oneof data { + Trade trade = 3; // Trade data (when data_type = TRADE) + Quote quote = 4; // Quote data (when data_type = QUOTE) + OrderBook order_book = 5; // Order book data (when data_type = ORDER_BOOK) + } + int64 timestamp = 6; // Market data timestamp (nanoseconds) +} + +// Market trade information +message Trade { + double price = 1; // Trade price + double volume = 2; // Trade volume + int64 timestamp = 3; // Trade timestamp (nanoseconds) +} + +// Market quote (bid/ask) information +message Quote { + double bid_price = 1; // Best bid price + double bid_size = 2; // Size at best bid + double ask_price = 3; // Best ask price + double ask_size = 4; // Size at best ask + int64 timestamp = 5; // Quote timestamp (nanoseconds) +} + +// Enums + +// Order direction (buy or sell) +enum OrderSide { + ORDER_SIDE_UNSPECIFIED = 0; // Default/unknown side + ORDER_SIDE_BUY = 1; // Buy order (long position) + ORDER_SIDE_SELL = 2; // Sell order (short position) +} + +// Order type determining execution behavior +enum OrderType { + ORDER_TYPE_UNSPECIFIED = 0; // Default/unknown type + ORDER_TYPE_MARKET = 1; // Execute immediately at market price + ORDER_TYPE_LIMIT = 2; // Execute only at specified price or better + ORDER_TYPE_STOP = 3; // Market order triggered at stop price + ORDER_TYPE_STOP_LIMIT = 4; // Limit order triggered at stop price +} + +// Current status of an order in its lifecycle +enum OrderStatus { + ORDER_STATUS_UNSPECIFIED = 0; // Default/unknown status + ORDER_STATUS_PENDING = 1; // Order created but not yet submitted + ORDER_STATUS_SUBMITTED = 2; // Order submitted to exchange + ORDER_STATUS_PARTIALLY_FILLED = 3; // Order partially executed + ORDER_STATUS_FILLED = 4; // Order completely executed + ORDER_STATUS_CANCELLED = 5; // Order cancelled by user or system + ORDER_STATUS_REJECTED = 6; // Order rejected by exchange or risk system +} + +// Type of order event notification +enum OrderEventType { + ORDER_EVENT_TYPE_UNSPECIFIED = 0; // Default/unknown event + ORDER_EVENT_TYPE_CREATED = 1; // Order was created + ORDER_EVENT_TYPE_UPDATED = 2; // Order details were updated + ORDER_EVENT_TYPE_FILLED = 3; // Order was executed (full or partial) + ORDER_EVENT_TYPE_CANCELLED = 4; // Order was cancelled + ORDER_EVENT_TYPE_PARTIALLY_FILLED = 5; // Order was partially filled + ORDER_EVENT_TYPE_REJECTED = 6; // Order was rejected +} + +// Type of position change event +enum PositionEventType { + POSITION_EVENT_TYPE_UNSPECIFIED = 0; // Default/unknown event + POSITION_EVENT_TYPE_OPENED = 1; // New position was opened + POSITION_EVENT_TYPE_UPDATED = 2; // Existing position was modified + POSITION_EVENT_TYPE_CLOSED = 3; // Position was closed +} + +// Type of market data being streamed +enum MarketDataType { + MARKET_DATA_TYPE_UNSPECIFIED = 0; // Default/unknown type + MARKET_DATA_TYPE_TRADE = 1; // Trade/transaction data + MARKET_DATA_TYPE_QUOTE = 2; // Best bid/ask quotes + MARKET_DATA_TYPE_ORDER_BOOK = 3; // Full order book depth +} diff --git a/proto/trading_agent.proto b/proto/trading_agent.proto new file mode 100644 index 000000000..c51737cc2 --- /dev/null +++ b/proto/trading_agent.proto @@ -0,0 +1,615 @@ +syntax = "proto3"; + +package trading_agent; + +// Trading Agent Service orchestrates trading decisions across universe selection, +// asset selection, portfolio allocation, and strategy coordination. +service TradingAgentService { + // Universe Management + // Select tradable universe based on liquidity, volatility, and ML signals + rpc SelectUniverse(SelectUniverseRequest) returns (SelectUniverseResponse); + + // Get current trading universe configuration + rpc GetUniverse(GetUniverseRequest) returns (GetUniverseResponse); + + // Update universe selection criteria + rpc UpdateUniverseCriteria(UpdateUniverseCriteriaRequest) returns (UpdateUniverseCriteriaResponse); + + // Asset Selection + // Select specific assets to trade within universe + rpc SelectAssets(SelectAssetsRequest) returns (SelectAssetsResponse); + + // Get current asset selection with scores + rpc GetSelectedAssets(GetSelectedAssetsRequest) returns (GetSelectedAssetsResponse); + + // Portfolio Allocation + // Allocate capital across selected assets + rpc AllocatePortfolio(AllocatePortfolioRequest) returns (AllocatePortfolioResponse); + + // Get current portfolio allocation + rpc GetAllocation(GetAllocationRequest) returns (GetAllocationResponse); + + // Rebalance portfolio based on target allocation + rpc RebalancePortfolio(RebalancePortfolioRequest) returns (RebalancePortfolioResponse); + + // Order Generation + // Generate orders based on allocation and ML signals + rpc GenerateOrders(GenerateOrdersRequest) returns (GenerateOrdersResponse); + + // Submit generated orders to Trading Service + rpc SubmitAgentOrders(SubmitAgentOrdersRequest) returns (SubmitAgentOrdersResponse); + + // Strategy Coordination + // Register a trading strategy with the agent + rpc RegisterStrategy(RegisterStrategyRequest) returns (RegisterStrategyResponse); + + // Get list of active strategies + rpc ListStrategies(ListStrategiesRequest) returns (ListStrategiesResponse); + + // Enable/disable a strategy + rpc UpdateStrategyStatus(UpdateStrategyStatusRequest) returns (UpdateStrategyStatusResponse); + + // Agent Monitoring + // Get comprehensive agent status and performance + rpc GetAgentStatus(GetAgentStatusRequest) returns (GetAgentStatusResponse); + + // Stream real-time agent decisions and actions + rpc StreamAgentActivity(StreamAgentActivityRequest) returns (stream AgentActivityEvent); + + // Get agent performance metrics + rpc GetAgentPerformance(GetAgentPerformanceRequest) returns (GetAgentPerformanceResponse); + + // Service Health + rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse); +} + +// Universe Selection Messages + +message SelectUniverseRequest { + UniverseCriteria criteria = 1; // Selection criteria + optional uint32 max_instruments = 2; // Maximum instruments in universe + bool force_refresh = 3; // Force recalculation +} + +message SelectUniverseResponse { + repeated Instrument instruments = 1; // Selected instruments + UniverseMetrics metrics = 2; // Universe quality metrics + int64 timestamp = 3; // Selection timestamp (nanoseconds) + string universe_id = 4; // Unique universe identifier +} + +message GetUniverseRequest { + optional string universe_id = 1; // Get specific universe, or current if not specified +} + +message GetUniverseResponse { + string universe_id = 1; + repeated Instrument instruments = 2; + UniverseCriteria criteria = 3; + UniverseMetrics metrics = 4; + int64 created_at = 5; // Unix timestamp (nanoseconds) + int64 updated_at = 6; +} + +message UpdateUniverseCriteriaRequest { + UniverseCriteria criteria = 1; +} + +message UpdateUniverseCriteriaResponse { + bool success = 1; + string message = 2; + string universe_id = 3; // New universe ID after update +} + +// Asset Selection Messages + +message SelectAssetsRequest { + string universe_id = 1; // Universe to select from + AssetSelectionCriteria criteria = 2; // Selection criteria + uint32 max_assets = 3; // Maximum assets to select +} + +message SelectAssetsResponse { + repeated AssetScore assets = 1; // Selected assets with scores + SelectionMetrics metrics = 2; // Selection quality metrics + int64 timestamp = 3; +} + +message GetSelectedAssetsRequest { + optional string universe_id = 1; +} + +message GetSelectedAssetsResponse { + repeated AssetScore assets = 1; + SelectionMetrics metrics = 2; + int64 timestamp = 3; +} + +// Portfolio Allocation Messages + +message AllocatePortfolioRequest { + repeated AssetScore assets = 1; // Assets to allocate across + AllocationStrategy strategy = 2; // Allocation algorithm + RiskConstraints risk_constraints = 3; // Risk limits + double total_capital = 4; // Total capital to allocate +} + +message AllocatePortfolioResponse { + repeated AssetAllocation allocations = 1; // Allocation per asset + AllocationMetrics metrics = 2; // Allocation quality metrics + int64 timestamp = 3; + string allocation_id = 4; +} + +message GetAllocationRequest { + optional string allocation_id = 1; // Get specific allocation, or current if not specified +} + +message GetAllocationResponse { + string allocation_id = 1; + repeated AssetAllocation allocations = 2; + AllocationMetrics metrics = 3; + int64 created_at = 4; + double total_capital = 5; +} + +message RebalancePortfolioRequest { + string allocation_id = 1; // Target allocation + double rebalance_threshold = 2; // Min deviation to trigger rebalance (%) + bool force_rebalance = 3; // Force rebalance regardless of threshold +} + +message RebalancePortfolioResponse { + repeated RebalanceAction actions = 1; // Required rebalancing actions + RebalanceMetrics metrics = 2; + bool rebalance_required = 3; + int64 timestamp = 4; +} + +// Order Generation Messages + +message GenerateOrdersRequest { + string allocation_id = 1; // Target allocation + repeated MLSignal ml_signals = 2; // ML predictions for timing + OrderGenerationStrategy strategy = 3; // Order generation algorithm +} + +message GenerateOrdersResponse { + repeated GeneratedOrder orders = 1; // Generated order instructions + OrderGenerationMetrics metrics = 2; + int64 timestamp = 3; + string order_batch_id = 4; +} + +message SubmitAgentOrdersRequest { + string order_batch_id = 1; // Batch ID from GenerateOrders + repeated GeneratedOrder orders = 2; // Orders to submit + bool dry_run = 3; // Test without actual submission +} + +message SubmitAgentOrdersResponse { + repeated OrderSubmissionResult results = 1; // Submission results per order + OrderSubmissionMetrics metrics = 2; + int64 timestamp = 3; +} + +// Strategy Coordination Messages + +message RegisterStrategyRequest { + string strategy_name = 1; // Unique strategy name + StrategyType strategy_type = 2; // Strategy category + StrategyConfig config = 3; // Strategy configuration + bool auto_enable = 4; // Enable immediately after registration +} + +message RegisterStrategyResponse { + bool success = 1; + string strategy_id = 2; + string message = 3; +} + +message ListStrategiesRequest { + optional StrategyStatus status_filter = 1; // Filter by status +} + +message ListStrategiesResponse { + repeated Strategy strategies = 1; +} + +message UpdateStrategyStatusRequest { + string strategy_id = 1; + StrategyStatus new_status = 2; + optional string reason = 3; +} + +message UpdateStrategyStatusResponse { + bool success = 1; + string message = 2; + Strategy updated_strategy = 3; +} + +// Agent Monitoring Messages + +message GetAgentStatusRequest { + bool include_performance = 1; // Include performance metrics + bool include_positions = 2; // Include current positions +} + +message GetAgentStatusResponse { + AgentStatus status = 1; + optional AgentPerformanceMetrics performance = 2; + optional PositionSummary positions = 3; + int64 timestamp = 4; +} + +message StreamAgentActivityRequest { + repeated ActivityType activity_types = 1; // Filter by activity type +} + +message AgentActivityEvent { + ActivityType activity_type = 1; + oneof event { + UniverseSelectionEvent universe_event = 2; + AssetSelectionEvent asset_event = 3; + AllocationEvent allocation_event = 4; + OrderGenerationEvent order_event = 5; + StrategyEvent strategy_event = 6; + } + int64 timestamp = 7; +} + +message GetAgentPerformanceRequest { + optional int64 start_time = 1; // Performance window start (nanoseconds) + optional int64 end_time = 2; // Performance window end (nanoseconds) + bool include_strategy_breakdown = 3; // Include per-strategy performance +} + +message GetAgentPerformanceResponse { + AgentPerformanceMetrics metrics = 1; + repeated StrategyPerformance strategy_performance = 2; + int64 timestamp = 3; +} + +message HealthCheckRequest {} + +message HealthCheckResponse { + bool healthy = 1; + string message = 2; + map details = 3; +} + +// Data Structures + +message Instrument { + string symbol = 1; // Trading symbol (ES.FUT, NQ.FUT) + string exchange = 2; // Exchange identifier + InstrumentType instrument_type = 3; // Futures, equity, FX, etc. + double liquidity_score = 4; // Liquidity rating (0.0-1.0) + double volatility = 5; // Annualized volatility + double ml_signal_strength = 6; // ML prediction confidence + map metadata = 7; +} + +message UniverseCriteria { + double min_liquidity_score = 1; // Minimum liquidity threshold + double min_volatility = 2; // Minimum volatility + double max_volatility = 3; // Maximum volatility + repeated InstrumentType allowed_types = 4; + repeated string exchanges = 5; // Allowed exchanges + double min_ml_confidence = 6; // Minimum ML signal confidence +} + +message UniverseMetrics { + uint32 total_instruments = 1; + double avg_liquidity_score = 2; + double avg_volatility = 3; + double portfolio_diversification = 4; // 0.0-1.0 +} + +message AssetSelectionCriteria { + double min_ml_signal_strength = 1; // Minimum ML confidence + double min_sharpe_ratio = 2; // Minimum risk-adjusted return + SelectionMode mode = 3; // Top-N, threshold-based, etc. +} + +message AssetScore { + string symbol = 1; + double ml_score = 2; // ML model prediction score + double momentum_score = 3; // Momentum factor score + double value_score = 4; // Value factor score + double quality_score = 5; // Quality factor score + double composite_score = 6; // Final weighted score + map model_scores = 7; // Per-model scores (DQN, MAMBA2, etc.) +} + +message SelectionMetrics { + uint32 assets_evaluated = 1; + uint32 assets_selected = 2; + double avg_composite_score = 3; + double min_score = 4; + double max_score = 5; +} + +message AllocationStrategy { + AllocationType allocation_type = 1; // Equal-weight, risk-parity, etc. + map parameters = 2; // Strategy-specific parameters +} + +message RiskConstraints { + double max_position_size_pct = 1; // Max % of portfolio per position + double max_sector_exposure_pct = 2; // Max % per sector + double max_volatility = 3; // Portfolio volatility limit + double max_var_95 = 4; // Value at Risk (95%) + double max_leverage = 5; // Maximum leverage ratio +} + +message AssetAllocation { + string symbol = 1; + double target_weight = 2; // Target allocation weight (0.0-1.0) + double target_capital = 3; // Target capital in USD + double target_quantity = 4; // Target position size + double current_weight = 5; // Current allocation weight + double current_quantity = 6; // Current position size + double rebalance_delta = 7; // Required change +} + +message AllocationMetrics { + double total_weight = 1; // Should be ~1.0 + double portfolio_volatility = 2; // Expected portfolio volatility + double portfolio_sharpe = 3; // Expected Sharpe ratio + double var_95 = 4; // Portfolio VaR (95%) + double max_drawdown_estimate = 5; // Expected max drawdown +} + +message RebalanceAction { + string symbol = 1; + double current_quantity = 2; + double target_quantity = 3; + double delta_quantity = 4; // Positive = buy, negative = sell + RebalanceReason reason = 5; +} + +message RebalanceMetrics { + uint32 total_rebalance_actions = 1; + double total_turnover = 2; // Total capital moved (USD) + double estimated_cost = 3; // Estimated transaction costs +} + +message MLSignal { + string symbol = 1; + string model_name = 2; // DQN, MAMBA2, PPO, TFT + double signal_strength = 3; // -1.0 to 1.0 (short to long) + double confidence = 4; // 0.0 to 1.0 + string predicted_action = 5; // BUY, SELL, HOLD + int64 timestamp = 6; +} + +message OrderGenerationStrategy { + OrderGenerationMode mode = 1; + double slippage_tolerance = 2; // Max acceptable slippage (%) + bool use_limit_orders = 3; // Use limit orders vs market + double limit_price_offset = 4; // Offset from mid price (%) +} + +message GeneratedOrder { + string symbol = 1; + OrderSide side = 2; // BUY or SELL + double quantity = 3; + OrderType order_type = 4; // MARKET, LIMIT, etc. + optional double price = 5; // Limit price if applicable + string rationale = 6; // Why this order was generated + map metadata = 7; +} + +message OrderGenerationMetrics { + uint32 orders_generated = 1; + double total_notional = 2; // Total order value (USD) + double avg_order_size = 3; +} + +message OrderSubmissionResult { + string symbol = 1; + bool success = 2; + optional string order_id = 3; // From Trading Service + optional string error_message = 4; +} + +message OrderSubmissionMetrics { + uint32 orders_submitted = 1; + uint32 orders_accepted = 2; + uint32 orders_rejected = 3; + double acceptance_rate = 4; +} + +message Strategy { + string strategy_id = 1; + string strategy_name = 2; + StrategyType strategy_type = 3; + StrategyStatus status = 4; + StrategyConfig config = 5; + StrategyPerformance performance = 6; + int64 created_at = 7; + int64 updated_at = 8; +} + +message StrategyConfig { + map parameters = 1; // Strategy-specific parameters + repeated string target_symbols = 2; // Symbols this strategy trades + double max_capital_pct = 3; // Max % of portfolio for this strategy +} + +message StrategyPerformance { + string strategy_id = 1; + double total_pnl = 2; + double sharpe_ratio = 3; + double win_rate = 4; + uint32 total_trades = 5; + int64 period_start = 6; + int64 period_end = 7; +} + +message AgentStatus { + AgentState state = 1; + string current_universe_id = 2; + uint32 active_strategies = 3; + uint32 selected_assets = 4; + double portfolio_utilization = 5; // % of capital deployed + int64 last_action_timestamp = 6; +} + +message AgentPerformanceMetrics { + double total_pnl = 1; + double sharpe_ratio = 2; + double max_drawdown = 3; + double win_rate = 4; + uint32 total_trades = 5; + double avg_trade_pnl = 6; + double portfolio_turnover = 7; // Annualized + int64 period_start = 8; + int64 period_end = 9; +} + +message PositionSummary { + repeated Position positions = 1; + double total_equity = 2; + double total_exposure = 3; + double leverage_ratio = 4; +} + +message Position { + string symbol = 1; + double quantity = 2; + double average_price = 3; + double market_value = 4; + double unrealized_pnl = 5; + double weight = 6; // % of portfolio +} + +message UniverseSelectionEvent { + string universe_id = 1; + repeated string added_symbols = 2; + repeated string removed_symbols = 3; + UniverseMetrics metrics = 4; +} + +message AssetSelectionEvent { + repeated AssetScore selected_assets = 1; + SelectionMetrics metrics = 2; +} + +message AllocationEvent { + string allocation_id = 1; + repeated AssetAllocation allocations = 2; + AllocationMetrics metrics = 3; +} + +message OrderGenerationEvent { + string order_batch_id = 1; + repeated GeneratedOrder orders = 2; + OrderGenerationMetrics metrics = 3; +} + +message StrategyEvent { + string strategy_id = 1; + StrategyEventType event_type = 2; + string message = 3; +} + +// Enums + +enum InstrumentType { + INSTRUMENT_TYPE_UNSPECIFIED = 0; + INSTRUMENT_TYPE_EQUITY = 1; + INSTRUMENT_TYPE_FUTURES = 2; + INSTRUMENT_TYPE_FX = 3; + INSTRUMENT_TYPE_OPTIONS = 4; + INSTRUMENT_TYPE_CRYPTO = 5; +} + +enum SelectionMode { + SELECTION_MODE_UNSPECIFIED = 0; + SELECTION_MODE_TOP_N = 1; // Select top N by score + SELECTION_MODE_THRESHOLD = 2; // Select all above threshold + SELECTION_MODE_QUANTILE = 3; // Select top quantile (e.g., top 20%) +} + +enum AllocationType { + ALLOCATION_TYPE_UNSPECIFIED = 0; + ALLOCATION_TYPE_EQUAL_WEIGHT = 1; // 1/N allocation + ALLOCATION_TYPE_RISK_PARITY = 2; // Equal risk contribution + ALLOCATION_TYPE_ML_OPTIMIZED = 3; // ML-based optimization + ALLOCATION_TYPE_KELLY = 4; // Kelly criterion + ALLOCATION_TYPE_MEAN_VARIANCE = 5; // Mean-variance optimization +} + +enum RebalanceReason { + REBALANCE_REASON_UNSPECIFIED = 0; + REBALANCE_REASON_DRIFT = 1; // Allocation drifted from target + REBALANCE_REASON_UNIVERSE_CHANGE = 2; // Universe updated + REBALANCE_REASON_RISK_LIMIT = 3; // Risk limit violation + REBALANCE_REASON_MANUAL = 4; // Manual rebalance request +} + +enum OrderGenerationMode { + ORDER_GENERATION_MODE_UNSPECIFIED = 0; + ORDER_GENERATION_MODE_AGGRESSIVE = 1; // Market orders, immediate execution + ORDER_GENERATION_MODE_PASSIVE = 2; // Limit orders, minimize slippage + ORDER_GENERATION_MODE_ADAPTIVE = 3; // Adapt based on market conditions +} + +enum OrderSide { + ORDER_SIDE_UNSPECIFIED = 0; + ORDER_SIDE_BUY = 1; + ORDER_SIDE_SELL = 2; +} + +enum OrderType { + ORDER_TYPE_UNSPECIFIED = 0; + ORDER_TYPE_MARKET = 1; + ORDER_TYPE_LIMIT = 2; + ORDER_TYPE_STOP = 3; + ORDER_TYPE_STOP_LIMIT = 4; +} + +enum StrategyType { + STRATEGY_TYPE_UNSPECIFIED = 0; + STRATEGY_TYPE_ML_ENSEMBLE = 1; // Ensemble ML predictions + STRATEGY_TYPE_MEAN_REVERSION = 2; // Mean reversion + STRATEGY_TYPE_MOMENTUM = 3; // Momentum/trend following + STRATEGY_TYPE_ARBITRAGE = 4; // Statistical arbitrage + STRATEGY_TYPE_MARKET_MAKING = 5; // Market making +} + +enum StrategyStatus { + STRATEGY_STATUS_UNSPECIFIED = 0; + STRATEGY_STATUS_ENABLED = 1; + STRATEGY_STATUS_DISABLED = 2; + STRATEGY_STATUS_PAUSED = 3; + STRATEGY_STATUS_ERROR = 4; +} + +enum AgentState { + AGENT_STATE_UNSPECIFIED = 0; + AGENT_STATE_INITIALIZING = 1; + AGENT_STATE_ACTIVE = 2; + AGENT_STATE_PAUSED = 3; + AGENT_STATE_ERROR = 4; + AGENT_STATE_SHUTDOWN = 5; +} + +enum ActivityType { + ACTIVITY_TYPE_UNSPECIFIED = 0; + ACTIVITY_TYPE_UNIVERSE_SELECTION = 1; + ACTIVITY_TYPE_ASSET_SELECTION = 2; + ACTIVITY_TYPE_ALLOCATION = 3; + ACTIVITY_TYPE_ORDER_GENERATION = 4; + ACTIVITY_TYPE_STRATEGY = 5; +} + +enum StrategyEventType { + STRATEGY_EVENT_TYPE_UNSPECIFIED = 0; + STRATEGY_EVENT_TYPE_REGISTERED = 1; + STRATEGY_EVENT_TYPE_ENABLED = 2; + STRATEGY_EVENT_TYPE_DISABLED = 3; + STRATEGY_EVENT_TYPE_ERROR = 4; +}