foxhunt/proto/data_acquisition.proto

syntax = "proto3";

package data_acquisition;

// Data Acquisition Service provides automated Databento data downloading and storage.
// This service handles scheduled downloads, cost tracking, quality validation, and automatic MinIO upload.
service DataAcquisitionService {
  // Download Management
  // Schedule a new data download from Databento
  rpc ScheduleDownload(ScheduleDownloadRequest) returns (ScheduleDownloadResponse);

  // Get status of a download job
  rpc GetDownloadStatus(GetDownloadStatusRequest) returns (GetDownloadStatusResponse);

  // Cancel a running or pending download job
  rpc CancelDownload(CancelDownloadRequest) returns (CancelDownloadResponse);

  // List all download jobs with optional filters
  rpc ListDownloadJobs(ListDownloadJobsRequest) returns (ListDownloadJobsResponse);

  // Service Health and Status
  // Check service health and resource availability
  rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse);

  // Server-streaming: polls ListDownloadJobs at gateway level
  rpc StreamDownloadStatus(StreamDownloadStatusRequest) returns (stream ListDownloadJobsResponse);
}

// Streaming request messages

message StreamDownloadStatusRequest {
  uint32 interval_seconds = 1; // 0 = server default (5s)
}

// --- Core Request/Response Messages ---

// Request to schedule a new data download
message ScheduleDownloadRequest {
  // Databento dataset (e.g., "GLBX.MDP3" for CME futures)
  string dataset = 1;

  // List of symbols to download (e.g., ["ES.FUT", "NQ.FUT"])
  repeated string symbols = 2;

  // Start date in YYYY-MM-DD format
  string start_date = 3;

  // End date in YYYY-MM-DD format
  string end_date = 4;

  // Schema type (e.g., "ohlcv-1m", "mbp-10", "trades")
  string schema = 5;

  // Optional description for this download
  string description = 6;

  // Optional tags for categorization
  map<string, string> tags = 7;

  // Priority level (1=low, 5=high)
  uint32 priority = 8;
}

message ScheduleDownloadResponse {
  // Unique job identifier
  string job_id = 1;

  // Initial job status
  DownloadStatus status = 2;

  // Estimated cost in USD
  double estimated_cost_usd = 3;

  // Human-readable message
  string message = 4;
}

message GetDownloadStatusRequest {
  string job_id = 1;
}

message GetDownloadStatusResponse {
  DownloadJobDetails job_details = 1;
}

message CancelDownloadRequest {
  string job_id = 1;
  string reason = 2; // Optional cancellation reason
}

message CancelDownloadResponse {
  bool success = 1;
  string message = 2;
}

message ListDownloadJobsRequest {
  uint32 page = 1;
  uint32 page_size = 2;
  DownloadStatus status_filter = 3;
  int64 start_time = 4; // Unix timestamp
  int64 end_time = 5; // Unix timestamp
}

message ListDownloadJobsResponse {
  repeated DownloadJobSummary jobs = 1;
  uint32 total_count = 2;
  uint32 page = 3;
  uint32 page_size = 4;
}

message HealthCheckRequest {}

message HealthCheckResponse {
  bool healthy = 1;
  string message = 2;
  map<string, string> details = 3;
}

// --- Status and Details Messages ---

enum DownloadStatus {
  DOWNLOAD_STATUS_UNKNOWN = 0;
  PENDING = 1;         // Queued, waiting to start
  DOWNLOADING = 2;     // Actively downloading from Databento
  VALIDATING = 3;      // Validating data quality
  UPLOADING = 4;       // Uploading to MinIO
  COMPLETED = 5;       // Successfully completed
  FAILED = 6;          // Failed with errors
  CANCELLED = 7;       // Cancelled by user
}

message DownloadJobDetails {
  string job_id = 1;
  DownloadStatus status = 2;
  string dataset = 3;
  repeated string symbols = 4;
  string start_date = 5;
  string end_date = 6;
  string schema = 7;
  string description = 8;
  map<string, string> tags = 9;
  uint32 priority = 10;

  // Progress tracking
  float progress_percentage = 11;  // 0.0 to 100.0
  uint64 bytes_downloaded = 12;
  uint64 total_bytes = 13;

  // Cost tracking
  double estimated_cost_usd = 14;
  double actual_cost_usd = 15;

  // Quality metrics
  uint64 records_count = 16;
  uint64 invalid_records = 17;
  double data_quality_score = 18;  // 0.0 to 1.0

  // Storage paths
  string local_path = 19;
  string minio_path = 20;

  // Timestamps
  int64 created_at = 21;  // Unix timestamp
  int64 started_at = 22;
  int64 completed_at = 23;

  // Error information
  string error_message = 24;
  uint32 retry_count = 25;

  // Metadata
  string created_by = 26;
}

message DownloadJobSummary {
  string job_id = 1;
  DownloadStatus status = 2;
  string dataset = 3;
  repeated string symbols = 4;
  string start_date = 5;
  string end_date = 6;
  float progress_percentage = 7;
  double actual_cost_usd = 8;
  int64 created_at = 9;
  int64 completed_at = 10;
}