feat: production readiness Phase 1-2 implementation

- fix(trading_engine): replace Prometheus panic! with graceful registration
- fix(trading_service): implement partial fill matching in order book
- feat(trading_service): replace feature extraction stub with real 51-dim pipeline
- feat(trading_service): wire RiskEngine with real VaR calculator
- fix(api_gateway): implement real ML prediction proxy
- feat(data_acquisition): implement DBN data downloader
- feat(data): wire DBN uploader with MinIO integration

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

data/Cargo.toml

@@ -105,6 +105,7 @@ redis = { workspace = true, optional = true }
 # Internal workspace crates
 trading_engine.workspace = true
 common = { path = "../common" }
+storage = { workspace = true }
 dbn = "0.42.0"
 
 # CLI argument parsing (needed for examples)

data/src/dbn_uploader.rs

@@ -129,10 +129,16 @@ impl DbnMetadata {
 pub struct DbnUploader {
     config: DbnUploaderConfig,
     detected_files: Arc<RwLock<Vec<PathBuf>>>,
+    /// Optional storage backend for MinIO/S3 uploads.
+    /// When `None`, upload operations will return an error.
+    storage_backend: Option<Arc<dyn storage::Storage>>,
 }
 
 impl DbnUploader {
-    /// Create new uploader
+    /// Create new uploader without a storage backend.
+    ///
+    /// The uploader can scan and compress files, but `upload_file` will return
+    /// an error unless a storage backend is attached via `with_storage`.
     pub async fn new(config: DbnUploaderConfig) -> DataResult<Self> {
         if !config.watch_path.exists() {
             return Err(DataError::Validation {
@@ -149,9 +155,19 @@ impl DbnUploader {
         Ok(Self {
             config,
             detected_files: Arc::new(RwLock::new(Vec::new())),
+            storage_backend: None,
         })
     }
 
+    /// Attach a storage backend (MinIO/S3) for actual uploads.
+    ///
+    /// Without a storage backend, `upload_file` and deduplication checks will
+    /// return errors indicating no backend is configured.
+    pub fn with_storage(mut self, backend: Arc<dyn storage::Storage>) -> Self {
+        self.storage_backend = Some(backend);
+        self
+    }
+
     /// Get list of detected files (for testing)
     pub async fn get_detected_files(&self) -> Vec<PathBuf> {
         self.detected_files.read().await.clone()
@@ -203,13 +219,40 @@ impl DbnUploader {
         }
     }
 
-    /// Check if file should be uploaded (deduplication)
-    pub async fn should_upload_file(&self, _path: &Path) -> DataResult<bool> {
+    /// Check if file should be uploaded (deduplication).
+    ///
+    /// When deduplication is enabled and a storage backend is configured,
+    /// this queries the backend to see if the upload key already exists.
+    /// Returns `true` if the file should be uploaded, `false` if it already
+    /// exists in the remote store.
+    ///
+    /// When no storage backend is configured, deduplication is skipped and
+    /// the file is always considered uploadable.
+    pub async fn should_upload_file(&self, path: &Path) -> DataResult<bool> {
         if !self.config.deduplication_enabled {
             return Ok(true);
         }
-        // TODO: Check if file exists in MinIO
-        Ok(true)
+
+        let Some(backend) = &self.storage_backend else {
+            // No backend configured -- cannot check remote, allow upload attempt
+            debug!("No storage backend configured, skipping deduplication check");
+            return Ok(true);
+        };
+
+        let key = Self::generate_upload_key(path, &self.config.upload_prefix)?;
+
+        match backend.exists(&key).await {
+            Ok(true) => {
+                info!("File already exists in remote storage, skipping: {}", key);
+                Ok(false)
+            },
+            Ok(false) => Ok(true),
+            Err(e) => {
+                error!("Failed to check deduplication in remote storage: {}", e);
+                // On error, allow the upload attempt rather than silently skipping
+                Ok(true)
+            },
+        }
     }
 
     /// Compress file with gzip
@@ -220,10 +263,27 @@ impl DbnUploader {
         Ok(encoder.finish()?)
     }
 
-    /// Generate MinIO upload key
-    pub fn generate_upload_key(path: &Path, prefix: &str) -> String {
-        let filename = path.file_name().unwrap().to_str().unwrap();
-        format!("{}{}.gz", prefix, filename)
+    /// Generate MinIO upload key from a file path and prefix.
+    ///
+    /// Returns the key in the form `{prefix}{filename}.gz`.
+    ///
+    /// # Errors
+    ///
+    /// Returns `DataError::Validation` if the path has no filename or the
+    /// filename is not valid UTF-8.
+    pub fn generate_upload_key(path: &Path, prefix: &str) -> DataResult<String> {
+        let filename = path
+            .file_name()
+            .ok_or_else(|| DataError::Validation {
+                field: "path".to_string(),
+                message: format!("Path has no filename component: {:?}", path),
+            })?
+            .to_str()
+            .ok_or_else(|| DataError::Validation {
+                field: "filename".to_string(),
+                message: format!("Filename is not valid UTF-8: {:?}", path),
+            })?;
+        Ok(format!("{}{}.gz", prefix, filename))
     }
 
     /// Generate metadata tags for MinIO
@@ -239,29 +299,66 @@ impl DbnUploader {
         tags
     }
 
-    /// Upload file to MinIO with metadata
+    /// Upload file to MinIO/S3 with metadata.
+    ///
+    /// Reads the file, optionally compresses it, checks for duplicates, and
+    /// stores it via the configured storage backend. The upload key is derived
+    /// from the file name and the configured prefix.
+    ///
+    /// # Errors
+    ///
+    /// Returns `DataError::Storage` if no storage backend is configured, or
+    /// any storage/IO error that occurs during the upload.
     pub async fn upload_file(&self, path: &Path) -> DataResult<()> {
         info!("Uploading file: {:?}", path);
 
+        let backend = self
+            .storage_backend
+            .as_ref()
+            .ok_or_else(|| {
+                DataError::Storage(
+                    "No storage backend configured. Call with_storage() to attach a \
+                     MinIO/S3 backend before uploading."
+                        .to_string(),
+                )
+            })?
+            .clone();
+
         let metadata = DbnMetadata::from_file(path).await?;
 
+        // Check deduplication before doing expensive compression
+        if !self.should_upload_file(path).await? {
+            info!("Skipping duplicate file: {:?}", path);
+            return Ok(());
+        }
+
         let data = if self.config.compression_enabled {
             Self::compress_file(path).await?
         } else {
             fs::read(path).await?
         };
 
-        let key = Self::generate_upload_key(path, &self.config.upload_prefix);
+        let key = Self::generate_upload_key(path, &self.config.upload_prefix)?;
         let tags = Self::generate_metadata_tags(&metadata);
 
         info!(
-            "Upload prepared: key={}, size={} bytes, tags={:?}",
+            "Uploading to storage: key={}, size={} bytes, tags={:?}",
             key,
             data.len(),
             tags
         );
 
-        // TODO: Actually upload to MinIO using storage::ObjectStoreBackend
+        backend.store(&key, &data).await.map_err(|e| {
+            DataError::Storage(format!("Failed to upload {} to storage backend: {}", key, e))
+        })?;
+
+        info!(
+            "Successfully uploaded {} ({} bytes, original {} bytes)",
+            key,
+            data.len(),
+            metadata.file_size_bytes,
+        );
+
         Ok(())
     }
 }
@@ -297,10 +394,18 @@ mod tests {
     #[tokio::test]
     async fn test_generate_upload_key() {
         let path = PathBuf::from("ES.FUT_ohlcv-1m_2024-01-02.dbn");
-        let key = DbnUploader::generate_upload_key(&path, "training-data/");
+        let key = DbnUploader::generate_upload_key(&path, "training-data/")
+            .expect("Failed to generate key");
         assert_eq!(key, "training-data/ES.FUT_ohlcv-1m_2024-01-02.dbn.gz");
     }
 
+    #[tokio::test]
+    async fn test_generate_upload_key_no_filename() {
+        let path = PathBuf::from("/");
+        let result = DbnUploader::generate_upload_key(&path, "prefix/");
+        assert!(result.is_err(), "Should fail for path with no filename");
+    }
+
     #[tokio::test]
     async fn test_generate_metadata_tags() {
         let metadata = DbnMetadata {
@@ -311,9 +416,146 @@ mod tests {
         };
 
         let tags = DbnUploader::generate_metadata_tags(&metadata);
-        assert_eq!(tags.get("symbol").unwrap(), "ES.FUT");
-        assert_eq!(tags.get("schema").unwrap(), "ohlcv-1m");
-        assert_eq!(tags.get("date_range").unwrap(), "2024-01-02");
-        assert_eq!(tags.get("original_size").unwrap(), "1024");
+        assert_eq!(
+            tags.get("symbol").cloned().as_deref(),
+            Some("ES.FUT")
+        );
+        assert_eq!(
+            tags.get("schema").cloned().as_deref(),
+            Some("ohlcv-1m")
+        );
+        assert_eq!(
+            tags.get("date_range").cloned().as_deref(),
+            Some("2024-01-02")
+        );
+        assert_eq!(
+            tags.get("original_size").cloned().as_deref(),
+            Some("1024")
+        );
+    }
+
+    #[tokio::test]
+    async fn test_upload_file_without_backend_returns_error() {
+        let temp_dir = tempfile::TempDir::new().expect("Failed to create temp dir");
+        let test_file = temp_dir.path().join("ES.FUT_ohlcv-1m_2024-01-02.dbn");
+        fs::write(&test_file, b"test data")
+            .await
+            .expect("Failed to write");
+
+        let config = DbnUploaderConfig {
+            watch_path: temp_dir.path().to_path_buf(),
+            compression_enabled: false,
+            deduplication_enabled: false,
+            ..DbnUploaderConfig::default()
+        };
+
+        let uploader = DbnUploader::new(config)
+            .await
+            .expect("Failed to create uploader");
+
+        let result = uploader.upload_file(&test_file).await;
+        assert!(result.is_err(), "Should fail without storage backend");
+        let err_msg = format!("{}", result.unwrap_err());
+        assert!(
+            err_msg.contains("No storage backend configured"),
+            "Error should mention missing backend, got: {}",
+            err_msg
+        );
+    }
+
+    #[tokio::test]
+    async fn test_upload_file_with_local_storage_backend() {
+        let watch_dir = tempfile::TempDir::new().expect("Failed to create watch dir");
+        let storage_dir = tempfile::TempDir::new().expect("Failed to create storage dir");
+
+        let test_file = watch_dir.path().join("ES.FUT_ohlcv-1m_2024-01-02.dbn");
+        fs::write(&test_file, b"fake dbn content for upload test")
+            .await
+            .expect("Failed to write");
+
+        // Use storage::local::LocalStorage as a real backend
+        let local_config = storage::local::LocalStorageConfig {
+            base_path: storage_dir.path().to_path_buf(),
+            ..Default::default()
+        };
+        let local_storage = storage::local::LocalStorage::new(local_config)
+            .await
+            .expect("Failed to create local storage");
+        let backend: Arc<dyn storage::Storage> = Arc::new(local_storage);
+
+        let config = DbnUploaderConfig {
+            watch_path: watch_dir.path().to_path_buf(),
+            compression_enabled: true,
+            deduplication_enabled: false,
+            ..DbnUploaderConfig::default()
+        };
+
+        let uploader = DbnUploader::new(config)
+            .await
+            .expect("Failed to create uploader")
+            .with_storage(backend.clone());
+
+        // First upload should succeed
+        uploader
+            .upload_file(&test_file)
+            .await
+            .expect("Upload should succeed");
+
+        // Verify the file was stored
+        let key = "training-data/ES.FUT_ohlcv-1m_2024-01-02.dbn.gz";
+        let exists = backend.exists(key).await.expect("Exists check failed");
+        assert!(exists, "Uploaded file should exist in storage backend");
+    }
+
+    #[tokio::test]
+    async fn test_deduplication_with_storage_backend() {
+        let watch_dir = tempfile::TempDir::new().expect("Failed to create watch dir");
+        let storage_dir = tempfile::TempDir::new().expect("Failed to create storage dir");
+
+        let test_file = watch_dir.path().join("ES.FUT_ohlcv-1m_2024-01-02.dbn");
+        fs::write(&test_file, b"dbn data")
+            .await
+            .expect("Failed to write");
+
+        let local_config = storage::local::LocalStorageConfig {
+            base_path: storage_dir.path().to_path_buf(),
+            ..Default::default()
+        };
+        let local_storage = storage::local::LocalStorage::new(local_config)
+            .await
+            .expect("Failed to create local storage");
+        let backend: Arc<dyn storage::Storage> = Arc::new(local_storage);
+
+        let config = DbnUploaderConfig {
+            watch_path: watch_dir.path().to_path_buf(),
+            compression_enabled: false,
+            deduplication_enabled: true,
+            ..DbnUploaderConfig::default()
+        };
+
+        let uploader = DbnUploader::new(config)
+            .await
+            .expect("Failed to create uploader")
+            .with_storage(backend.clone());
+
+        // File does not exist yet, should_upload should be true
+        let should = uploader
+            .should_upload_file(&test_file)
+            .await
+            .expect("Check failed");
+        assert!(should, "Should upload new file");
+
+        // Upload once
+        uploader
+            .upload_file(&test_file)
+            .await
+            .expect("First upload should succeed");
+
+        // Now the file exists, should_upload should be false
+        let should = uploader
+            .should_upload_file(&test_file)
+            .await
+            .expect("Check failed");
+        assert!(!should, "Should NOT upload duplicate file");
     }
 }

data/tests/dbn_uploader_tests.rs

@@ -229,7 +229,8 @@ async fn test_upload_generates_correct_minio_key() {
     let filename = PathBuf::from("ES.FUT_ohlcv-1m_2024-01-02.dbn");
     let prefix = "training-data/";
 
-    let key = DbnUploader::generate_upload_key(&filename, prefix);
+    let key = DbnUploader::generate_upload_key(&filename, prefix)
+        .expect("Failed to generate upload key");
 
     assert_eq!(
         key, "training-data/ES.FUT_ohlcv-1m_2024-01-02.dbn.gz",