9c3d741a08
Move 17 library crates into crates/, CLI binary into bin/fxt, consolidate 10 test crates into testing/, split config crate from deployment config files. Root directory reduced from 38+ to ~17 directories. All Cargo.toml paths and build.rs proto refs updated. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
74 lines
2.2 KiB
Rust
74 lines
2.2 KiB
Rust
//! # Model Quantization for Ultra-Low Latency Inference
|
|
//!
|
|
//! This module implements INT8/INT4 quantization techniques for transformer models
|
|
//! to achieve maximum inference speed in HFT applications.
|
|
|
|
use candle_core::Device;
|
|
use candle_core::{Device, Result as CandleResult, Tensor};
|
|
use serde::{Deserialize, Serialize};
|
|
use tracing::{info, warn};
|
|
use tracing::{info, warn};
|
|
use tracing::{info, warn};
|
|
|
|
use super::*;
|
|
|
|
|
|
#[test]
|
|
fn test_quantization_config() {
|
|
let config = QuantizationConfig::default();
|
|
assert_eq!(config.bits, 8);
|
|
assert!(config.symmetric);
|
|
}
|
|
|
|
#[test]
|
|
fn test_int8_quantization() {
|
|
let device = Device::Cpu;
|
|
let config = QuantizationConfig::default();
|
|
let quantizer = QuantizedTransformer::new(config, device.clone());
|
|
|
|
// Create test tensor
|
|
let test_data = vec![1.0, 2.0, 3.0, 4.0, 5.0, -1.0, -2.0, -3.0];
|
|
let tensor = Tensor::from_vec(test_data, (2, 4), &device)?;
|
|
|
|
// Test quantization
|
|
let result = quantizer.quantize_tensor(&tensor);
|
|
assert!(result.is_ok());
|
|
|
|
let (quantized, scale, zero_point) = result?;
|
|
assert!(scale > 0.0);
|
|
|
|
// Test dequantization
|
|
let dequantized = quantizer.dequantize_tensor(&quantized, scale, zero_point);
|
|
assert!(dequantized.is_ok());
|
|
}
|
|
|
|
#[test]
|
|
fn test_int4_quantization() {
|
|
let device = Device::Cpu;
|
|
let config = QuantizationConfig {
|
|
bits: 4,
|
|
..Default::default()
|
|
};
|
|
let quantizer = QuantizedTransformer::new(config, device.clone());
|
|
|
|
let test_data = vec![1.0, 2.0, 3.0, 4.0];
|
|
let tensor = Tensor::from_vec(test_data, (2, 2), &device)?;
|
|
|
|
let result = quantizer.quantize_tensor(&tensor);
|
|
assert!(result.is_ok());
|
|
}
|
|
|
|
#[test]
|
|
fn test_quantized_matmul() {
|
|
let device = Device::Cpu;
|
|
let config = QuantizationConfig::default();
|
|
let quantizer = QuantizedTransformer::new(config, device.clone());
|
|
|
|
let a = Tensor::from_vec(vec![1.0, 2.0, 3.0, 4.0], (2, 2), &device)?;
|
|
let b = Tensor::from_vec(vec![5.0, 6.0, 7.0, 8.0], (2, 2), &device)?;
|
|
|
|
let result = quantizer.quantized_matmul(&a, &b);
|
|
assert!(result.is_ok());
|
|
}
|
|
}
|