433af5c25d
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
70 lines
2.0 KiB
Python
70 lines
2.0 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Inspect NQ.FUT Parquet file schema and validate data.
|
|
|
|
Agent: W12-08
|
|
"""
|
|
|
|
import pandas as pd
|
|
import pyarrow.parquet as pq
|
|
|
|
PARQUET_FILE = "test_data/NQ_FUT_180d.parquet"
|
|
|
|
def main():
|
|
"""Inspect Parquet file."""
|
|
print("=" * 80)
|
|
print("NQ.FUT Parquet Inspection")
|
|
print("=" * 80)
|
|
print()
|
|
|
|
# Read Parquet metadata
|
|
parquet_file = pq.ParquetFile(PARQUET_FILE)
|
|
print("📊 Parquet Metadata:")
|
|
print(f" Format version: {parquet_file.metadata.format_version}")
|
|
print(f" Num rows: {parquet_file.metadata.num_rows:,}")
|
|
print(f" Num row groups: {parquet_file.metadata.num_row_groups}")
|
|
print(f" Serialized size: {parquet_file.metadata.serialized_size:,} bytes")
|
|
print()
|
|
|
|
# Schema
|
|
print("📋 Schema:")
|
|
for i, field in enumerate(parquet_file.schema):
|
|
print(f" {i}: {field.name} ({field.physical_type})")
|
|
print()
|
|
|
|
# Read data
|
|
df = pd.read_parquet(PARQUET_FILE)
|
|
|
|
print("📊 DataFrame Info:")
|
|
print(f" Shape: {df.shape}")
|
|
print(f" Columns: {list(df.columns)}")
|
|
print(f" Index: {df.index.name} ({df.index.dtype})")
|
|
print()
|
|
|
|
print("📈 OHLCV Statistics:")
|
|
print(f" Open: min=${df['open'].min():.2f}, max=${df['open'].max():.2f}")
|
|
print(f" High: min=${df['high'].min():.2f}, max=${df['high'].max():.2f}")
|
|
print(f" Low: min=${df['low'].min():.2f}, max=${df['low'].max():.2f}")
|
|
print(f" Close: min=${df['close'].min():.2f}, max=${df['close'].max():.2f}")
|
|
print(f" Volume: total={df['volume'].sum():,.0f}, avg={df['volume'].mean():.0f}")
|
|
print()
|
|
|
|
print("🕐 Timestamp Info:")
|
|
print(f" First: {df.index[0]}")
|
|
print(f" Last: {df.index[-1]}")
|
|
print(f" Total bars: {len(df):,}")
|
|
print()
|
|
|
|
# Sample rows
|
|
print("📋 First 5 rows:")
|
|
print(df.head(5)[['open', 'high', 'low', 'close', 'volume']])
|
|
print()
|
|
|
|
print("=" * 80)
|
|
print("✅ Parquet file validated successfully!")
|
|
print("=" * 80)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|