chore: Major codebase cleanup - remove deprecated files and organize structure
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
This commit is contained in:
91
scripts/python/data/concat_zn_existing.py
Executable file
91
scripts/python/data/concat_zn_existing.py
Executable file
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Concatenate existing ZN.FUT DBN files into a single 90-day file.
|
||||
Workaround for W12-05 symbology issues.
|
||||
"""
|
||||
|
||||
import os
|
||||
import glob
|
||||
import databento as db
|
||||
|
||||
SOURCE_DIR = "/home/jgrusewski/Work/foxhunt/test_data/real/databento/ml_training"
|
||||
OUTPUT_FILE = "/home/jgrusewski/Work/foxhunt/test_data/ZN_FUT_90d.dbn"
|
||||
LOG_FILE = "/tmp/zn_fut_download_log.txt"
|
||||
|
||||
def log(message):
|
||||
"""Write to both console and log file."""
|
||||
print(message)
|
||||
with open(LOG_FILE, "a") as f:
|
||||
f.write(message + "\n")
|
||||
|
||||
def main():
|
||||
log("\n" + "=" * 80)
|
||||
log("Agent W12-05: Concatenate Existing ZN.FUT Data")
|
||||
log("=" * 80)
|
||||
|
||||
# Find all ZN files
|
||||
pattern = os.path.join(SOURCE_DIR, "ZN.FUT_ohlcv-1m_*.dbn")
|
||||
files = sorted(glob.glob(pattern))
|
||||
|
||||
log(f"Found {len(files)} ZN.FUT files")
|
||||
log(f"Date range: {os.path.basename(files[0])} to {os.path.basename(files[-1])}")
|
||||
log("")
|
||||
|
||||
# Concatenate files
|
||||
log("📦 Concatenating files...")
|
||||
total_size = 0
|
||||
total_bars = 0
|
||||
|
||||
with open(OUTPUT_FILE, 'wb') as outf:
|
||||
for i, file_path in enumerate(files):
|
||||
with open(file_path, 'rb') as inf:
|
||||
data = inf.read()
|
||||
outf.write(data)
|
||||
total_size += len(data)
|
||||
|
||||
# Count bars
|
||||
try:
|
||||
store = db.DBNStore.from_file(file_path)
|
||||
df = store.to_df()
|
||||
bars = len(df)
|
||||
total_bars += bars
|
||||
log(f" [{i+1}/{len(files)}] {os.path.basename(file_path)}: {bars:,} bars")
|
||||
except:
|
||||
log(f" [{i+1}/{len(files)}] {os.path.basename(file_path)}: ? bars")
|
||||
|
||||
file_size_mb = total_size / (1024 * 1024)
|
||||
|
||||
log("")
|
||||
log("=" * 80)
|
||||
log("✅ ZN.FUT CONCATENATION COMPLETE")
|
||||
log("=" * 80)
|
||||
log(f"Output: {OUTPUT_FILE}")
|
||||
log(f"Files: {len(files)} days (90-day dataset)")
|
||||
log(f"Size: {file_size_mb:.2f} MB")
|
||||
log(f"Bars: {total_bars:,}")
|
||||
log("")
|
||||
|
||||
# Cost estimation
|
||||
estimated_cost = total_bars * 0.0035 / 1000
|
||||
log(f"Original Cost (estimate): ${estimated_cost:.2f}")
|
||||
log("")
|
||||
|
||||
log("⚠️ NOTE: Could not download full 180-day dataset due to Databento")
|
||||
log(" symbology issues with ZN.FUT. Using existing 90-day dataset")
|
||||
log(" (2024-01-02 to 2024-05-06) instead.")
|
||||
log("")
|
||||
|
||||
# Write completion flag
|
||||
with open("/tmp/w12_05_complete.flag", "w") as f:
|
||||
f.write(f"ZN.FUT concatenation complete (90-day dataset)\n")
|
||||
f.write(f"File: {OUTPUT_FILE}\n")
|
||||
f.write(f"Size: {file_size_mb:.2f} MB\n")
|
||||
f.write(f"Bars: {total_bars:,}\n")
|
||||
f.write(f"Days: {len(files)}\n")
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Append to existing log
|
||||
log("\n\n")
|
||||
exit(main())
|
||||
Reference in New Issue
Block a user