chore: Major codebase cleanup - remove deprecated files and organize structure
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
This commit is contained in:
64
scripts/python/data/list_datasets.py
Normal file
64
scripts/python/data/list_datasets.py
Normal file
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
List available datasets and check symbology rules.
|
||||
"""
|
||||
import os
|
||||
import databento as db
|
||||
import sys
|
||||
|
||||
def main():
|
||||
# Check for API key
|
||||
api_key = os.environ.get("DATABENTO_API_KEY")
|
||||
if not api_key:
|
||||
print("ERROR: DATABENTO_API_KEY environment variable not set")
|
||||
sys.exit(1)
|
||||
|
||||
# Create client
|
||||
client = db.Historical(api_key)
|
||||
|
||||
print("=" * 70)
|
||||
print("Available Datasets")
|
||||
print("=" * 70)
|
||||
|
||||
# List datasets
|
||||
try:
|
||||
datasets = client.metadata.list_datasets()
|
||||
print(f"\nFound {len(datasets)} datasets:\n")
|
||||
|
||||
for dataset in datasets:
|
||||
print(f"Dataset: {dataset}")
|
||||
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("CME Group Datasets (for futures)")
|
||||
print("=" * 70)
|
||||
|
||||
cme_datasets = [d for d in datasets if 'CME' in str(d) or 'GLBX' in str(d)]
|
||||
if cme_datasets:
|
||||
for ds in cme_datasets:
|
||||
print(f" - {ds}")
|
||||
else:
|
||||
print(" (filtering by name, showing all)")
|
||||
for ds in datasets:
|
||||
print(f" - {ds}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error listing datasets: {e}")
|
||||
|
||||
# Let's also try to get symbology info
|
||||
print()
|
||||
print("=" * 70)
|
||||
print("Dataset Details for GLBX.MDP3")
|
||||
print("=" * 70)
|
||||
|
||||
try:
|
||||
# Get dataset info
|
||||
dataset_info = client.metadata.list_schemas("GLBX.MDP3")
|
||||
print(f"\nAvailable schemas for GLBX.MDP3:")
|
||||
for schema in dataset_info:
|
||||
print(f" - {schema}")
|
||||
except Exception as e:
|
||||
print(f"Error getting dataset info: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user