chore: Major codebase cleanup - remove deprecated files and organize structure
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
This commit is contained in:
jgrusewski
91
scripts/python/utils/check_registry_creds_v2.py
Executable file
91
scripts/python/utils/check_registry_creds_v2.py
Executable file
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load RunPod API key
|
||||
env_path = '.env.runpod'
|
||||
load_dotenv(env_path)
|
||||
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
|
||||
|
||||
if not RUNPOD_API_KEY:
|
||||
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
||||
exit(1)
|
||||
|
||||
# Try alternative field names
|
||||
queries = [
|
||||
# Attempt 1: containerRegistryAuths
|
||||
"""
|
||||
query {
|
||||
myself {
|
||||
containerRegistryAuths {
|
||||
id
|
||||
name
|
||||
username
|
||||
createdAt
|
||||
}
|
||||
}
|
||||
}
|
||||
""",
|
||||
# Attempt 2: savedRegistryAuths
|
||||
"""
|
||||
query {
|
||||
myself {
|
||||
savedRegistryAuths {
|
||||
id
|
||||
name
|
||||
username
|
||||
createdAt
|
||||
}
|
||||
}
|
||||
}
|
||||
""",
|
||||
# Attempt 3: Get user info to see available fields
|
||||
"""
|
||||
query {
|
||||
myself {
|
||||
id
|
||||
email
|
||||
}
|
||||
}
|
||||
"""
|
||||
]
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
for i, query in enumerate(queries, 1):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Attempt {i}:")
|
||||
print(f"{'='*60}")
|
||||
|
||||
response = requests.post(
|
||||
"https://api.runpod.io/graphql",
|
||||
json={"query": query},
|
||||
headers=headers
|
||||
)
|
||||
|
||||
result = response.json()
|
||||
print("Response:", result)
|
||||
|
||||
if 'data' in result and 'errors' not in result:
|
||||
print("\n✅ Query successful!")
|
||||
if i < 3: # Registry auth queries
|
||||
auth_field = 'containerRegistryAuths' if i == 1 else 'savedRegistryAuths'
|
||||
if result['data']['myself'] and auth_field in result['data']['myself']:
|
||||
creds = result['data']['myself'][auth_field]
|
||||
if creds:
|
||||
print(f"\n✅ Found credentials ({len(creds)}):")
|
||||
for cred in creds:
|
||||
print(f" ID: {cred['id']}")
|
||||
print(f" Name: {cred['name']}")
|
||||
print(f" Username: {cred['username']}")
|
||||
print(f" Created: {cred['createdAt']}")
|
||||
print()
|
||||
else:
|
||||
print("\n⚠️ No credentials configured")
|
||||
break
|
||||
else:
|
||||
print(f"❌ Failed: {result.get('errors', 'Unknown error')}")
|
||||
87
scripts/python/utils/check_registry_creds_v3.py
Executable file
87
scripts/python/utils/check_registry_creds_v3.py
Executable file
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import requests
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load RunPod API key
|
||||
env_path = '.env.runpod'
|
||||
load_dotenv(env_path)
|
||||
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
|
||||
|
||||
if not RUNPOD_API_KEY:
|
||||
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
||||
exit(1)
|
||||
|
||||
# GraphQL query with correct field name
|
||||
query = """
|
||||
query {
|
||||
myself {
|
||||
containerRegistryCreds {
|
||||
id
|
||||
name
|
||||
username
|
||||
createdAt
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
||||
}
|
||||
|
||||
response = requests.post(
|
||||
"https://api.runpod.io/graphql",
|
||||
json={"query": query},
|
||||
headers=headers
|
||||
)
|
||||
|
||||
result = response.json()
|
||||
print("Full API Response:")
|
||||
print("=" * 60)
|
||||
print(result)
|
||||
print("=" * 60)
|
||||
|
||||
if 'data' in result and result['data']['myself']:
|
||||
creds = result['data']['myself'].get('containerRegistryCreds', [])
|
||||
if creds:
|
||||
print("\n✅ Found Docker Hub credentials:")
|
||||
for cred in creds:
|
||||
print(f"\n Credential #{creds.index(cred) + 1}:")
|
||||
print(f" ID: {cred['id']}")
|
||||
print(f" Name: {cred['name']}")
|
||||
print(f" Username: {cred['username']}")
|
||||
print(f" Created: {cred['createdAt']}")
|
||||
|
||||
# Check .env.runpod
|
||||
print("\n" + "=" * 60)
|
||||
print("Checking .env.runpod configuration:")
|
||||
print("=" * 60)
|
||||
configured_id = os.getenv('RUNPOD_CONTAINER_REGISTRY_AUTH_ID')
|
||||
if configured_id:
|
||||
print(f"✅ RUNPOD_CONTAINER_REGISTRY_AUTH_ID is set to: {configured_id}")
|
||||
|
||||
# Check if it matches any credential
|
||||
matching = [c for c in creds if c['id'] == configured_id]
|
||||
if matching:
|
||||
print(f"✅ Credential ID matches: {matching[0]['name']}")
|
||||
else:
|
||||
print(f"⚠️ WARNING: Credential ID does not match any RunPod credentials!")
|
||||
print(f" Available IDs:")
|
||||
for cred in creds:
|
||||
print(f" - {cred['id']} ({cred['name']})")
|
||||
else:
|
||||
print("⚠️ RUNPOD_CONTAINER_REGISTRY_AUTH_ID is NOT set in .env.runpod")
|
||||
print(f" Set it to one of these IDs:")
|
||||
for cred in creds:
|
||||
print(f" - {cred['id']} ({cred['name']})")
|
||||
else:
|
||||
print("\n⚠️ No Docker Hub credentials configured in RunPod")
|
||||
print("You need to create one via RunPod console or API")
|
||||
print("\nManual steps:")
|
||||
print("1. Go to: https://www.runpod.io/console/user/settings")
|
||||
print("2. Click 'Container Registry Credentials' tab")
|
||||
print("3. Add credentials for Docker Hub (jgrusewski)")
|
||||
else:
|
||||
print("\n❌ Error querying credentials:", result)
|
||||
92
scripts/python/utils/check_subscription.py
Normal file
92
scripts/python/utils/check_subscription.py
Normal file
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Check Databento subscription and data availability.
|
||||
"""
|
||||
import os
|
||||
import databento as db
|
||||
import sys
|
||||
|
||||
def main():
|
||||
api_key = os.environ.get("DATABENTO_API_KEY")
|
||||
if not api_key:
|
||||
print("ERROR: DATABENTO_API_KEY environment variable not set")
|
||||
sys.exit(1)
|
||||
|
||||
client = db.Historical(api_key)
|
||||
|
||||
print("=" * 70)
|
||||
print("Databento Subscription Check")
|
||||
print("=" * 70)
|
||||
|
||||
# Get billing information
|
||||
try:
|
||||
print("\n1. Checking Account Status...")
|
||||
# Note: There's no direct API for billing, but we can try to get cost for a known symbol
|
||||
# Let's try ES (S&P 500 E-mini) which is very common
|
||||
|
||||
test_cases = [
|
||||
("XNAS.ITCH", "AAPL", "trades", "Nasdaq stocks"),
|
||||
("GLBX.MDP3", "ES", "ohlcv-1m", "CME E-mini S&P 500"),
|
||||
("GLBX.MDP3", "ES.FUT", "ohlcv-1m", "CME E-mini S&P 500 (FUT)"),
|
||||
("GLBX.MDP3", "ESH24", "ohlcv-1m", "CME E-mini S&P 500 (Mar 2024)"),
|
||||
]
|
||||
|
||||
print("\n2. Testing Data Access...\n")
|
||||
|
||||
for dataset, symbol, schema, description in test_cases:
|
||||
print(f"Testing: {description}")
|
||||
print(f" Dataset: {dataset}, Symbol: {symbol}, Schema: {schema}")
|
||||
|
||||
try:
|
||||
cost = client.metadata.get_cost(
|
||||
dataset=dataset,
|
||||
symbols=[symbol],
|
||||
schema=schema,
|
||||
start="2024-01-02",
|
||||
end="2024-01-05"
|
||||
)
|
||||
print(f" ✅ ACCESS GRANTED - Cost: ${cost:.4f} (3 days)")
|
||||
except Exception as e:
|
||||
error_msg = str(e)
|
||||
if "401" in error_msg or "unauthorized" in error_msg.lower():
|
||||
print(f" ❌ UNAUTHORIZED - Need subscription")
|
||||
elif "404" in error_msg or "not found" in error_msg.lower():
|
||||
print(f" ❌ NOT FOUND - Symbol doesn't exist")
|
||||
elif "symbology" in error_msg.lower():
|
||||
print(f" ❌ SYMBOLOGY ERROR - Symbol can't be resolved")
|
||||
else:
|
||||
print(f" ❌ ERROR: {error_msg[:80]}")
|
||||
print()
|
||||
|
||||
print("=" * 70)
|
||||
print("Diagnosis")
|
||||
print("=" * 70)
|
||||
print("""
|
||||
Based on the results above:
|
||||
|
||||
1. If ALL tests show UNAUTHORIZED:
|
||||
→ Your subscription doesn't include historical data access
|
||||
→ You may only have live data access
|
||||
|
||||
2. If ONLY GLBX.MDP3 (CME) tests fail:
|
||||
→ Your subscription doesn't include CME futures data
|
||||
→ CME data requires a separate subscription tier
|
||||
|
||||
3. If SYMBOLOGY ERROR appears:
|
||||
→ The symbol format is incorrect for that dataset
|
||||
→ Try checking Databento's symbology guide
|
||||
|
||||
4. If some symbols work:
|
||||
→ Your subscription is working, but specific instruments need adjustment
|
||||
|
||||
Recommendations:
|
||||
- Check your Databento account subscription at: https://databento.com/account
|
||||
- For CME futures data (6E, ES, etc.), verify you have GLBX.MDP3 access
|
||||
- Contact Databento support if you believe you should have access
|
||||
""")
|
||||
|
||||
except Exception as e:
|
||||
print(f"\nError checking subscription: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user