chore: Second cleanup wave - organize root directory
- Archive: 85 agent .txt files → docs/archive/agents/legacy_txt/ - Scripts: Move 110 shell scripts → scripts/ (keep deploy.sh in root) - Models: Move 18 .safetensors → ml/models/checkpoints/training_artifacts/ - Delete: 34 directories (~33GB freed) - target/, coverage_*, test artifacts - Build: Clean 14 build artifacts (.rlib, .o, .pid, binaries) - Tests: Move 14 .rs files → tests/standalone/ - SQL: Move 5 files → sql/ (keep init-db*.sql for Docker) - Wave 153: Archive to docs/archive/historical/wave153/ - Docs: Archive 9 markdown files to wave_d/reports/ and historical/ Total impact: ~34GB freed (both waves), root directory cleaned from 583 to ~40 essential files Directory count reduced from 65 to 31 (52% reduction) All historical data preserved in organized archive structure
This commit is contained in:
jgrusewski
164
sql/PAPER_TRADING_DIAGNOSTIC_QUERIES.sql
Normal file
164
sql/PAPER_TRADING_DIAGNOSTIC_QUERIES.sql
Normal file
@@ -0,0 +1,164 @@
|
||||
-- ============================================================================
|
||||
-- PAPER TRADING DIAGNOSTIC QUERIES
|
||||
-- Agent 131 - 2025-10-14
|
||||
-- ============================================================================
|
||||
|
||||
-- PROBLEM VERIFICATION
|
||||
-- ----------------------------------------------------------------------------
|
||||
|
||||
-- 1. Count total predictions (Expected: 3000)
|
||||
SELECT COUNT(*) as total_predictions FROM ensemble_predictions;
|
||||
|
||||
-- 2. Count executed orders (Expected: 0 - THIS IS THE BUG!)
|
||||
SELECT COUNT(*) as total_orders
|
||||
FROM orders
|
||||
WHERE account_id LIKE '%paper%';
|
||||
|
||||
-- 3. Check prediction linkage (Expected: 0 - no predictions linked to orders)
|
||||
SELECT COUNT(*) as linked_predictions
|
||||
FROM ensemble_predictions
|
||||
WHERE order_id IS NOT NULL;
|
||||
|
||||
-- 4. Conversion rate calculation (Expected: 0%)
|
||||
SELECT
|
||||
COUNT(*) as total_predictions,
|
||||
SUM(CASE WHEN order_id IS NOT NULL THEN 1 ELSE 0 END) as executed_predictions,
|
||||
ROUND(100.0 * SUM(CASE WHEN order_id IS NOT NULL THEN 1 ELSE 0 END) / COUNT(*), 2) as conversion_rate_percent
|
||||
FROM ensemble_predictions
|
||||
WHERE ensemble_action IN ('BUY', 'SELL');
|
||||
|
||||
|
||||
-- PREDICTION ANALYSIS
|
||||
-- ----------------------------------------------------------------------------
|
||||
|
||||
-- 5. Prediction breakdown by action
|
||||
SELECT
|
||||
ensemble_action,
|
||||
COUNT(*) as count,
|
||||
ROUND(100.0 * COUNT(*) / SUM(COUNT(*)) OVER (), 2) as percentage,
|
||||
ROUND(AVG(ensemble_confidence)::numeric, 4) as avg_confidence,
|
||||
ROUND(AVG(disagreement_rate)::numeric, 4) as avg_disagreement
|
||||
FROM ensemble_predictions
|
||||
GROUP BY ensemble_action
|
||||
ORDER BY count DESC;
|
||||
|
||||
-- 6. Symbol distribution (Expected: Only TEST_SYM - THIS IS WRONG!)
|
||||
SELECT
|
||||
symbol,
|
||||
COUNT(*) as count,
|
||||
MIN(timestamp) as first_prediction,
|
||||
MAX(timestamp) as last_prediction
|
||||
FROM ensemble_predictions
|
||||
GROUP BY symbol
|
||||
ORDER BY count DESC;
|
||||
|
||||
-- 7. High-confidence predictions (>60%) that SHOULD be executed
|
||||
SELECT
|
||||
COUNT(*) as high_confidence_predictions,
|
||||
ROUND(100.0 * COUNT(*) / (SELECT COUNT(*) FROM ensemble_predictions), 2) as percentage
|
||||
FROM ensemble_predictions
|
||||
WHERE ensemble_confidence >= 0.60
|
||||
AND ensemble_action IN ('BUY', 'SELL');
|
||||
|
||||
-- 8. High-confidence predictions by symbol (should be real symbols!)
|
||||
SELECT
|
||||
symbol,
|
||||
ensemble_action,
|
||||
COUNT(*) as count,
|
||||
AVG(ensemble_confidence)::numeric(5,2) as avg_confidence
|
||||
FROM ensemble_predictions
|
||||
WHERE ensemble_confidence >= 0.60
|
||||
AND ensemble_action IN ('BUY', 'SELL')
|
||||
GROUP BY symbol, ensemble_action
|
||||
ORDER BY count DESC;
|
||||
|
||||
|
||||
-- MODEL VOTE ANALYSIS
|
||||
-- ----------------------------------------------------------------------------
|
||||
|
||||
-- 9. Individual model participation (Expected: All NULL - models not trained)
|
||||
SELECT
|
||||
COUNT(*) as total_predictions,
|
||||
SUM(CASE WHEN dqn_signal IS NOT NULL THEN 1 ELSE 0 END) as dqn_votes,
|
||||
SUM(CASE WHEN ppo_signal IS NOT NULL THEN 1 ELSE 0 END) as ppo_votes,
|
||||
SUM(CASE WHEN mamba2_signal IS NOT NULL THEN 1 ELSE 0 END) as mamba2_votes,
|
||||
SUM(CASE WHEN tft_signal IS NOT NULL THEN 1 ELSE 0 END) as tft_votes
|
||||
FROM ensemble_predictions;
|
||||
|
||||
-- 10. High disagreement events (>50% disagreement)
|
||||
SELECT
|
||||
COUNT(*) as high_disagreement_count,
|
||||
ROUND(100.0 * COUNT(*) / (SELECT COUNT(*) FROM ensemble_predictions), 2) as percentage,
|
||||
AVG(disagreement_rate)::numeric(5,2) as avg_disagreement
|
||||
FROM ensemble_predictions
|
||||
WHERE disagreement_rate >= 0.50;
|
||||
|
||||
|
||||
-- TEMPORAL ANALYSIS
|
||||
-- ----------------------------------------------------------------------------
|
||||
|
||||
-- 11. Prediction timeline (when predictions were generated)
|
||||
SELECT
|
||||
DATE_TRUNC('minute', timestamp) as minute,
|
||||
COUNT(*) as predictions_per_minute
|
||||
FROM ensemble_predictions
|
||||
GROUP BY minute
|
||||
ORDER BY minute DESC
|
||||
LIMIT 10;
|
||||
|
||||
-- 12. Time since last prediction (Expected: >1 hour - system stopped)
|
||||
SELECT
|
||||
MAX(timestamp) as last_prediction_time,
|
||||
NOW() - MAX(timestamp) as time_since_last_prediction
|
||||
FROM ensemble_predictions;
|
||||
|
||||
|
||||
-- MISSING CONSUMER VALIDATION
|
||||
-- ----------------------------------------------------------------------------
|
||||
|
||||
-- 13. Predictions that SHOULD be executed (but aren't due to missing consumer)
|
||||
SELECT
|
||||
id,
|
||||
timestamp,
|
||||
symbol,
|
||||
ensemble_action,
|
||||
ensemble_signal,
|
||||
ensemble_confidence,
|
||||
order_id
|
||||
FROM ensemble_predictions
|
||||
WHERE order_id IS NULL -- Not yet executed
|
||||
AND ensemble_confidence >= 0.60 -- High confidence
|
||||
AND ensemble_action IN ('BUY', 'SELL') -- Actionable
|
||||
AND timestamp > NOW() - INTERVAL '5 minutes' -- Recent
|
||||
ORDER BY ensemble_confidence DESC
|
||||
LIMIT 20;
|
||||
|
||||
-- 14. Count of executable predictions (if consumer existed)
|
||||
SELECT
|
||||
COUNT(*) as executable_predictions,
|
||||
ROUND(100.0 * COUNT(*) / (SELECT COUNT(*) FROM ensemble_predictions), 2) as executable_percentage
|
||||
FROM ensemble_predictions
|
||||
WHERE order_id IS NULL
|
||||
AND ensemble_confidence >= 0.60
|
||||
AND ensemble_action IN ('BUY', 'SELL');
|
||||
|
||||
|
||||
-- EXPECTED RESULTS AFTER FIX
|
||||
-- ----------------------------------------------------------------------------
|
||||
|
||||
-- After implementing PaperTradingExecutor:
|
||||
--
|
||||
-- Query 2 (total_orders): >1500 (not 0!)
|
||||
-- Query 3 (linked_predictions): >1500 (not 0!)
|
||||
-- Query 4 (conversion_rate_percent): >50% (not 0%)
|
||||
-- Query 6 (symbol): ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT (not TEST_SYM!)
|
||||
-- Query 14 (executable_predictions): Decreasing over time as consumer executes
|
||||
|
||||
|
||||
-- ============================================================================
|
||||
-- RUN ALL DIAGNOSTICS
|
||||
-- ============================================================================
|
||||
|
||||
-- Usage:
|
||||
-- psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -f PAPER_TRADING_DIAGNOSTIC_QUERIES.sql
|
||||
|
||||
94
sql/test_pg_performance.sql
Normal file
94
sql/test_pg_performance.sql
Normal file
@@ -0,0 +1,94 @@
|
||||
-- PostgreSQL Performance Test Script
|
||||
-- Tests connection pool performance and throughput
|
||||
|
||||
\timing on
|
||||
|
||||
-- Test 1: Basic query performance
|
||||
SELECT 'Test 1: Basic query performance' as test;
|
||||
SELECT COUNT(*) FROM config_settings;
|
||||
|
||||
-- Test 2: Temporary table creation and inserts
|
||||
SELECT 'Test 2: Creating temporary test table' as test;
|
||||
CREATE TEMP TABLE perf_test (
|
||||
id SERIAL PRIMARY KEY,
|
||||
trade_id VARCHAR(50),
|
||||
symbol VARCHAR(10),
|
||||
price DECIMAL(18,8),
|
||||
quantity DECIMAL(18,8),
|
||||
timestamp TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Test 3: Bulk insert performance (1000 records)
|
||||
SELECT 'Test 3: Bulk insert 1000 records' as test;
|
||||
INSERT INTO perf_test (trade_id, symbol, price, quantity)
|
||||
SELECT
|
||||
'TRADE_' || generate_series || '_' || extract(epoch from now()),
|
||||
CASE (random() * 5)::int
|
||||
WHEN 0 THEN 'BTC/USD'
|
||||
WHEN 1 THEN 'ETH/USD'
|
||||
WHEN 2 THEN 'AAPL'
|
||||
WHEN 3 THEN 'GOOGL'
|
||||
ELSE 'MSFT'
|
||||
END,
|
||||
(random() * 1000)::decimal(18,8),
|
||||
(random() * 100)::decimal(18,8)
|
||||
FROM generate_series(1, 1000);
|
||||
|
||||
-- Test 4: Query performance on test data
|
||||
SELECT 'Test 4: Query performance (aggregation)' as test;
|
||||
SELECT symbol, COUNT(*) as trade_count, AVG(price) as avg_price
|
||||
FROM perf_test
|
||||
GROUP BY symbol;
|
||||
|
||||
-- Test 5: Index creation and query optimization
|
||||
SELECT 'Test 5: Creating index' as test;
|
||||
CREATE INDEX idx_perf_test_symbol_timestamp ON perf_test(symbol, timestamp DESC);
|
||||
|
||||
-- Test 6: Indexed query performance
|
||||
SELECT 'Test 6: Indexed query performance' as test;
|
||||
SELECT * FROM perf_test WHERE symbol = 'BTC/USD' ORDER BY timestamp DESC LIMIT 100;
|
||||
|
||||
-- Test 7: Transaction performance (1000 individual inserts)
|
||||
SELECT 'Test 7: Transaction performance (1000 individual inserts)' as test;
|
||||
BEGIN;
|
||||
DO $$
|
||||
DECLARE
|
||||
i INT;
|
||||
BEGIN
|
||||
FOR i IN 1..1000 LOOP
|
||||
INSERT INTO perf_test (trade_id, symbol, price, quantity)
|
||||
VALUES (
|
||||
'TRADE_TX_' || i || '_' || extract(epoch from now()),
|
||||
CASE (random() * 5)::int
|
||||
WHEN 0 THEN 'BTC/USD'
|
||||
WHEN 1 THEN 'ETH/USD'
|
||||
WHEN 2 THEN 'AAPL'
|
||||
WHEN 3 THEN 'GOOGL'
|
||||
ELSE 'MSFT'
|
||||
END,
|
||||
(random() * 1000)::decimal(18,8),
|
||||
(random() * 100)::decimal(18,8)
|
||||
);
|
||||
END LOOP;
|
||||
END $$;
|
||||
COMMIT;
|
||||
|
||||
-- Test 8: Final statistics
|
||||
SELECT 'Test 8: Final statistics' as test;
|
||||
SELECT COUNT(*) as total_records FROM perf_test;
|
||||
|
||||
-- Test 9: Connection and pool statistics
|
||||
SELECT 'Test 9: Database statistics' as test;
|
||||
SELECT
|
||||
numbackends as active_connections,
|
||||
xact_commit as committed_transactions,
|
||||
xact_rollback as rolled_back_transactions,
|
||||
blks_read as blocks_read,
|
||||
blks_hit as blocks_hit,
|
||||
tup_returned as tuples_returned,
|
||||
tup_fetched as tuples_fetched,
|
||||
tup_inserted as tuples_inserted
|
||||
FROM pg_stat_database
|
||||
WHERE datname = 'foxhunt';
|
||||
|
||||
SELECT 'Performance test completed!' as status;
|
||||
39
sql/test_stop_loss_debug.sql
Normal file
39
sql/test_stop_loss_debug.sql
Normal file
@@ -0,0 +1,39 @@
|
||||
-- Debug script for stop-loss integration test
|
||||
|
||||
-- Check if regime state exists
|
||||
SELECT 'Regime State:' as step;
|
||||
SELECT symbol, regime, confidence FROM regime_states WHERE symbol = 'NQ.FUT' ORDER BY event_timestamp DESC LIMIT 1;
|
||||
|
||||
-- Check if market data exists
|
||||
SELECT 'Market Data Count:' as step;
|
||||
SELECT COUNT(*) as bar_count FROM prices WHERE symbol = 'NQ.FUT';
|
||||
|
||||
-- Check market data values
|
||||
SELECT 'Market Data Sample:' as step;
|
||||
SELECT
|
||||
high::FLOAT8 / 100.0 as high,
|
||||
low::FLOAT8 / 100.0 as low,
|
||||
close::FLOAT8 / 100.0 as close
|
||||
FROM prices
|
||||
WHERE symbol = 'NQ.FUT'
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT 5;
|
||||
|
||||
-- Test ATR calculation manually
|
||||
SELECT 'Manual ATR Check:' as step;
|
||||
WITH bars AS (
|
||||
SELECT
|
||||
high::FLOAT8 / 100.0 as high,
|
||||
low::FLOAT8 / 100.0 as low,
|
||||
close::FLOAT8 / 100.0 as close,
|
||||
timestamp
|
||||
FROM prices
|
||||
WHERE symbol = 'NQ.FUT'
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT 20
|
||||
)
|
||||
SELECT
|
||||
AVG(high - low) as avg_range,
|
||||
MAX(high - low) as max_range,
|
||||
MIN(high - low) as min_range
|
||||
FROM bars;
|
||||
59
sql/trading_workload.sql
Normal file
59
sql/trading_workload.sql
Normal file
@@ -0,0 +1,59 @@
|
||||
-- Trading workload SQL for pgbench
|
||||
-- 40% INSERT, 30% SELECT, 20% UPDATE, 10% Complex queries
|
||||
|
||||
\set account_id 'test_account_' :client_id
|
||||
\set venue 'test_venue'
|
||||
\set symbol random(1, 3)
|
||||
|
||||
-- Map random number to symbol
|
||||
\if :symbol = 1
|
||||
\set symbol_str 'BTC/USD'
|
||||
\elif :symbol = 2
|
||||
\set symbol_str 'ETH/USD'
|
||||
\else
|
||||
\set symbol_str 'SOL/USD'
|
||||
\endif
|
||||
|
||||
\set operation random(1, 10)
|
||||
|
||||
-- 40% INSERT (operations 1-4)
|
||||
\if :operation <= 4
|
||||
INSERT INTO orders (account_id, symbol, side, order_type, quantity, limit_price, venue, created_at, updated_at)
|
||||
VALUES (:'account_id', :'symbol_str', 'buy', 'limit', 100000000, 5000000000000, :'venue',
|
||||
EXTRACT(EPOCH FROM NOW()) * 1000000000, EXTRACT(EPOCH FROM NOW()) * 1000000000);
|
||||
|
||||
-- 30% SELECT (operations 5-7)
|
||||
\elif :operation <= 7
|
||||
SELECT id, status, quantity, filled_quantity
|
||||
FROM orders
|
||||
WHERE symbol = :'symbol_str'
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 10;
|
||||
|
||||
-- 20% UPDATE (operations 8-9)
|
||||
\elif :operation <= 9
|
||||
UPDATE orders
|
||||
SET status = 'partially_filled',
|
||||
filled_quantity = filled_quantity + 10000000,
|
||||
updated_at = EXTRACT(EPOCH FROM NOW()) * 1000000000
|
||||
WHERE id = (
|
||||
SELECT id
|
||||
FROM orders
|
||||
WHERE status = 'pending'
|
||||
AND symbol = :'symbol_str'
|
||||
LIMIT 1
|
||||
);
|
||||
|
||||
-- 10% Complex query (operation 10)
|
||||
\else
|
||||
SELECT symbol,
|
||||
COUNT(*) as order_count,
|
||||
SUM(quantity) as total_quantity,
|
||||
AVG(limit_price) as avg_price,
|
||||
COUNT(DISTINCT account_id) as unique_accounts
|
||||
FROM orders
|
||||
WHERE created_at > EXTRACT(EPOCH FROM (NOW() - INTERVAL '1 hour')) * 1000000000
|
||||
GROUP BY symbol
|
||||
ORDER BY order_count DESC;
|
||||
|
||||
\endif
|
||||
Reference in New Issue
Block a user