8d89fe80ff
- Archive: 85 agent .txt files → docs/archive/agents/legacy_txt/ - Scripts: Move 110 shell scripts → scripts/ (keep deploy.sh in root) - Models: Move 18 .safetensors → ml/models/checkpoints/training_artifacts/ - Delete: 34 directories (~33GB freed) - target/, coverage_*, test artifacts - Build: Clean 14 build artifacts (.rlib, .o, .pid, binaries) - Tests: Move 14 .rs files → tests/standalone/ - SQL: Move 5 files → sql/ (keep init-db*.sql for Docker) - Wave 153: Archive to docs/archive/historical/wave153/ - Docs: Archive 9 markdown files to wave_d/reports/ and historical/ Total impact: ~34GB freed (both waves), root directory cleaned from 583 to ~40 essential files Directory count reduced from 65 to 31 (52% reduction) All historical data preserved in organized archive structure
363 lines
12 KiB
Bash
Executable File
363 lines
12 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Database Validation Script for Foxhunt HFT Trading System
|
|
# Validates PostgreSQL, Redis, SQLite, and InfluxDB connections and performance
|
|
|
|
set -e
|
|
|
|
echo "🚀 Foxhunt Database Layer Validation"
|
|
echo "===================================="
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Configuration
|
|
POSTGRES_URL="${DATABASE_URL:-postgresql://postgres:password@localhost:5432/foxhunt_test}"
|
|
REDIS_URL="${REDIS_URL:-localhost:6379}"
|
|
INFLUX_URL="${INFLUX_URL:-localhost:8086}"
|
|
SQLITE_DB="validation_test.db"
|
|
|
|
# Counters
|
|
TESTS_PASSED=0
|
|
TESTS_FAILED=0
|
|
TOTAL_TESTS=0
|
|
|
|
# Helper functions
|
|
log_info() {
|
|
echo -e "${BLUE}[INFO]${NC} $1"
|
|
}
|
|
|
|
log_success() {
|
|
echo -e "${GREEN}[PASS]${NC} $1"
|
|
((TESTS_PASSED++))
|
|
((TOTAL_TESTS++))
|
|
}
|
|
|
|
log_warning() {
|
|
echo -e "${YELLOW}[WARN]${NC} $1"
|
|
((TOTAL_TESTS++))
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}[FAIL]${NC} $1"
|
|
((TESTS_FAILED++))
|
|
((TOTAL_TESTS++))
|
|
}
|
|
|
|
# Test PostgreSQL
|
|
test_postgresql() {
|
|
echo ""
|
|
log_info "🔍 Testing PostgreSQL..."
|
|
|
|
# Check if PostgreSQL is accessible
|
|
if command -v psql >/dev/null 2>&1; then
|
|
# Test connection
|
|
if psql "$POSTGRES_URL" -c "SELECT 1;" >/dev/null 2>&1; then
|
|
log_success "PostgreSQL connection successful"
|
|
|
|
# Performance test - measure simple query latency
|
|
local start_time=$(date +%s%N)
|
|
for i in {1..100}; do
|
|
psql "$POSTGRES_URL" -t -c "SELECT 1;" >/dev/null 2>&1
|
|
done
|
|
local end_time=$(date +%s%N)
|
|
local total_time=$((end_time - start_time))
|
|
local avg_latency_ms=$((total_time / 100000000)) # Convert to ms
|
|
|
|
log_info "Average query latency: ${avg_latency_ms}ms"
|
|
|
|
if [ $avg_latency_ms -lt 10 ]; then
|
|
log_success "PostgreSQL performance meets requirements (<10ms)"
|
|
else
|
|
log_warning "PostgreSQL performance suboptimal (${avg_latency_ms}ms > 10ms)"
|
|
fi
|
|
|
|
# Test ACID transaction
|
|
psql "$POSTGRES_URL" >/dev/null 2>&1 << EOF
|
|
BEGIN;
|
|
CREATE TABLE IF NOT EXISTS acid_test_$$
|
|
(id SERIAL PRIMARY KEY, value INTEGER);
|
|
INSERT INTO acid_test_$$ (value) VALUES (100);
|
|
ROLLBACK;
|
|
EOF
|
|
|
|
# Verify rollback worked
|
|
local count=$(psql "$POSTGRES_URL" -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_name='acid_test_$$';" 2>/dev/null | tr -d ' ' || echo "0")
|
|
if [ "$count" = "0" ]; then
|
|
log_success "ACID transaction rollback test passed"
|
|
else
|
|
log_error "ACID transaction rollback test failed"
|
|
fi
|
|
|
|
else
|
|
log_error "PostgreSQL connection failed"
|
|
fi
|
|
else
|
|
log_warning "psql not available - skipping PostgreSQL tests"
|
|
fi
|
|
}
|
|
|
|
# Test Redis
|
|
test_redis() {
|
|
echo ""
|
|
log_info "🔍 Testing Redis..."
|
|
|
|
if command -v redis-cli >/dev/null 2>&1; then
|
|
# Test connection
|
|
if redis-cli -h "${REDIS_URL%:*}" -p "${REDIS_URL#*:}" ping >/dev/null 2>&1; then
|
|
log_success "Redis connection successful"
|
|
|
|
# Performance test
|
|
local start_time=$(date +%s%N)
|
|
for i in {1..100}; do
|
|
redis-cli -h "${REDIS_URL%:*}" -p "${REDIS_URL#*:}" set "test_key_$i" "test_value_$i" >/dev/null 2>&1
|
|
redis-cli -h "${REDIS_URL%:*}" -p "${REDIS_URL#*:}" get "test_key_$i" >/dev/null 2>&1
|
|
done
|
|
local end_time=$(date +%s%N)
|
|
local total_time=$((end_time - start_time))
|
|
local avg_latency_ms=$((total_time / 100000000))
|
|
|
|
log_info "Average operation latency: ${avg_latency_ms}ms"
|
|
|
|
if [ $avg_latency_ms -lt 5 ]; then
|
|
log_success "Redis performance meets requirements (<5ms)"
|
|
else
|
|
log_warning "Redis performance suboptimal (${avg_latency_ms}ms > 5ms)"
|
|
fi
|
|
|
|
# Test TTL functionality
|
|
redis-cli -h "${REDIS_URL%:*}" -p "${REDIS_URL#*:}" set ttl_test_key ttl_test_value >/dev/null 2>&1
|
|
redis-cli -h "${REDIS_URL%:*}" -p "${REDIS_URL#*:}" expire ttl_test_key 1 >/dev/null 2>&1
|
|
sleep 2
|
|
local exists=$(redis-cli -h "${REDIS_URL%:*}" -p "${REDIS_URL#*:}" exists ttl_test_key 2>/dev/null)
|
|
if [ "$exists" = "0" ]; then
|
|
log_success "Redis TTL functionality working"
|
|
else
|
|
log_error "Redis TTL functionality failed"
|
|
fi
|
|
|
|
# Cleanup test keys
|
|
for i in {1..100}; do
|
|
redis-cli -h "${REDIS_URL%:*}" -p "${REDIS_URL#*:}" del "test_key_$i" >/dev/null 2>&1
|
|
done
|
|
|
|
else
|
|
log_error "Redis connection failed"
|
|
fi
|
|
else
|
|
log_warning "redis-cli not available - skipping Redis tests"
|
|
fi
|
|
}
|
|
|
|
# Test SQLite
|
|
test_sqlite() {
|
|
echo ""
|
|
log_info "🔍 Testing SQLite..."
|
|
|
|
if command -v sqlite3 >/dev/null 2>&1; then
|
|
# Remove existing test database
|
|
rm -f "$SQLITE_DB"
|
|
|
|
# Test database creation and operations
|
|
if sqlite3 "$SQLITE_DB" "CREATE TABLE config_test (key TEXT PRIMARY KEY, value TEXT, updated_at INTEGER);" >/dev/null 2>&1; then
|
|
log_success "SQLite database creation successful"
|
|
|
|
# Performance test
|
|
local start_time=$(date +%s%N)
|
|
for i in {1..50}; do
|
|
sqlite3 "$SQLITE_DB" "INSERT OR REPLACE INTO config_test (key, value, updated_at) VALUES ('key_$i', 'value_$i', $i);" >/dev/null 2>&1
|
|
sqlite3 "$SQLITE_DB" "SELECT value FROM config_test WHERE key='key_$i';" >/dev/null 2>&1
|
|
done
|
|
local end_time=$(date +%s%N)
|
|
local total_time=$((end_time - start_time))
|
|
local avg_latency_ms=$((total_time / 50000000))
|
|
|
|
log_info "Average operation latency: ${avg_latency_ms}ms"
|
|
|
|
if [ $avg_latency_ms -lt 20 ]; then
|
|
log_success "SQLite performance acceptable (<20ms)"
|
|
else
|
|
log_warning "SQLite performance suboptimal (${avg_latency_ms}ms > 20ms)"
|
|
fi
|
|
|
|
# Test hot-reload simulation
|
|
sqlite3 "$SQLITE_DB" "INSERT OR REPLACE INTO config_test (key, value, updated_at) VALUES ('hot_reload_test', 'initial_value', 1);" >/dev/null 2>&1
|
|
sqlite3 "$SQLITE_DB" "UPDATE config_test SET value='updated_value', updated_at=2 WHERE key='hot_reload_test';" >/dev/null 2>&1
|
|
local updated_value=$(sqlite3 "$SQLITE_DB" "SELECT value FROM config_test WHERE key='hot_reload_test';" 2>/dev/null)
|
|
|
|
if [ "$updated_value" = "updated_value" ]; then
|
|
log_success "SQLite configuration hot-reload simulation successful"
|
|
else
|
|
log_error "SQLite configuration hot-reload simulation failed"
|
|
fi
|
|
|
|
else
|
|
log_error "SQLite database creation failed"
|
|
fi
|
|
|
|
# Cleanup
|
|
rm -f "$SQLITE_DB"
|
|
else
|
|
log_warning "sqlite3 not available - skipping SQLite tests"
|
|
fi
|
|
}
|
|
|
|
# Test InfluxDB
|
|
test_influxdb() {
|
|
echo ""
|
|
log_info "🔍 Testing InfluxDB..."
|
|
|
|
if command -v curl >/dev/null 2>&1; then
|
|
# Test health endpoint
|
|
if curl -s -o /dev/null -w "%{http_code}" "http://${INFLUX_URL}/health" | grep -q "200"; then
|
|
log_success "InfluxDB health check successful"
|
|
|
|
# Test write operation (simplified)
|
|
local timestamp=$(date +%s%N)
|
|
local line_protocol="test_measurement,tag1=value1 field1=123 $timestamp"
|
|
local write_url="http://${INFLUX_URL}/api/v2/write?org=test&bucket=test"
|
|
|
|
if curl -s -o /dev/null -w "%{http_code}" -X POST "$write_url" \
|
|
-H "Content-Type: text/plain" \
|
|
-d "$line_protocol" | grep -q "204\|200"; then
|
|
log_success "InfluxDB write test successful (or acceptable without auth)"
|
|
else
|
|
log_warning "InfluxDB write test failed (may need authentication)"
|
|
fi
|
|
|
|
elif curl -s -o /dev/null -w "%{http_code}" "http://${INFLUX_URL}/ping" | grep -q "204"; then
|
|
log_success "InfluxDB ping successful (legacy endpoint)"
|
|
else
|
|
log_error "InfluxDB connection failed"
|
|
fi
|
|
else
|
|
log_warning "curl not available - skipping InfluxDB tests"
|
|
fi
|
|
}
|
|
|
|
# Test backup capabilities
|
|
test_backup_capabilities() {
|
|
echo ""
|
|
log_info "🔍 Testing backup capabilities..."
|
|
|
|
# Check if backup tools are available
|
|
local backup_tools=("pg_dump" "redis-cli" "influxd" "tar")
|
|
local available_tools=0
|
|
|
|
for tool in "${backup_tools[@]}"; do
|
|
if command -v "$tool" >/dev/null 2>&1; then
|
|
((available_tools++))
|
|
log_success "$tool available for backups"
|
|
else
|
|
log_warning "$tool not available for backups"
|
|
fi
|
|
done
|
|
|
|
if [ $available_tools -gt 2 ]; then
|
|
log_success "Sufficient backup tools available ($available_tools/4)"
|
|
else
|
|
log_warning "Limited backup capabilities ($available_tools/4 tools available)"
|
|
fi
|
|
}
|
|
|
|
# Test health monitoring
|
|
test_health_monitoring() {
|
|
echo ""
|
|
log_info "🔍 Testing health monitoring capabilities..."
|
|
|
|
# Check if monitoring tools are available
|
|
local monitoring_tools=("systemctl" "ps" "netstat" "ss")
|
|
local available_tools=0
|
|
|
|
for tool in "${monitoring_tools[@]}"; do
|
|
if command -v "$tool" >/dev/null 2>&1; then
|
|
((available_tools++))
|
|
fi
|
|
done
|
|
|
|
if [ $available_tools -gt 2 ]; then
|
|
log_success "Health monitoring tools available ($available_tools/4)"
|
|
else
|
|
log_warning "Limited health monitoring capabilities ($available_tools/4)"
|
|
fi
|
|
|
|
# Test basic system health
|
|
local load_avg=$(uptime | awk '{print $(NF-2)}' | sed 's/,//')
|
|
if (( $(echo "$load_avg < 2.0" | bc -l 2>/dev/null || echo "1") )); then
|
|
log_success "System load acceptable ($load_avg)"
|
|
else
|
|
log_warning "High system load detected ($load_avg)"
|
|
fi
|
|
|
|
# Test memory usage
|
|
local mem_usage=$(free | awk '/Mem:/ { printf("%.1f", $3/$2 * 100.0) }')
|
|
if (( $(echo "$mem_usage < 80.0" | bc -l 2>/dev/null || echo "1") )); then
|
|
log_success "Memory usage acceptable (${mem_usage}%)"
|
|
else
|
|
log_warning "High memory usage detected (${mem_usage}%)"
|
|
fi
|
|
}
|
|
|
|
# Main execution
|
|
main() {
|
|
local start_time=$(date +%s)
|
|
|
|
log_info "Starting database validation at $(date)"
|
|
log_info "Configuration:"
|
|
log_info " PostgreSQL: $POSTGRES_URL"
|
|
log_info " Redis: $REDIS_URL"
|
|
log_info " InfluxDB: $INFLUX_URL"
|
|
log_info " SQLite: $SQLITE_DB"
|
|
|
|
# Run all tests
|
|
test_postgresql
|
|
test_redis
|
|
test_sqlite
|
|
test_influxdb
|
|
test_backup_capabilities
|
|
test_health_monitoring
|
|
|
|
local end_time=$(date +%s)
|
|
local total_duration=$((end_time - start_time))
|
|
|
|
# Generate summary
|
|
echo ""
|
|
echo "📊 VALIDATION SUMMARY"
|
|
echo "===================="
|
|
echo "Total Time: ${total_duration}s"
|
|
echo "Tests Passed: $TESTS_PASSED"
|
|
echo "Tests Failed: $TESTS_FAILED"
|
|
echo "Total Tests: $TOTAL_TESTS"
|
|
echo ""
|
|
|
|
if [ $TESTS_FAILED -eq 0 ]; then
|
|
log_success "🎉 All database validation tests completed successfully!"
|
|
echo ""
|
|
echo "✅ Database Layer Status:"
|
|
echo " - PostgreSQL: Connection and ACID transactions working"
|
|
echo " - Redis: Caching and TTL functionality working"
|
|
echo " - SQLite: Configuration storage and hot-reload ready"
|
|
echo " - InfluxDB: Metrics storage capability verified"
|
|
echo " - Backup/Recovery: Tools available for data protection"
|
|
echo " - Health Monitoring: System monitoring capabilities ready"
|
|
echo ""
|
|
echo "🚀 The database layer is ready for HFT operations!"
|
|
exit 0
|
|
else
|
|
log_error "💥 Database validation completed with $TESTS_FAILED failures"
|
|
echo ""
|
|
echo "❌ Issues found:"
|
|
echo " - $TESTS_FAILED out of $TOTAL_TESTS tests failed"
|
|
echo " - Review the logs above for specific failure details"
|
|
echo " - Consider addressing failed tests before production deployment"
|
|
echo ""
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Execute main function
|
|
main "$@" |