From 8ee1f810e6b5f59a2e5bc75d3b0ddd53ecca8381 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 28 Feb 2026 23:26:43 +0100 Subject: [PATCH] fix(production): resolve all runtime errors and add CI migration automation - Add ohlcv_bars table migration (050) for trading_service prediction loop - Fix kube-state-metrics RBAC: add admissionregistration.k8s.io and certificates.k8s.io API groups to suppress forbidden errors - Fix web-gateway gRPC stream reconnect: detect Unimplemented via both status code and error message text (proxy may remap codes) - Add automated migration step to CI deploy pipeline with tracking table (_applied_migrations) so migrations are applied before service rollouts Co-Authored-By: Claude Opus 4.6 --- .gitlab-ci.yml | 26 ++++++++++++++++++++ crates/web-gateway/src/grpc/streams.rs | 13 +++++++++- infra/k8s/monitoring/kube-state-metrics.yaml | 9 +++++++ migrations/050_create_ohlcv_bars.sql | 16 ++++++++++++ 4 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 migrations/050_create_ohlcv_bars.sql diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 951f92917..a99a2e931 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1419,6 +1419,32 @@ deploy: # S3 binary share: services use generic runtime image + initContainer to fetch binaries from S3. # Deploy = apply K8s manifests (picks up YAML changes) + rollout restart (fetch new binaries). - echo "Deploying ${CI_COMMIT_SHA}..." + # Apply pending database migrations before deploying services + - | + echo "Applying database migrations..." + DB_POD=$(kubectl -n foxhunt get pod -l app.kubernetes.io/name=postgres -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true) + if [ -n "$DB_POD" ]; then + # Create tracking table if it doesn't exist + kubectl -n foxhunt exec "$DB_POD" -- psql -U foxhunt -d foxhunt -c \ + "CREATE TABLE IF NOT EXISTS _applied_migrations (name TEXT PRIMARY KEY, applied_at TIMESTAMPTZ DEFAULT now())" 2>/dev/null || true + # Apply each migration in order, skipping already-applied ones + for f in $(ls migrations/*.sql | grep -v '\.down\.' | sort); do + MIGRATION_NAME=$(basename "$f") + ALREADY=$(kubectl -n foxhunt exec "$DB_POD" -- psql -U foxhunt -d foxhunt -tAc \ + "SELECT 1 FROM _applied_migrations WHERE name = '${MIGRATION_NAME}'" 2>/dev/null || true) + if [ "$ALREADY" = "1" ]; then + echo " skip: $MIGRATION_NAME (already applied)" + else + echo " apply: $MIGRATION_NAME" + kubectl -n foxhunt exec -i "$DB_POD" -- psql -U foxhunt -d foxhunt < "$f" + kubectl -n foxhunt exec "$DB_POD" -- psql -U foxhunt -d foxhunt -c \ + "INSERT INTO _applied_migrations (name) VALUES ('${MIGRATION_NAME}')" + fi + done + echo "Migrations complete" + else + echo "WARNING: postgres pod not found, skipping migrations" + fi # Deploy monitoring stack (Loki, Tempo, Promtail, node-exporter, kube-state-metrics) - kubectl apply -f infra/k8s/monitoring/loki.yaml -f infra/k8s/monitoring/tempo.yaml -f infra/k8s/monitoring/promtail.yaml -f infra/k8s/monitoring/node-exporter.yaml -f infra/k8s/monitoring/kube-state-metrics.yaml # Apply proxy (picks up nginx config changes for dashboard.fxhnt.ai routing) diff --git a/crates/web-gateway/src/grpc/streams.rs b/crates/web-gateway/src/grpc/streams.rs index f3b3705ca..1929abbf5 100644 --- a/crates/web-gateway/src/grpc/streams.rs +++ b/crates/web-gateway/src/grpc/streams.rs @@ -183,6 +183,17 @@ async fn heartbeat_loop(tx: broadcast::Sender) { } } +/// Check whether a gRPC error indicates the endpoint is not implemented. +/// Matches both the standard Unimplemented status code and error messages +/// containing "not implemented" (which proxies may return with a different code). +fn is_unimplemented(e: &tonic::Status) -> bool { + if e.code() == tonic::Code::Unimplemented { + return true; + } + let msg = e.message().to_lowercase(); + msg.contains("not implemented") || msg.contains("unimplemented") +} + /// Reconnect wrapper with exponential backoff for a gRPC stream bridge task. /// Stops retrying if the server returns Unimplemented (endpoint doesn't exist). async fn reconnect_loop(stream_name: &str, connect_fn: F) @@ -202,7 +213,7 @@ where backoff = Duration::from_secs(1); info!("gRPC stream {} ended, reconnecting", stream_name); } - Err(e) if e.code() == tonic::Code::Unimplemented => { + Err(e) if is_unimplemented(&e) => { info!( "gRPC stream {} not implemented on server, disabling", stream_name diff --git a/infra/k8s/monitoring/kube-state-metrics.yaml b/infra/k8s/monitoring/kube-state-metrics.yaml index 9b24d6aa2..3117c9302 100644 --- a/infra/k8s/monitoring/kube-state-metrics.yaml +++ b/infra/k8s/monitoring/kube-state-metrics.yaml @@ -65,6 +65,15 @@ rules: resources: - leases verbs: ["list", "watch"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: ["list", "watch"] + - apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests + verbs: ["list", "watch"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/migrations/050_create_ohlcv_bars.sql b/migrations/050_create_ohlcv_bars.sql new file mode 100644 index 000000000..4dbe71444 --- /dev/null +++ b/migrations/050_create_ohlcv_bars.sql @@ -0,0 +1,16 @@ +-- OHLCV bar data for prediction generation +-- Used by trading_service prediction_generation_loop to query recent market data +CREATE TABLE IF NOT EXISTS ohlcv_bars ( + id BIGSERIAL PRIMARY KEY, + symbol VARCHAR(32) NOT NULL, + timestamp TIMESTAMPTZ NOT NULL, + open DOUBLE PRECISION NOT NULL, + high DOUBLE PRECISION NOT NULL, + low DOUBLE PRECISION NOT NULL, + close DOUBLE PRECISION NOT NULL, + volume DOUBLE PRECISION NOT NULL DEFAULT 0, + UNIQUE (symbol, timestamp) +); + +CREATE INDEX IF NOT EXISTS idx_ohlcv_bars_symbol_ts + ON ohlcv_bars (symbol, timestamp DESC);