Prometheus was configured to scrape pods via prometheus.io/scrape annotations, but all services and training jobs used gitlab.com/ prefix from legacy GitLab-managed Prometheus — causing Prometheus to never discover any foxhunt pods. This resulted in stale/missing metrics on the Grafana training dashboard. 12 files updated across services/, gpu-overlays/, training/, and monitoring/ (node-exporter, dcgm-exporter cleanup). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
164 lines
4.8 KiB
YAML
164 lines
4.8 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: backtesting-service
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: backtesting-service
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
replicas: 1
|
|
strategy:
|
|
type: RollingUpdate
|
|
rollingUpdate:
|
|
maxSurge: 0
|
|
maxUnavailable: 1
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/name: backtesting-service
|
|
template:
|
|
metadata:
|
|
annotations:
|
|
prometheus.io/scrape: "true"
|
|
prometheus.io/port: "9093"
|
|
prometheus.io/path: "/metrics"
|
|
labels:
|
|
app.kubernetes.io/name: backtesting-service
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
securityContext:
|
|
runAsNonRoot: true
|
|
runAsUser: 1000
|
|
runAsGroup: 1000
|
|
fsGroup: 1000
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
imagePullSecrets:
|
|
- name: gitlab-registry
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
initContainers:
|
|
- name: fetch-binary
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
set -e
|
|
BINARY="backtesting-service"
|
|
curl -fSL -o "/binaries/${BINARY}" \
|
|
--header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \
|
|
"${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}"
|
|
chmod +x "/binaries/${BINARY}"
|
|
echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)"
|
|
env:
|
|
- name: GITLAB_DEPLOY_TOKEN
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: gitlab-deploy-token
|
|
key: token
|
|
- name: GITLAB_API
|
|
value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4"
|
|
- name: FOXHUNT_RELEASE
|
|
value: "latest"
|
|
volumeMounts:
|
|
- name: binaries
|
|
mountPath: /binaries
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 128Mi
|
|
containers:
|
|
- name: backtesting-service
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
readOnlyRootFilesystem: true
|
|
capabilities:
|
|
drop: ["ALL"]
|
|
command: ["/binaries/backtesting-service"]
|
|
ports:
|
|
- containerPort: 50053
|
|
name: grpc
|
|
- containerPort: 9093
|
|
name: metrics
|
|
env:
|
|
- name: DATABASE_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: db-credentials
|
|
key: password
|
|
- name: DATABASE_URL
|
|
value: "postgresql://foxhunt:$(DATABASE_PASSWORD)@postgres:5432/foxhunt"
|
|
- name: REDIS_URL
|
|
value: "redis://redis:6379"
|
|
- name: JWT_SECRET
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: jwt-secret
|
|
key: secret
|
|
- name: JWT_ISSUER
|
|
value: foxhunt-api
|
|
- name: JWT_AUDIENCE
|
|
value: foxhunt-services
|
|
- name: BENZINGA_API_KEY
|
|
value: "placeholder"
|
|
- name: RUST_LOG
|
|
value: "info,opentelemetry=warn,h2=warn,tonic=warn,hyper=warn"
|
|
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
|
value: "http://tempo.foxhunt.svc.cluster.local:4317"
|
|
volumeMounts:
|
|
- name: binaries
|
|
mountPath: /binaries
|
|
readOnly: true
|
|
- name: tmp
|
|
mountPath: /tmp
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8082
|
|
initialDelaySeconds: 15
|
|
periodSeconds: 10
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /health
|
|
port: 8082
|
|
initialDelaySeconds: 30
|
|
periodSeconds: 15
|
|
failureThreshold: 5
|
|
resources:
|
|
requests:
|
|
cpu: 200m
|
|
memory: 256Mi
|
|
limits:
|
|
cpu: 500m
|
|
memory: 512Mi
|
|
volumes:
|
|
- name: binaries
|
|
emptyDir:
|
|
sizeLimit: 200Mi
|
|
- name: tmp
|
|
emptyDir:
|
|
sizeLimit: 50Mi
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: backtesting-service
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: backtesting-service
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
selector:
|
|
app.kubernetes.io/name: backtesting-service
|
|
ports:
|
|
- port: 50053
|
|
targetPort: 50053
|
|
name: grpc
|
|
- port: 9093
|
|
targetPort: 9093
|
|
name: metrics
|