fix(cli): close prebuild runner prometheus server last (#21053)

ethanndickson · web-flow · commit bf40d678ec26 · 2025-12-02T12:10:50.000Z
## Description Fixes the prebuilds scaletest command where the prometheus server was being shut down before waiting for metrics to be scraped. The issue was the defer order - since defers execute in LIFO (last-in, first-out) order: **Before (broken):** 1. Register tracing defer (includes wait for prometheus scrape) 2. Register prometheus server defer Execution order: prometheus closes first, then wait happens (server already gone!) **After (fixed):** 1. Register prometheus server defer 2. Register tracing defer (includes wait for prometheus scrape) Execution order: wait happens first (server still up), then prometheus closes. This matches the pattern used in other scaletest commands. ## Impact The `coderd_scaletest_prebuild_deletion_jobs_completed` metric (and potentially others) was always showing 0 because the server shut down before Prometheus could scrape the final values. _This PR was generated by [`mux`](https://github.com/coder/mux) and reviewed by a human._
diff --git a/cli/exp_scaletest.go b/cli/exp_scaletest.go
@@ -1559,6 +1559,15 @@ func (r *RootCmd) scaletestDashboard() *serpent.Command {
 			if err != nil {
 				return xerrors.Errorf("create tracer provider: %w", err)
 			}
+			tracer := tracerProvider.Tracer(scaletestTracerName)
+			outputs, err := output.parse()
+			if err != nil {
+				return xerrors.Errorf("could not parse --output flags")
+			}
+			reg := prometheus.NewRegistry()
+			prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), prometheusFlags.Address, "prometheus")
+			defer prometheusSrvClose()
+
 			defer func() {
 				// Allow time for traces to flush even if command context is
 				// canceled. This is a no-op if tracing is not enabled.
@@ -1570,14 +1579,7 @@ func (r *RootCmd) scaletestDashboard() *serpent.Command {
 				_, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", prometheusFlags.Wait)
 				<-time.After(prometheusFlags.Wait)
 			}()
-			tracer := tracerProvider.Tracer(scaletestTracerName)
-			outputs, err := output.parse()
-			if err != nil {
-				return xerrors.Errorf("could not parse --output flags")
-			}
-			reg := prometheus.NewRegistry()
-			prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), prometheusFlags.Address, "prometheus")
-			defer prometheusSrvClose()
+
 			metrics := dashboard.NewMetrics(reg)
 
 			th := harness.NewTestHarness(strategy.toStrategy(), cleanupStrategy.toStrategy())
diff --git a/cli/exp_scaletest_prebuilds.go b/cli/exp_scaletest_prebuilds.go
@@ -84,14 +84,6 @@ func (r *RootCmd) scaletestPrebuilds() *serpent.Command {
 			if err != nil {
 				return xerrors.Errorf("create tracer provider: %w", err)
 			}
-			defer func() {
-				_, _ = fmt.Fprintln(inv.Stderr, "\nUploading traces...")
-				if err := closeTracing(ctx); err != nil {
-					_, _ = fmt.Fprintf(inv.Stderr, "\nError uploading traces: %+v\n", err)
-				}
-				_, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", prometheusFlags.Wait)
-				<-time.After(prometheusFlags.Wait)
-			}()
 			tracer := tracerProvider.Tracer(scaletestTracerName)
 
 			reg := prometheus.NewRegistry()
@@ -101,6 +93,15 @@ func (r *RootCmd) scaletestPrebuilds() *serpent.Command {
 			prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), prometheusFlags.Address, "prometheus")
 			defer prometheusSrvClose()
 
+			defer func() {
+				_, _ = fmt.Fprintln(inv.Stderr, "\nUploading traces...")
+				if err := closeTracing(ctx); err != nil {
+					_, _ = fmt.Fprintf(inv.Stderr, "\nError uploading traces: %+v\n", err)
+				}
+				_, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", prometheusFlags.Wait)
+				<-time.After(prometheusFlags.Wait)
+			}()
+
 			err = client.PutPrebuildsSettings(ctx, codersdk.PrebuildsSettings{
 				ReconciliationPaused: true,
 			})
