From 11795c5b63e39ad70229ccceb44751e65eee2f1d Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Tue, 2 Dec 2025 12:28:12 -0600 Subject: [PATCH 1/2] chore: fix race condition on aggregating terraform logs The logs could be aggregated before all cmd output was read --- provisioner/terraform/executor.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/provisioner/terraform/executor.go b/provisioner/terraform/executor.go index 3d9270a6ddbab..4f21f26c8004f 100644 --- a/provisioner/terraform/executor.go +++ b/provisioner/terraform/executor.go @@ -386,6 +386,8 @@ func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr l } } + // DoneOut must be completed before we aggregate timings to ensure all logs have been processed. + <-doneOut msg := &proto.PlanComplete{ Parameters: state.Parameters, Resources: state.Resources, @@ -617,6 +619,8 @@ func (e *executor) apply( return nil, xerrors.Errorf("read statefile %q: %w", statefilePath, err) } + // DoneOut must be completed before we aggregate timings to ensure all logs have been processed. + <-doneOut agg := e.timings.aggregate() return &proto.ApplyComplete{ Parameters: state.Parameters, From 6b8d46686749d06bb3225e7266132ee706e557d0 Mon Sep 17 00:00:00 2001 From: Steven Masley Date: Wed, 3 Dec 2025 05:57:27 -0600 Subject: [PATCH 2/2] chore: move timing aggregation and timing to outside the cmd exec --- provisioner/terraform/executor.go | 12 +----------- provisioner/terraform/provision.go | 7 ++++++- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/provisioner/terraform/executor.go b/provisioner/terraform/executor.go index 4f21f26c8004f..b132b78982bea 100644 --- a/provisioner/terraform/executor.go +++ b/provisioner/terraform/executor.go @@ -325,9 +325,7 @@ func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr l <-doneErr }() - endStage := e.timings.startStage(database.ProvisionerJobTimingStagePlan) err := e.execWriteOutput(ctx, killCtx, args, env, outWriter, errWriter) - endStage(err) if err != nil { return nil, xerrors.Errorf("terraform plan: %w", err) } @@ -386,13 +384,11 @@ func (e *executor) plan(ctx, killCtx context.Context, env, vars []string, logr l } } - // DoneOut must be completed before we aggregate timings to ensure all logs have been processed. - <-doneOut msg := &proto.PlanComplete{ Parameters: state.Parameters, Resources: state.Resources, ExternalAuthProviders: state.ExternalAuthProviders, - Timings: append(e.timings.aggregate(), graphTimings.aggregate()...), + Timings: graphTimings.aggregate(), Presets: state.Presets, Plan: planJSON, ResourceReplacements: resReps, @@ -601,9 +597,7 @@ func (e *executor) apply( }() // `terraform apply` - endStage := e.timings.startStage(database.ProvisionerJobTimingStageApply) err := e.execWriteOutput(ctx, killCtx, args, env, outWriter, errWriter) - endStage(err) if err != nil { return nil, xerrors.Errorf("terraform apply: %w", err) } @@ -619,15 +613,11 @@ func (e *executor) apply( return nil, xerrors.Errorf("read statefile %q: %w", statefilePath, err) } - // DoneOut must be completed before we aggregate timings to ensure all logs have been processed. - <-doneOut - agg := e.timings.aggregate() return &proto.ApplyComplete{ Parameters: state.Parameters, Resources: state.Resources, ExternalAuthProviders: state.ExternalAuthProviders, State: stateContent, - Timings: agg, AiTasks: state.AITasks, }, nil } diff --git a/provisioner/terraform/provision.go b/provisioner/terraform/provision.go index c99ee55ad8cc6..5fedf16c21eb1 100644 --- a/provisioner/terraform/provision.go +++ b/provisioner/terraform/provision.go @@ -160,14 +160,16 @@ func (s *server) Plan( return provisionersdk.PlanErrorf("plan vars: %s", err) } + endPlanStage := e.timings.startStage(database.ProvisionerJobTimingStagePlan) resp, err := e.plan(ctx, killCtx, env, vars, sess, request) + endPlanStage(err) if err != nil { return provisionersdk.PlanErrorf("%s", err.Error()) } // Prepend init timings since they occur prior to plan timings. // Order is irrelevant; this is merely indicative. - resp.Timings = append(initTimings.aggregate(), resp.Timings...) // mergeInitTimings(initTimings.aggregate(), resp.Timings) + resp.Timings = append(resp.Timings, append(initTimings.aggregate(), e.timings.aggregate()...)...) resp.Modules = modules return resp } @@ -204,9 +206,11 @@ func (s *server) Apply( return provisionersdk.ApplyErrorf("provision env: %s", err) } env = otelEnvInject(ctx, env) + endStage := e.timings.startStage(database.ProvisionerJobTimingStageApply) resp, err := e.apply( ctx, killCtx, env, sess, ) + endStage(err) if err != nil { errorMessage := err.Error() // Terraform can fail and apply and still need to store it's state. @@ -217,6 +221,7 @@ func (s *server) Apply( Error: errorMessage, } } + resp.Timings = e.timings.aggregate() return resp }