diff --git a/coderd/autobuild/lifecycle_executor_test.go b/coderd/autobuild/lifecycle_executor_test.go index 19000b9d8f967..ac9a50bff2111 100644 --- a/coderd/autobuild/lifecycle_executor_test.go +++ b/coderd/autobuild/lifecycle_executor_test.go @@ -40,10 +40,10 @@ func TestExecutorAutostartOK(t *testing.T) { t.Parallel() var ( - sched = mustSchedule(t, "CRON_TZ=UTC 0 * * * *") - tickCh = make(chan time.Time) - statsCh = make(chan autobuild.Stats) - client = coderdtest.New(t, &coderdtest.Options{ + sched = mustSchedule(t, "CRON_TZ=UTC 0 * * * *") + tickCh = make(chan time.Time) + statsCh = make(chan autobuild.Stats) + client, db = coderdtest.NewWithDatabase(t, &coderdtest.Options{ AutobuildTicker: tickCh, IncludeProvisionerDaemon: true, AutobuildStats: statsCh, @@ -55,10 +55,13 @@ func TestExecutorAutostartOK(t *testing.T) { ) // Given: workspace is stopped workspace = coderdtest.MustTransitionWorkspace(t, client, workspace.ID, codersdk.WorkspaceTransitionStart, codersdk.WorkspaceTransitionStop) - + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), workspace.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) // When: the autobuild executor ticks after the scheduled time go func() { - tickCh <- sched.Next(workspace.LatestBuild.CreatedAt) + tickTime := sched.Next(workspace.LatestBuild.CreatedAt) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + tickCh <- tickTime close(tickCh) }() @@ -114,8 +117,11 @@ func TestMultipleLifecycleExecutors(t *testing.T) { // Have the workspace stopped so we can perform an autostart workspace = coderdtest.MustTransitionWorkspace(t, clientA, workspace.ID, codersdk.WorkspaceTransitionStart, codersdk.WorkspaceTransitionStop) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), workspace.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) // Get both clients to perform a lifecycle execution tick next := sched.Next(workspace.LatestBuild.CreatedAt) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), next) startCh := make(chan struct{}) go func() { @@ -187,14 +193,14 @@ func TestExecutorAutostartTemplateUpdated(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() var ( - sched = mustSchedule(t, "CRON_TZ=UTC 0 * * * *") - ctx = context.Background() - err error - tickCh = make(chan time.Time) - statsCh = make(chan autobuild.Stats) - logger = slogtest.Make(t, &slogtest.Options{IgnoreErrors: !tc.expectStart}).Leveled(slog.LevelDebug) - enqueuer = notificationstest.FakeEnqueuer{} - client = coderdtest.New(t, &coderdtest.Options{ + sched = mustSchedule(t, "CRON_TZ=UTC 0 * * * *") + ctx = context.Background() + err error + tickCh = make(chan time.Time) + statsCh = make(chan autobuild.Stats) + logger = slogtest.Make(t, &slogtest.Options{IgnoreErrors: !tc.expectStart}).Leveled(slog.LevelDebug) + enqueuer = notificationstest.FakeEnqueuer{} + client, db = coderdtest.NewWithDatabase(t, &coderdtest.Options{ AutobuildTicker: tickCh, IncludeProvisionerDaemon: true, AutobuildStats: statsCh, @@ -247,10 +253,15 @@ func TestExecutorAutostartTemplateUpdated(t *testing.T) { }, )) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), workspace.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + t.Log("sending autobuild tick") // When: the autobuild executor ticks after the scheduled time go func() { - tickCh <- sched.Next(workspace.LatestBuild.CreatedAt) + tickTime := sched.Next(workspace.LatestBuild.CreatedAt) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + tickCh <- tickTime close(tickCh) }() @@ -414,9 +425,9 @@ func TestExecutorAutostopOK(t *testing.T) { t.Parallel() var ( - tickCh = make(chan time.Time) - statsCh = make(chan autobuild.Stats) - client = coderdtest.New(t, &coderdtest.Options{ + tickCh = make(chan time.Time) + statsCh = make(chan autobuild.Stats) + client, db = coderdtest.NewWithDatabase(t, &coderdtest.Options{ AutobuildTicker: tickCh, IncludeProvisionerDaemon: true, AutobuildStats: statsCh, @@ -428,9 +439,14 @@ func TestExecutorAutostopOK(t *testing.T) { require.Equal(t, codersdk.WorkspaceTransitionStart, workspace.LatestBuild.Transition) require.NotZero(t, workspace.LatestBuild.Deadline) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), workspace.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + // When: the autobuild executor ticks *after* the deadline: go func() { - tickCh <- workspace.LatestBuild.Deadline.Time.Add(time.Minute) + tickTime := workspace.LatestBuild.Deadline.Time.Add(time.Minute) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + tickCh <- tickTime close(tickCh) }() @@ -449,10 +465,10 @@ func TestExecutorAutostopExtend(t *testing.T) { t.Parallel() var ( - ctx = context.Background() - tickCh = make(chan time.Time) - statsCh = make(chan autobuild.Stats) - client = coderdtest.New(t, &coderdtest.Options{ + ctx = context.Background() + tickCh = make(chan time.Time) + statsCh = make(chan autobuild.Stats) + client, db = coderdtest.NewWithDatabase(t, &coderdtest.Options{ AutobuildTicker: tickCh, IncludeProvisionerDaemon: true, AutobuildStats: statsCh, @@ -472,9 +488,14 @@ func TestExecutorAutostopExtend(t *testing.T) { }) require.NoError(t, err, "extend workspace deadline") + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), workspace.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + // When: the autobuild executor ticks *after* the original deadline: go func() { - tickCh <- originalDeadline.Time.Add(time.Minute) + tickTime := originalDeadline.Time.Add(time.Minute) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + tickCh <- tickTime }() // Then: nothing should happen and the workspace should stay running @@ -484,7 +505,9 @@ func TestExecutorAutostopExtend(t *testing.T) { // When: the autobuild executor ticks after the *new* deadline: go func() { - tickCh <- newDeadline.Add(time.Minute) + tickTime := newDeadline.Add(time.Minute) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + tickCh <- tickTime close(tickCh) }() @@ -677,7 +700,7 @@ func TestExecuteAutostopSuspendedUser(t *testing.T) { admin := coderdtest.CreateFirstUser(t, client) // Wait for provisioner to be available - mustWaitForProvisionersWithClient(t, client) + coderdtest.MustWaitForProvisionersWithClient(t, client) version := coderdtest.CreateTemplateVersion(t, client, admin.OrganizationID, nil) coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) template := coderdtest.CreateTemplate(t, client, admin.OrganizationID, version.ID) @@ -755,17 +778,17 @@ func TestExecutorAutostartMultipleOK(t *testing.T) { t.Parallel() var ( - sched = mustSchedule(t, "CRON_TZ=UTC 0 * * * *") - tickCh = make(chan time.Time) - tickCh2 = make(chan time.Time) - statsCh1 = make(chan autobuild.Stats) - statsCh2 = make(chan autobuild.Stats) - client = coderdtest.New(t, &coderdtest.Options{ + sched = mustSchedule(t, "CRON_TZ=UTC 0 * * * *") + tickCh = make(chan time.Time) + tickCh2 = make(chan time.Time) + statsCh1 = make(chan autobuild.Stats) + statsCh2 = make(chan autobuild.Stats) + client, db = coderdtest.NewWithDatabase(t, &coderdtest.Options{ AutobuildTicker: tickCh, IncludeProvisionerDaemon: true, AutobuildStats: statsCh1, }) - _ = coderdtest.New(t, &coderdtest.Options{ + _, _ = coderdtest.NewWithDatabase(t, &coderdtest.Options{ AutobuildTicker: tickCh2, IncludeProvisionerDaemon: true, AutobuildStats: statsCh2, @@ -778,10 +801,15 @@ func TestExecutorAutostartMultipleOK(t *testing.T) { // Given: workspace is stopped workspace = coderdtest.MustTransitionWorkspace(t, client, workspace.ID, codersdk.WorkspaceTransitionStart, codersdk.WorkspaceTransitionStop) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), workspace.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + // When: the autobuild executor ticks past the scheduled time go func() { - tickCh <- sched.Next(workspace.LatestBuild.CreatedAt) - tickCh2 <- sched.Next(workspace.LatestBuild.CreatedAt) + tickTime := sched.Next(workspace.LatestBuild.CreatedAt) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + tickCh <- tickTime + tickCh2 <- tickTime close(tickCh) close(tickCh2) }() @@ -811,10 +839,10 @@ func TestExecutorAutostartWithParameters(t *testing.T) { ) var ( - sched = mustSchedule(t, "CRON_TZ=UTC 0 * * * *") - tickCh = make(chan time.Time) - statsCh = make(chan autobuild.Stats) - client = coderdtest.New(t, &coderdtest.Options{ + sched = mustSchedule(t, "CRON_TZ=UTC 0 * * * *") + tickCh = make(chan time.Time) + statsCh = make(chan autobuild.Stats) + client, db = coderdtest.NewWithDatabase(t, &coderdtest.Options{ AutobuildTicker: tickCh, IncludeProvisionerDaemon: true, AutobuildStats: statsCh, @@ -843,9 +871,14 @@ func TestExecutorAutostartWithParameters(t *testing.T) { // Given: workspace is stopped workspace = coderdtest.MustTransitionWorkspace(t, client, workspace.ID, codersdk.WorkspaceTransitionStart, codersdk.WorkspaceTransitionStop) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), workspace.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + // When: the autobuild executor ticks after the scheduled time go func() { - tickCh <- sched.Next(workspace.LatestBuild.CreatedAt) + tickTime := sched.Next(workspace.LatestBuild.CreatedAt) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + tickCh <- tickTime close(tickCh) }() @@ -913,7 +946,7 @@ func TestExecutorAutostopTemplateDisabled(t *testing.T) { tickCh = make(chan time.Time) statsCh = make(chan autobuild.Stats) - client = coderdtest.New(t, &coderdtest.Options{ + client, db = coderdtest.NewWithDatabase(t, &coderdtest.Options{ AutobuildTicker: tickCh, IncludeProvisionerDaemon: true, AutobuildStats: statsCh, @@ -937,9 +970,14 @@ func TestExecutorAutostopTemplateDisabled(t *testing.T) { // Then: the deadline should be set to the template default TTL assert.WithinDuration(t, workspace.LatestBuild.CreatedAt.Add(time.Hour), workspace.LatestBuild.Deadline.Time, time.Minute) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), workspace.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + // When: the autobuild executor ticks after the workspace setting, but before the template setting: go func() { - tickCh <- workspace.LatestBuild.Job.CompletedAt.Add(45 * time.Minute) + tickTime := workspace.LatestBuild.Job.CompletedAt.Add(45 * time.Minute) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + tickCh <- tickTime }() // Then: nothing should happen @@ -949,7 +987,9 @@ func TestExecutorAutostopTemplateDisabled(t *testing.T) { // When: the autobuild executor ticks after the template setting: go func() { - tickCh <- workspace.LatestBuild.Job.CompletedAt.Add(61 * time.Minute) + tickTime := workspace.LatestBuild.Job.CompletedAt.Add(61 * time.Minute) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + tickCh <- tickTime close(tickCh) }() @@ -979,7 +1019,8 @@ func TestExecutorRequireActiveVersion(t *testing.T) { }) ) // Wait for provisioner to be available - mustWaitForProvisionersWithClient(t, ownerClient) + coderdtest.MustWaitForProvisioners(t, db) + ctx := testutil.Context(t, testutil.WaitShort) owner := coderdtest.CreateFirstUser(t, ownerClient) me, err := ownerClient.User(ctx, codersdk.Me) @@ -1016,7 +1057,13 @@ func TestExecutorRequireActiveVersion(t *testing.T) { req.TemplateVersionID = inactiveVersion.ID }) require.Equal(t, inactiveVersion.ID, ws.LatestBuild.TemplateVersionID) - ticker <- sched.Next(ws.LatestBuild.CreatedAt) + + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), ws.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + + tickTime := sched.Next(ws.LatestBuild.CreatedAt) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + ticker <- tickTime stats := <-statCh require.Len(t, stats.Transitions, 1) @@ -1136,7 +1183,7 @@ func TestNotifications(t *testing.T) { statCh = make(chan autobuild.Stats) notifyEnq = notificationstest.FakeEnqueuer{} timeTilDormant = time.Minute - client = coderdtest.New(t, &coderdtest.Options{ + client, db = coderdtest.NewWithDatabase(t, &coderdtest.Options{ AutobuildTicker: ticker, AutobuildStats: statCh, IncludeProvisionerDaemon: true, @@ -1173,9 +1220,14 @@ func TestNotifications(t *testing.T) { workspace = coderdtest.MustTransitionWorkspace(t, client, workspace.ID, codersdk.WorkspaceTransitionStart, codersdk.WorkspaceTransitionStop) _ = coderdtest.AwaitWorkspaceBuildJobCompleted(t, userClient, workspace.LatestBuild.ID) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), workspace.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + // Wait for workspace to become dormant notifyEnq.Clear() - ticker <- workspace.LastUsedAt.Add(timeTilDormant * 3) + tickTime := workspace.LastUsedAt.Add(timeTilDormant * 3) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + ticker <- tickTime _ = testutil.TryReceive(testutil.Context(t, testutil.WaitShort), t, statCh) // Check that the workspace is dormant @@ -1249,9 +1301,14 @@ func TestExecutorPrebuilds(t *testing.T) { require.Equal(t, codersdk.WorkspaceTransitionStart, prebuild.LatestBuild.Transition) require.NotZero(t, prebuild.LatestBuild.Deadline) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), prebuild.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + // When: the autobuild executor ticks *after* the deadline: go func() { - tickCh <- prebuild.LatestBuild.Deadline.Time.Add(time.Minute) + tickTime := prebuild.LatestBuild.Deadline.Time.Add(time.Minute) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + tickCh <- tickTime }() // Then: the prebuilt workspace should remain in a start transition @@ -1268,7 +1325,9 @@ func TestExecutorPrebuilds(t *testing.T) { // When: the autobuild executor ticks *after* the deadline: go func() { - tickCh <- workspace.LatestBuild.Deadline.Time.Add(time.Minute) + tickTime := workspace.LatestBuild.Deadline.Time.Add(time.Minute) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + tickCh <- tickTime close(tickCh) }() @@ -1603,71 +1662,6 @@ func mustWorkspaceParameters(t *testing.T, client *codersdk.Client, workspaceID require.NotEmpty(t, buildParameters) } -func mustWaitForProvisioners(t *testing.T, db database.Store) { - t.Helper() - ctx := testutil.Context(t, testutil.WaitShort) - require.Eventually(t, func() bool { - daemons, err := db.GetProvisionerDaemons(ctx) - return err == nil && len(daemons) > 0 - }, testutil.WaitShort, testutil.IntervalFast) -} - -func mustWaitForProvisionersWithClient(t *testing.T, client *codersdk.Client) { - t.Helper() - ctx := testutil.Context(t, testutil.WaitShort) - require.Eventually(t, func() bool { - daemons, err := client.ProvisionerDaemons(ctx) - return err == nil && len(daemons) > 0 - }, testutil.WaitShort, testutil.IntervalFast) -} - -// mustWaitForProvisionersAvailable waits for provisioners to be available for a specific workspace -func mustWaitForProvisionersAvailable(t *testing.T, db database.Store, workspace codersdk.Workspace) { - t.Helper() - ctx := testutil.Context(t, testutil.WaitShort) - - // Get the workspace from the database - require.Eventually(t, func() bool { - ws, err := db.GetWorkspaceByID(ctx, workspace.ID) - if err != nil { - return false - } - - // Get the latest build - latestBuild, err := db.GetWorkspaceBuildByID(ctx, workspace.LatestBuild.ID) - if err != nil { - return false - } - - // Get the template version job - templateVersionJob, err := db.GetProvisionerJobByID(ctx, latestBuild.JobID) - if err != nil { - return false - } - - // Check if provisioners are available using the same logic as hasAvailableProvisioners - provisionerDaemons, err := db.GetProvisionerDaemonsByOrganization(ctx, database.GetProvisionerDaemonsByOrganizationParams{ - OrganizationID: ws.OrganizationID, - WantTags: templateVersionJob.Tags, - }) - if err != nil { - return false - } - - // Check if any provisioners are active (not stale) - now := time.Now() - for _, pd := range provisionerDaemons { - if pd.LastSeenAt.Valid { - age := now.Sub(pd.LastSeenAt.Time) - if age <= autobuild.TestingStaleInterval { - return true // Found an active provisioner - } - } - } - return false // No active provisioners found - }, testutil.WaitShort, testutil.IntervalFast) -} - func TestMain(m *testing.M) { goleak.VerifyTestMain(m, testutil.GoleakOptions...) } @@ -1705,10 +1699,10 @@ func TestExecutorAutostartSkipsWhenNoProvisionersAvailable(t *testing.T) { codersdk.WorkspaceTransitionStart, codersdk.WorkspaceTransitionStop) // Wait for provisioner to be registered - mustWaitForProvisioners(t, db) + coderdtest.MustWaitForProvisioners(t, db) // Wait for provisioner to be available for this specific workspace - mustWaitForProvisionersAvailable(t, db, workspace) + coderdtest.MustWaitForProvisionersAvailable(t, db, workspace, autobuild.TestingStaleInterval) // Now shut down the provisioner daemon ctx := testutil.Context(t, testutil.WaitShort) diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go index f809379a5828a..cf432002c9bfb 100644 --- a/coderd/coderdtest/coderdtest.go +++ b/coderd/coderdtest/coderdtest.go @@ -55,6 +55,7 @@ import ( "cdr.dev/slog/sloggers/slogtest" "github.com/coder/coder/v2/archive" "github.com/coder/coder/v2/coderd/files" + "github.com/coder/coder/v2/coderd/provisionerdserver" "github.com/coder/coder/v2/coderd/wsbuilder" "github.com/coder/quartz" @@ -1591,3 +1592,117 @@ func DeploymentValues(t testing.TB, mut ...func(*codersdk.DeploymentValues)) *co } return cfg } + +// GetProvisionerForWorkspace returns the first valid provisioner for a workspace + template tags. +func GetProvisionerForWorkspace(t *testing.T, tx database.Store, curTime time.Time, orgID uuid.UUID, templateVersionJob database.ProvisionerJob) (database.ProvisionerDaemon, error) { + queryParams := database.GetProvisionerDaemonsByOrganizationParams{ + OrganizationID: orgID, + WantTags: templateVersionJob.Tags, + } + + // nolint: gocritic // The user (in this case, the user/context for autostart builds) may not have the full + // permissions to read provisioner daemons, but we need to check if there's any for the job prior to the + // execution of the job via autostart to fix: https://github.com/coder/coder/issues/17941 + provisionerDaemons, err := tx.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(context.Background()), queryParams) + if err != nil { + return database.ProvisionerDaemon{}, xerrors.Errorf("get provisioner daemons: %w", err) + } + + // Check if any provisioners are active (not stale) + for _, pd := range provisionerDaemons { + if pd.LastSeenAt.Valid { + age := curTime.Sub(pd.LastSeenAt.Time) + if age <= provisionerdserver.StaleInterval { + t.Logf("hasAvailableProvisioners found active provisioner, daemon_id: %+v", pd.ID) + return pd, nil + } + } + } + t.Logf("hasAvailableProvisioners: no active provisioners found") + return database.ProvisionerDaemon{}, nil +} + +func ctxWithProvisionerPermissions(ctx context.Context) context.Context { + // Use system restricted context which has permissions to update provisioner daemons + //nolint: gocritic // We need system context to modify this. + return dbauthz.AsSystemRestricted(ctx) +} + +// UpdateProvisionerLastSeenAt updates the provisioner daemon's LastSeenAt timestamp +// to the specified time to prevent it from appearing stale during autobuild operations +func UpdateProvisionerLastSeenAt(t *testing.T, db database.Store, id uuid.UUID, prev, tickTime time.Time) { + t.Helper() + ctx := ctxWithProvisionerPermissions(context.Background()) + t.Logf("Updating provisioner %s LastSeenAt from %v to %v", id, prev, tickTime) + err := db.UpdateProvisionerDaemonLastSeenAt(ctx, database.UpdateProvisionerDaemonLastSeenAtParams{ + ID: id, + LastSeenAt: sql.NullTime{Time: tickTime, Valid: true}, + }) + require.NoError(t, err) + t.Logf("Successfully updated provisioner LastSeenAt") +} + +func MustWaitForProvisioners(t *testing.T, db database.Store) { + t.Helper() + ctx := ctxWithProvisionerPermissions(testutil.Context(t, testutil.WaitShort)) + require.Eventually(t, func() bool { + daemons, err := db.GetProvisionerDaemons(ctx) + return err == nil && len(daemons) > 0 + }, testutil.WaitShort, testutil.IntervalFast) +} + +func MustWaitForProvisionersWithClient(t *testing.T, client *codersdk.Client) { + t.Helper() + ctx := ctxWithProvisionerPermissions(testutil.Context(t, testutil.WaitShort)) + require.Eventually(t, func() bool { + daemons, err := client.ProvisionerDaemons(ctx) + return err == nil && len(daemons) > 0 + }, testutil.WaitShort, testutil.IntervalFast) +} + +// mustWaitForProvisionersAvailable waits for provisioners to be available for a specific workspace +func MustWaitForProvisionersAvailable(t *testing.T, db database.Store, workspace codersdk.Workspace, staleDuration time.Duration) { + t.Helper() + ctx := ctxWithProvisionerPermissions(testutil.Context(t, testutil.WaitShort)) + + // Get the workspace from the database + require.Eventually(t, func() bool { + ws, err := db.GetWorkspaceByID(ctx, workspace.ID) + if err != nil { + return false + } + + // Get the latest build + latestBuild, err := db.GetWorkspaceBuildByID(ctx, workspace.LatestBuild.ID) + if err != nil { + return false + } + + // Get the template version job + templateVersionJob, err := db.GetProvisionerJobByID(ctx, latestBuild.JobID) + if err != nil { + return false + } + + // Check if provisioners are available using the same logic as hasAvailableProvisioners + provisionerDaemons, err := db.GetProvisionerDaemonsByOrganization(ctx, database.GetProvisionerDaemonsByOrganizationParams{ + OrganizationID: ws.OrganizationID, + WantTags: templateVersionJob.Tags, + }) + if err != nil { + return false + } + + // Check if any provisioners are active (not stale) + now := time.Now() + for _, pd := range provisionerDaemons { + if pd.LastSeenAt.Valid { + age := now.Sub(pd.LastSeenAt.Time) + if age <= staleDuration { + return true // Found an active provisioner + } + } + } + return false // No active provisioners found + }, testutil.WaitShort, testutil.IntervalFast) +} diff --git a/enterprise/coderd/workspaces_test.go b/enterprise/coderd/workspaces_test.go index 2278fb2a71939..0a008043d2f44 100644 --- a/enterprise/coderd/workspaces_test.go +++ b/enterprise/coderd/workspaces_test.go @@ -619,7 +619,7 @@ func TestWorkspaceAutobuild(t *testing.T) { failureTTL = time.Minute ) - client, user := coderdenttest.New(t, &coderdenttest.Options{ + client, db, user := coderdenttest.NewWithDatabase(t, &coderdenttest.Options{ Options: &coderdtest.Options{ Logger: &logger, AutobuildTicker: ticker, @@ -644,7 +644,12 @@ func TestWorkspaceAutobuild(t *testing.T) { ws := coderdtest.CreateWorkspace(t, client, template.ID) build := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, ws.LatestBuild.ID) require.Equal(t, codersdk.WorkspaceStatusFailed, build.Status) - ticker <- build.Job.CompletedAt.Add(failureTTL * 2) + tickTime := build.Job.CompletedAt.Add(failureTTL * 2) + + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), ws.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + ticker <- tickTime stats := <-statCh // Expect workspace to transition to stopped state for breaching // failure TTL. @@ -666,7 +671,7 @@ func TestWorkspaceAutobuild(t *testing.T) { failureTTL = time.Minute ) - client, user := coderdenttest.New(t, &coderdenttest.Options{ + client, db, user := coderdenttest.NewWithDatabase(t, &coderdenttest.Options{ Options: &coderdtest.Options{ Logger: &logger, AutobuildTicker: ticker, @@ -691,7 +696,12 @@ func TestWorkspaceAutobuild(t *testing.T) { build := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, ws.LatestBuild.ID) require.Equal(t, codersdk.WorkspaceStatusFailed, build.Status) // Make it impossible to trigger the failure TTL. - ticker <- build.Job.CompletedAt.Add(-failureTTL * 2) + tickTime := build.Job.CompletedAt.Add(-failureTTL * 2) + + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), ws.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + ticker <- tickTime stats := <-statCh // Expect no transitions since not enough time has elapsed. require.Len(t, stats.Transitions, 0) @@ -759,10 +769,11 @@ func TestWorkspaceAutobuild(t *testing.T) { client, db, user := coderdenttest.NewWithDatabase(t, &coderdenttest.Options{ Options: &coderdtest.Options{ - AutobuildTicker: ticker, - AutobuildStats: statCh, - TemplateScheduleStore: schedule.NewEnterpriseTemplateScheduleStore(agplUserQuietHoursScheduleStore(), notifications.NewNoopEnqueuer(), logger, nil), - Auditor: auditRecorder, + AutobuildTicker: ticker, + AutobuildStats: statCh, + IncludeProvisionerDaemon: true, + TemplateScheduleStore: schedule.NewEnterpriseTemplateScheduleStore(agplUserQuietHoursScheduleStore(), notifications.NewNoopEnqueuer(), logger, nil), + Auditor: auditRecorder, }, LicenseOptions: &coderdenttest.LicenseOptions{ Features: license.Features{codersdk.FeatureAdvancedTemplateScheduling: 1}, @@ -790,7 +801,12 @@ func TestWorkspaceAutobuild(t *testing.T) { auditRecorder.ResetLogs() // Simulate being inactive. - ticker <- workspace.LastUsedAt.Add(inactiveTTL * 2) + tickTime := workspace.LastUsedAt.Add(inactiveTTL * 2) + + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), workspace.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + ticker <- tickTime stats := <-statCh // Expect workspace to transition to stopped state for breaching @@ -813,7 +829,7 @@ func TestWorkspaceAutobuild(t *testing.T) { dormantLastUsedAt := ws.LastUsedAt // nolint:gocritic // this test is not testing RBAC. - err := client.UpdateWorkspaceDormancy(ctx, ws.ID, codersdk.UpdateWorkspaceDormancy{Dormant: false}) + err = client.UpdateWorkspaceDormancy(ctx, ws.ID, codersdk.UpdateWorkspaceDormancy{Dormant: false}) require.NoError(t, err) // Assert that we updated our last_used_at so that we don't immediately @@ -888,7 +904,12 @@ func TestWorkspaceAutobuild(t *testing.T) { } // Simulate being inactive. - ticker <- time.Now().Add(time.Hour) + // Fix provisioner stale issue by updating LastSeenAt to the tick time + tickTime := time.Now().Add(time.Hour) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), workspaces[0].OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + ticker <- tickTime stats := <-statCh // Expect workspace to transition to stopped state for breaching @@ -997,7 +1018,7 @@ func TestWorkspaceAutobuild(t *testing.T) { ) logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug) - client, user := coderdenttest.New(t, &coderdenttest.Options{ + client, db, user := coderdenttest.NewWithDatabase(t, &coderdenttest.Options{ Options: &coderdtest.Options{ AutobuildTicker: ticker, IncludeProvisionerDaemon: true, @@ -1029,7 +1050,11 @@ func TestWorkspaceAutobuild(t *testing.T) { ws = coderdtest.MustTransitionWorkspace(t, client, ws.ID, codersdk.WorkspaceTransitionStart, codersdk.WorkspaceTransitionStop) // Simulate not having accessed the workspace in a while. - ticker <- ws.LastUsedAt.Add(2 * inactiveTTL) + tickTime := ws.LastUsedAt.Add(2 * inactiveTTL) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), ws.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + ticker <- tickTime stats := <-statCh // Expect no transitions since workspace is stopped. require.Len(t, stats.Transitions, 0) @@ -1051,7 +1076,7 @@ func TestWorkspaceAutobuild(t *testing.T) { ) logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug) - client, user := coderdenttest.New(t, &coderdenttest.Options{ + client, db, user := coderdenttest.NewWithDatabase(t, &coderdenttest.Options{ Options: &coderdtest.Options{ AutobuildTicker: ticker, IncludeProvisionerDaemon: true, @@ -1079,7 +1104,11 @@ func TestWorkspaceAutobuild(t *testing.T) { require.Equal(t, codersdk.WorkspaceStatusRunning, build.Status) // Simulate not having accessed the workspace in a while. - ticker <- ws.LastUsedAt.Add(2 * transitionTTL) + tickTime := ws.LastUsedAt.Add(2 * transitionTTL) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), ws.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + ticker <- tickTime stats := <-statCh // Expect workspace to transition to stopped state for breaching // inactive TTL. @@ -1094,7 +1123,9 @@ func TestWorkspaceAutobuild(t *testing.T) { _ = coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, ws.LatestBuild.ID) // Simulate the workspace being dormant beyond the threshold. - ticker <- ws.DormantAt.Add(2 * transitionTTL) + tickTime2 := ws.DormantAt.Add(2 * transitionTTL) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + ticker <- tickTime2 stats = <-statCh require.Len(t, stats.Transitions, 1) // The workspace should be scheduled for deletion. @@ -1106,7 +1137,7 @@ func TestWorkspaceAutobuild(t *testing.T) { // Assert that the workspace is actually deleted. //nolint:gocritic // ensuring workspace is deleted and not just invisible to us due to RBAC - _, err := client.Workspace(testutil.Context(t, testutil.WaitShort), ws.ID) + _, err = client.Workspace(testutil.Context(t, testutil.WaitShort), ws.ID) require.Error(t, err) cerr, ok := codersdk.AsError(err) require.True(t, ok) @@ -1123,7 +1154,7 @@ func TestWorkspaceAutobuild(t *testing.T) { ) logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug) - client, user := coderdenttest.New(t, &coderdenttest.Options{ + client, db, user := coderdenttest.NewWithDatabase(t, &coderdenttest.Options{ Options: &coderdtest.Options{ AutobuildTicker: ticker, IncludeProvisionerDaemon: true, @@ -1158,7 +1189,11 @@ func TestWorkspaceAutobuild(t *testing.T) { require.NotNil(t, ws.DormantAt) // Ensure we haven't breached our threshold. - ticker <- ws.DormantAt.Add(-dormantTTL * 2) + tickTime := ws.DormantAt.Add(-dormantTTL * 2) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), ws.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + ticker <- tickTime stats := <-statCh // Expect no transitions since not enough time has elapsed. require.Len(t, stats.Transitions, 0) @@ -1169,7 +1204,9 @@ func TestWorkspaceAutobuild(t *testing.T) { require.NoError(t, err) // Simlute the workspace breaching the threshold. - ticker <- ws.DormantAt.Add(dormantTTL * 2) + tickTime2 := ws.DormantAt.Add(dormantTTL * 2) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime2) + ticker <- tickTime2 stats = <-statCh require.Len(t, stats.Transitions, 1) require.Equal(t, database.WorkspaceTransitionDelete, stats.Transitions[ws.ID]) @@ -1186,7 +1223,7 @@ func TestWorkspaceAutobuild(t *testing.T) { ) logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug) - client, user := coderdenttest.New(t, &coderdenttest.Options{ + client, db, user := coderdenttest.NewWithDatabase(t, &coderdenttest.Options{ Options: &coderdtest.Options{ AutobuildTicker: tickCh, IncludeProvisionerDaemon: true, @@ -1217,7 +1254,11 @@ func TestWorkspaceAutobuild(t *testing.T) { ws = coderdtest.MustTransitionWorkspace(t, client, ws.ID, codersdk.WorkspaceTransitionStart, codersdk.WorkspaceTransitionStop) // Assert that autostart works when the workspace isn't dormant.. - tickCh <- sched.Next(ws.LatestBuild.CreatedAt) + tickTime := sched.Next(ws.LatestBuild.CreatedAt) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), ws.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + tickCh <- tickTime stats := <-statsCh require.Len(t, stats.Errors, 0) require.Len(t, stats.Transitions, 1) @@ -1237,7 +1278,9 @@ func TestWorkspaceAutobuild(t *testing.T) { require.NoError(t, err) // We should see the workspace get stopped now. - tickCh <- ws.LastUsedAt.Add(inactiveTTL * 2) + tickTime2 := ws.LastUsedAt.Add(inactiveTTL * 2) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + tickCh <- tickTime2 stats = <-statsCh require.Len(t, stats.Errors, 0) require.Len(t, stats.Transitions, 1) @@ -1267,7 +1310,7 @@ func TestWorkspaceAutobuild(t *testing.T) { ) logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug) - client, user := coderdenttest.New(t, &coderdenttest.Options{ + client, db, user := coderdenttest.NewWithDatabase(t, &coderdenttest.Options{ Options: &coderdtest.Options{ AutobuildTicker: ticker, IncludeProvisionerDaemon: true, @@ -1335,13 +1378,19 @@ func TestWorkspaceAutobuild(t *testing.T) { // Simulate ticking an hour after the workspace is expected to be deleted. // Under normal circumstances this should result in a transition but // since our last build resulted in failure it should be skipped. - ticker <- build.Job.CompletedAt.Add(time.Hour) + tickTime := build.Job.CompletedAt.Add(time.Hour) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), ws.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + ticker <- tickTime stats := <-statCh require.Len(t, stats.Transitions, 0) // Simulate ticking a day after the workspace was last attempted to // be deleted. This should result in an attempt. - ticker <- build.Job.CompletedAt.Add(time.Hour * 25) + tickTime2 := build.Job.CompletedAt.Add(time.Hour * 25) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime2) + ticker <- tickTime2 stats = <-statCh require.Len(t, stats.Transitions, 1) require.Equal(t, database.WorkspaceTransitionDelete, stats.Transitions[ws.ID]) @@ -1356,7 +1405,7 @@ func TestWorkspaceAutobuild(t *testing.T) { ) logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug) - client, user := coderdenttest.New(t, &coderdenttest.Options{ + client, db, user := coderdenttest.NewWithDatabase(t, &coderdenttest.Options{ Options: &coderdtest.Options{ AutobuildTicker: tickCh, IncludeProvisionerDaemon: true, @@ -1401,7 +1450,11 @@ func TestWorkspaceAutobuild(t *testing.T) { require.NoError(t, err) // Kick of an autostart build. - tickCh <- sched.Next(ws.LatestBuild.CreatedAt) + tickTime := sched.Next(ws.LatestBuild.CreatedAt) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), ws.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + tickCh <- tickTime stats := <-statsCh require.Len(t, stats.Errors, 0) require.Len(t, stats.Transitions, 1) @@ -1429,7 +1482,9 @@ func TestWorkspaceAutobuild(t *testing.T) { }) // Force an autostart transition again. - tickCh <- sched.Next(firstBuild.CreatedAt) + tickTime2 := sched.Next(firstBuild.CreatedAt) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + tickCh <- tickTime2 stats = <-statsCh require.Len(t, stats.Errors, 0) require.Len(t, stats.Transitions, 1) @@ -1453,7 +1508,7 @@ func TestWorkspaceAutobuild(t *testing.T) { clock.Set(dbtime.Now()) logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug) - client, user := coderdenttest.New(t, &coderdenttest.Options{ + client, db, user := coderdenttest.NewWithDatabase(t, &coderdenttest.Options{ Options: &coderdtest.Options{ AutobuildTicker: tickCh, IncludeProvisionerDaemon: true, @@ -1494,6 +1549,9 @@ func TestWorkspaceAutobuild(t *testing.T) { next = sched.Next(next) clock.Set(next) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), ws.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), next) tickCh <- next stats := <-statsCh ws = coderdtest.MustWorkspace(t, client, ws.ID) @@ -2207,11 +2265,20 @@ func TestExecutorPrebuilds(t *testing.T) { // Then: the claimed workspace should inherit and respect that same NextStartAt require.True(t, workspace.NextStartAt.Equal(*prebuild.NextStartAt)) - // Tick at the next scheduled time after the prebuild’s LatestBuild.CreatedAt, - // since the next allowed autostart is calculated starting from that point. - // When: the autobuild executor ticks after the scheduled time + // Wait for provisioner to be registered + coderdtest.MustWaitForProvisioners(t, db) + + // Wait for provisioner to be available for this specific workspace + coderdtest.MustWaitForProvisionersAvailable(t, db, prebuild, provisionerdserver.StaleInterval) + + tickTime := sched.Next(prebuild.LatestBuild.CreatedAt).Add(time.Minute) + p, err := coderdtest.GetProvisionerForWorkspace(t, db, time.Now(), workspace.OrganizationID, database.ProvisionerJob{}) + require.NoError(t, err) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, time.Now(), tickTime) + + // When: the autobuild executor ticks at the scheduled time go func() { - tickCh <- sched.Next(prebuild.LatestBuild.CreatedAt).Add(time.Minute) + tickCh <- tickTime }() // Then: the workspace should have a NextStartAt equal to the next autostart schedule