Skip to content

Commit ff46917

Browse files
mafredrimafredri
authored
feat: add retention config for workspace_agent_logs (#21039)
Replace hardcoded 7-day retention for workspace agent logs with configurable retention from deployment settings. Defaults to 7d to preserve existing behavior. Depends on #21038 Updates #20743
1 parent d9888ce commit ff46917

File tree

20 files changed

+224
-45
lines changed

20 files changed

+224
-45
lines changed

‎cli/testdata/coder_server_--help.golden‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -717,6 +717,12 @@ that data type.
717717
How long connection log entries are retained. Set to 0 to disable
718718
(keep indefinitely).
719719

720+
--workspace-agent-logs-retention duration, $CODER_WORKSPACE_AGENT_LOGS_RETENTION (default: 7d)
721+
How long workspace agent logs are retained. Logs from non-latest
722+
builds are deleted if the agent hasn't connected within this period.
723+
Logs from the latest build are always retained. Set to 0 to disable
724+
automatic deletion.
725+
720726
TELEMETRY OPTIONS:
721727
Telemetry is critical to our ability to improve Coder. We strip all personal
722728
information before sending data to our servers. Please only disable telemetry

‎cli/testdata/server-config.yaml.golden‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -761,3 +761,8 @@ retention:
761761
# an expired key. Set to 0 to disable automatic deletion of expired keys.
762762
# (default: 7d, type: duration)
763763
api_keys: 168h0m0s
764+
# How long workspace agent logs are retained. Logs from non-latest builds are
765+
# deleted if the agent hasn't connected within this period. Logs from the latest
766+
# build are always retained. Set to 0 to disable automatic deletion.
767+
# (default: 7d, type: duration)
768+
workspace_agent_logs: 168h0m0s

‎coderd/apidoc/docs.go‎

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/apidoc/swagger.json‎

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/dbauthz/dbauthz.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1784,9 +1784,9 @@ func (q *querier) DeleteOldTelemetryLocks(ctx context.Context, beforeTime time.T
17841784
return q.db.DeleteOldTelemetryLocks(ctx, beforeTime)
17851785
}
17861786

1787-
func (q *querier) DeleteOldWorkspaceAgentLogs(ctx context.Context, threshold time.Time) error {
1787+
func (q *querier) DeleteOldWorkspaceAgentLogs(ctx context.Context, threshold time.Time) (int64, error) {
17881788
if err := q.authorizeContext(ctx, policy.ActionDelete, rbac.ResourceSystem); err != nil {
1789-
return err
1789+
return 0, err
17901790
}
17911791
return q.db.DeleteOldWorkspaceAgentLogs(ctx, threshold)
17921792
}

‎coderd/database/dbauthz/dbauthz_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3227,7 +3227,7 @@ func (s *MethodTestSuite) TestSystemFunctions() {
32273227
}))
32283228
s.Run("DeleteOldWorkspaceAgentLogs", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
32293229
t := time.Time{}
3230-
dbm.EXPECT().DeleteOldWorkspaceAgentLogs(gomock.Any(), t).Return(nil).AnyTimes()
3230+
dbm.EXPECT().DeleteOldWorkspaceAgentLogs(gomock.Any(), t).Return(int64(0), nil).AnyTimes()
32313231
check.Args(t).Asserts(rbac.ResourceSystem, policy.ActionDelete)
32323232
}))
32333233
s.Run("InsertWorkspaceAgentStats", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {

‎coderd/database/dbmetrics/querymetrics.go‎

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/dbmock/dbmock.go‎

Lines changed: 4 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎coderd/database/dbpurge/dbpurge.go‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@ import (
1818
)
1919

2020
const (
21-
delay = 10 * time.Minute
22-
maxAgentLogAge = 7 * 24 * time.Hour
21+
delay = 10 * time.Minute
2322
// Connection events are now inserted into the `connection_logs` table.
2423
// We'll slowly remove old connection events from the `audit_logs` table.
2524
// The `connection_logs` table is purged based on the configured retention.
@@ -66,9 +65,14 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, vals *coder
6665
return nil
6766
}
6867

69-
deleteOldWorkspaceAgentLogsBefore := start.Add(-maxAgentLogAge)
70-
if err := tx.DeleteOldWorkspaceAgentLogs(ctx, deleteOldWorkspaceAgentLogsBefore); err != nil {
71-
return xerrors.Errorf("failed to delete old workspace agent logs: %w", err)
68+
var purgedWorkspaceAgentLogs int64
69+
workspaceAgentLogsRetention := vals.Retention.WorkspaceAgentLogs.Value()
70+
if workspaceAgentLogsRetention > 0 {
71+
deleteOldWorkspaceAgentLogsBefore := start.Add(-workspaceAgentLogsRetention)
72+
purgedWorkspaceAgentLogs, err = tx.DeleteOldWorkspaceAgentLogs(ctx, deleteOldWorkspaceAgentLogsBefore)
73+
if err != nil {
74+
return xerrors.Errorf("failed to delete old workspace agent logs: %w", err)
75+
}
7276
}
7377
if err := tx.DeleteOldWorkspaceAgentStats(ctx); err != nil {
7478
return xerrors.Errorf("failed to delete old workspace agent stats: %w", err)
@@ -148,6 +152,7 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, vals *coder
148152
}
149153

150154
logger.Debug(ctx, "purged old database entries",
155+
slog.F("workspace_agent_logs", purgedWorkspaceAgentLogs),
151156
slog.F("expired_api_keys", expiredAPIKeys),
152157
slog.F("aibridge_records", purgedAIBridgeRecords),
153158
slog.F("connection_logs", purgedConnectionLogs),

‎coderd/database/dbpurge/dbpurge_test.go‎

Lines changed: 89 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,11 @@ func TestDeleteOldWorkspaceAgentLogs(t *testing.T) {
246246
// After dbpurge completes, the ticker is reset. Trap this call.
247247

248248
done := awaitDoTick(ctx, t, clk)
249-
closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{}, clk)
249+
closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{
250+
Retention: codersdk.RetentionConfig{
251+
WorkspaceAgentLogs: serpent.Duration(7 * 24 * time.Hour),
252+
},
253+
}, clk)
250254
defer closer.Close()
251255
<-done // doTick() has now run.
252256

@@ -392,6 +396,90 @@ func mustCreateAgentLogs(ctx context.Context, t *testing.T, db database.Store, a
392396
require.NotEmpty(t, agentLogs, "agent logs must be present")
393397
}
394398

399+
func TestDeleteOldWorkspaceAgentLogsRetention(t *testing.T) {
400+
t.Parallel()
401+
402+
now := time.Date(2025, 1, 15, 7, 30, 0, 0, time.UTC)
403+
404+
testCases := []struct {
405+
name string
406+
retentionConfig codersdk.RetentionConfig
407+
logsAge time.Duration
408+
expectDeleted bool
409+
}{
410+
{
411+
name: "RetentionEnabled",
412+
retentionConfig: codersdk.RetentionConfig{
413+
WorkspaceAgentLogs: serpent.Duration(7 * 24 * time.Hour), // 7 days
414+
},
415+
logsAge: 8 * 24 * time.Hour, // 8 days ago
416+
expectDeleted: true,
417+
},
418+
{
419+
name: "RetentionDisabled",
420+
retentionConfig: codersdk.RetentionConfig{
421+
WorkspaceAgentLogs: serpent.Duration(0),
422+
},
423+
logsAge: 60 * 24 * time.Hour, // 60 days ago
424+
expectDeleted: false,
425+
},
426+
427+
{
428+
name: "CustomRetention30Days",
429+
retentionConfig: codersdk.RetentionConfig{
430+
WorkspaceAgentLogs: serpent.Duration(30 * 24 * time.Hour), // 30 days
431+
},
432+
logsAge: 31 * 24 * time.Hour, // 31 days ago
433+
expectDeleted: true,
434+
},
435+
}
436+
437+
for _, tc := range testCases {
438+
t.Run(tc.name, func(t *testing.T) {
439+
t.Parallel()
440+
441+
ctx := testutil.Context(t, testutil.WaitShort)
442+
clk := quartz.NewMock(t)
443+
clk.Set(now).MustWait(ctx)
444+
445+
oldTime := now.Add(-tc.logsAge)
446+
447+
db, _ := dbtestutil.NewDB(t, dbtestutil.WithDumpOnFailure())
448+
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
449+
org := dbgen.Organization(t, db, database.Organization{})
450+
user := dbgen.User(t, db, database.User{})
451+
_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user.ID, OrganizationID: org.ID})
452+
tv := dbgen.TemplateVersion(t, db, database.TemplateVersion{OrganizationID: org.ID, CreatedBy: user.ID})
453+
tmpl := dbgen.Template(t, db, database.Template{OrganizationID: org.ID, ActiveVersionID: tv.ID, CreatedBy: user.ID})
454+
455+
ws := dbgen.Workspace(t, db, database.WorkspaceTable{Name: "test-ws", OwnerID: user.ID, OrganizationID: org.ID, TemplateID: tmpl.ID})
456+
wb1 := mustCreateWorkspaceBuild(t, db, org, tv, ws.ID, oldTime, 1)
457+
wb2 := mustCreateWorkspaceBuild(t, db, org, tv, ws.ID, oldTime, 2)
458+
agent1 := mustCreateAgent(t, db, wb1)
459+
agent2 := mustCreateAgent(t, db, wb2)
460+
mustCreateAgentLogs(ctx, t, db, agent1, &oldTime, "agent 1 logs")
461+
mustCreateAgentLogs(ctx, t, db, agent2, &oldTime, "agent 2 logs")
462+
463+
// Run the purge.
464+
done := awaitDoTick(ctx, t, clk)
465+
closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{
466+
Retention: tc.retentionConfig,
467+
}, clk)
468+
defer closer.Close()
469+
testutil.TryReceive(ctx, t, done)
470+
471+
// Verify results.
472+
if tc.expectDeleted {
473+
assertNoWorkspaceAgentLogs(ctx, t, db, agent1.ID)
474+
} else {
475+
assertWorkspaceAgentLogs(ctx, t, db, agent1.ID, "agent 1 logs")
476+
}
477+
// Latest build logs are always retained.
478+
assertWorkspaceAgentLogs(ctx, t, db, agent2.ID, "agent 2 logs")
479+
})
480+
}
481+
}
482+
395483
//nolint:paralleltest // It uses LockIDDBPurge.
396484
func TestDeleteOldProvisionerDaemons(t *testing.T) {
397485
// TODO: must refactor DeleteOldProvisionerDaemons to allow passing in cutoff

0 commit comments

Comments
 (0)