Skip to content

Commit 95e8a6c

Browse files
mafredrimafredri
committed
feat(coderd/database/dbpurge): add retention for workspace agent logs
Replace hardcoded 7-day retention for workspace agent logs with configurable retention from deployment settings. Falls back to global retention when not set, and skips deletion entirely when effective retention is 0. Depends on #21038 Updates #20743
1 parent 7e8a005 commit 95e8a6c

File tree

10 files changed

+2261
-2012
lines changed

10 files changed

+2261
-2012
lines changed

‎cli/testdata/coder_server_--help.golden‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -725,6 +725,11 @@ that data type.
725725
disable (data is kept indefinitely unless individual settings are
726726
configured).
727727

728+
--workspace-agent-logs-retention duration, $CODER_WORKSPACE_AGENT_LOGS_RETENTION (default: 7d)
729+
How long workspace agent logs are retained. Logs from non-latest
730+
workspace builds are deleted after this period to free up storage
731+
space. Set to 0 to disable automatic deletion of workspace agent logs.
732+
728733
TELEMETRY OPTIONS:
729734
Telemetry is critical to our ability to improve Coder. We strip all personal
730735
information before sending data to our servers. Please only disable telemetry

‎cli/testdata/server-config.yaml.golden‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -780,3 +780,8 @@ retention:
780780
# an expired key. Set to 0 to disable automatic deletion of expired keys.
781781
# (default: 7d, type: duration)
782782
api_keys: 168h0m0s
783+
# How long workspace agent logs are retained. Logs from non-latest workspace
784+
# builds are deleted after this period to free up storage space. Set to 0 to
785+
# disable automatic deletion of workspace agent logs.
786+
# (default: 7d, type: duration)
787+
workspace_agent_logs: 168h0m0s

‎coderd/database/dbpurge/dbpurge.go‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,7 @@ import (
1818
)
1919

2020
const (
21-
delay = 10 * time.Minute
22-
maxAgentLogAge = 7 * 24 * time.Hour
21+
delay = 10 * time.Minute
2322
// Connection events are now inserted into the `connection_logs` table.
2423
// We'll slowly remove old connection events from the `audit_logs` table,
2524
// but we won't touch the `connection_logs` table.
@@ -67,9 +66,15 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, vals *coder
6766
return nil
6867
}
6968

70-
deleteOldWorkspaceAgentLogsBefore := start.Add(-maxAgentLogAge)
71-
if err := tx.DeleteOldWorkspaceAgentLogs(ctx, deleteOldWorkspaceAgentLogsBefore); err != nil {
72-
return xerrors.Errorf("failed to delete old workspace agent logs: %w", err)
69+
workspaceAgentLogsRetention := vals.Retention.WorkspaceAgentLogs.Value()
70+
if workspaceAgentLogsRetention == 0 {
71+
workspaceAgentLogsRetention = vals.Retention.Global.Value()
72+
}
73+
if workspaceAgentLogsRetention > 0 {
74+
deleteOldWorkspaceAgentLogsBefore := start.Add(-workspaceAgentLogsRetention)
75+
if err := tx.DeleteOldWorkspaceAgentLogs(ctx, deleteOldWorkspaceAgentLogsBefore); err != nil {
76+
return xerrors.Errorf("failed to delete old workspace agent logs: %w", err)
77+
}
7378
}
7479
if err := tx.DeleteOldWorkspaceAgentStats(ctx); err != nil {
7580
return xerrors.Errorf("failed to delete old workspace agent stats: %w", err)

‎coderd/database/dbpurge/dbpurge_test.go‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,135 @@ func mustCreateAgentLogs(ctx context.Context, t *testing.T, db database.Store, a
392392
require.NotEmpty(t, agentLogs, "agent logs must be present")
393393
}
394394

395+
func TestDeleteOldWorkspaceAgentLogsRetention(t *testing.T) {
396+
t.Parallel()
397+
398+
t.Run("RetentionEnabled", func(t *testing.T) {
399+
t.Parallel()
400+
401+
ctx := testutil.Context(t, testutil.WaitShort)
402+
403+
clk := quartz.NewMock(t)
404+
now := time.Date(2025, 1, 15, 7, 30, 0, 0, time.UTC)
405+
retentionPeriod := 7 * 24 * time.Hour
406+
oldTime := now.Add(-retentionPeriod).Add(-24 * time.Hour) // 8 days ago (should be deleted)
407+
clk.Set(now).MustWait(ctx)
408+
409+
db, _ := dbtestutil.NewDB(t, dbtestutil.WithDumpOnFailure())
410+
org := dbgen.Organization(t, db, database.Organization{})
411+
user := dbgen.User(t, db, database.User{})
412+
_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user.ID, OrganizationID: org.ID})
413+
tv := dbgen.TemplateVersion(t, db, database.TemplateVersion{OrganizationID: org.ID, CreatedBy: user.ID})
414+
tmpl := dbgen.Template(t, db, database.Template{OrganizationID: org.ID, ActiveVersionID: tv.ID, CreatedBy: user.ID})
415+
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
416+
417+
ws := dbgen.Workspace(t, db, database.WorkspaceTable{Name: "test-ws", OwnerID: user.ID, OrganizationID: org.ID, TemplateID: tmpl.ID})
418+
wb1 := mustCreateWorkspaceBuild(t, db, org, tv, ws.ID, oldTime, 1)
419+
wb2 := mustCreateWorkspaceBuild(t, db, org, tv, ws.ID, oldTime, 2)
420+
agent1 := mustCreateAgent(t, db, wb1)
421+
agent2 := mustCreateAgent(t, db, wb2)
422+
mustCreateAgentLogs(ctx, t, db, agent1, &oldTime, "agent 1 logs")
423+
mustCreateAgentLogs(ctx, t, db, agent2, &oldTime, "agent 2 logs")
424+
425+
done := awaitDoTick(ctx, t, clk)
426+
closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{
427+
Retention: codersdk.RetentionConfig{
428+
WorkspaceAgentLogs: serpent.Duration(retentionPeriod),
429+
},
430+
}, clk)
431+
defer closer.Close()
432+
testutil.TryReceive(ctx, t, done)
433+
434+
// Non-latest build logs should be deleted.
435+
assertNoWorkspaceAgentLogs(ctx, t, db, agent1.ID)
436+
// Latest build logs should be retained.
437+
assertWorkspaceAgentLogs(ctx, t, db, agent2.ID, "agent 2 logs")
438+
})
439+
440+
t.Run("RetentionDisabled", func(t *testing.T) {
441+
t.Parallel()
442+
443+
ctx := testutil.Context(t, testutil.WaitShort)
444+
445+
clk := quartz.NewMock(t)
446+
now := time.Date(2025, 1, 15, 7, 30, 0, 0, time.UTC)
447+
oldTime := now.Add(-60 * 24 * time.Hour) // 60 days ago
448+
clk.Set(now).MustWait(ctx)
449+
450+
db, _ := dbtestutil.NewDB(t, dbtestutil.WithDumpOnFailure())
451+
org := dbgen.Organization(t, db, database.Organization{})
452+
user := dbgen.User(t, db, database.User{})
453+
_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user.ID, OrganizationID: org.ID})
454+
tv := dbgen.TemplateVersion(t, db, database.TemplateVersion{OrganizationID: org.ID, CreatedBy: user.ID})
455+
tmpl := dbgen.Template(t, db, database.Template{OrganizationID: org.ID, ActiveVersionID: tv.ID, CreatedBy: user.ID})
456+
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
457+
458+
ws := dbgen.Workspace(t, db, database.WorkspaceTable{Name: "test-ws", OwnerID: user.ID, OrganizationID: org.ID, TemplateID: tmpl.ID})
459+
wb1 := mustCreateWorkspaceBuild(t, db, org, tv, ws.ID, oldTime, 1)
460+
wb2 := mustCreateWorkspaceBuild(t, db, org, tv, ws.ID, oldTime, 2)
461+
agent1 := mustCreateAgent(t, db, wb1)
462+
agent2 := mustCreateAgent(t, db, wb2)
463+
mustCreateAgentLogs(ctx, t, db, agent1, &oldTime, "agent 1 logs")
464+
mustCreateAgentLogs(ctx, t, db, agent2, &oldTime, "agent 2 logs")
465+
466+
done := awaitDoTick(ctx, t, clk)
467+
closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{
468+
Retention: codersdk.RetentionConfig{
469+
WorkspaceAgentLogs: serpent.Duration(0), // disabled
470+
},
471+
}, clk)
472+
defer closer.Close()
473+
testutil.TryReceive(ctx, t, done)
474+
475+
// All logs should be retained when retention is disabled.
476+
assertWorkspaceAgentLogs(ctx, t, db, agent1.ID, "agent 1 logs")
477+
assertWorkspaceAgentLogs(ctx, t, db, agent2.ID, "agent 2 logs")
478+
})
479+
480+
t.Run("GlobalRetentionFallback", func(t *testing.T) {
481+
t.Parallel()
482+
483+
ctx := testutil.Context(t, testutil.WaitShort)
484+
485+
clk := quartz.NewMock(t)
486+
now := time.Date(2025, 1, 15, 7, 30, 0, 0, time.UTC)
487+
retentionPeriod := 14 * 24 * time.Hour
488+
oldTime := now.Add(-retentionPeriod).Add(-24 * time.Hour) // 15 days ago
489+
clk.Set(now).MustWait(ctx)
490+
491+
db, _ := dbtestutil.NewDB(t, dbtestutil.WithDumpOnFailure())
492+
org := dbgen.Organization(t, db, database.Organization{})
493+
user := dbgen.User(t, db, database.User{})
494+
_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{UserID: user.ID, OrganizationID: org.ID})
495+
tv := dbgen.TemplateVersion(t, db, database.TemplateVersion{OrganizationID: org.ID, CreatedBy: user.ID})
496+
tmpl := dbgen.Template(t, db, database.Template{OrganizationID: org.ID, ActiveVersionID: tv.ID, CreatedBy: user.ID})
497+
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
498+
499+
ws := dbgen.Workspace(t, db, database.WorkspaceTable{Name: "test-ws", OwnerID: user.ID, OrganizationID: org.ID, TemplateID: tmpl.ID})
500+
wb1 := mustCreateWorkspaceBuild(t, db, org, tv, ws.ID, oldTime, 1)
501+
wb2 := mustCreateWorkspaceBuild(t, db, org, tv, ws.ID, oldTime, 2)
502+
agent1 := mustCreateAgent(t, db, wb1)
503+
agent2 := mustCreateAgent(t, db, wb2)
504+
mustCreateAgentLogs(ctx, t, db, agent1, &oldTime, "agent 1 logs")
505+
mustCreateAgentLogs(ctx, t, db, agent2, &oldTime, "agent 2 logs")
506+
507+
done := awaitDoTick(ctx, t, clk)
508+
closer := dbpurge.New(ctx, logger, db, &codersdk.DeploymentValues{
509+
Retention: codersdk.RetentionConfig{
510+
Global: serpent.Duration(retentionPeriod), // Use global
511+
WorkspaceAgentLogs: serpent.Duration(0), // Not set, falls back to global
512+
},
513+
}, clk)
514+
defer closer.Close()
515+
testutil.TryReceive(ctx, t, done)
516+
517+
// Non-latest build logs should be deleted via global retention.
518+
assertNoWorkspaceAgentLogs(ctx, t, db, agent1.ID)
519+
// Latest build logs should be retained.
520+
assertWorkspaceAgentLogs(ctx, t, db, agent2.ID, "agent 2 logs")
521+
})
522+
}
523+
395524
//nolint:paralleltest // It uses LockIDDBPurge.
396525
func TestDeleteOldProvisionerDaemons(t *testing.T) {
397526
// TODO: must refactor DeleteOldProvisionerDaemons to allow passing in cutoff

‎codersdk/deployment.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -834,6 +834,9 @@ type RetentionConfig struct {
834834
// Keys are only deleted if they have been expired for at least this duration.
835835
// Defaults to 7 days to preserve existing behavior.
836836
APIKeys serpent.Duration `json:"api_keys" typescript:",notnull"`
837+
// WorkspaceAgentLogs controls how long workspace agent logs are retained.
838+
// Defaults to 7 days to preserve existing behavior.
839+
WorkspaceAgentLogs serpent.Duration `json:"workspace_agent_logs" typescript:",notnull"`
837840
}
838841

839842
type NotificationsConfig struct {
@@ -3436,6 +3439,17 @@ Write out the current server config as YAML to stdout.`,
34363439
YAML: "api_keys",
34373440
Annotations: serpent.Annotations{}.Mark(annotationFormatDuration, "true"),
34383441
},
3442+
{
3443+
Name: "Workspace Agent Logs Retention",
3444+
Description: "How long workspace agent logs are retained. Logs from non-latest workspace builds are deleted after this period to free up storage space. Set to 0 to disable automatic deletion of workspace agent logs.",
3445+
Flag: "workspace-agent-logs-retention",
3446+
Env: "CODER_WORKSPACE_AGENT_LOGS_RETENTION",
3447+
Value: &c.Retention.WorkspaceAgentLogs,
3448+
Default: "7d",
3449+
Group: &deploymentGroupRetention,
3450+
YAML: "workspace_agent_logs",
3451+
Annotations: serpent.Annotations{}.Mark(annotationFormatDuration, "true"),
3452+
},
34393453
{
34403454
Name: "Enable Authorization Recordings",
34413455
Description: "All api requests will have a header including all authorization calls made during the request. " +

‎docs/admin/setup/data-retention.md‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# Data Retention
22

33
Coder supports configurable retention policies that automatically purge old
4-
Audit Logs, Connection Logs, and API keys. These policies help manage database
5-
growth by removing records older than a specified duration.
4+
Audit Logs, Connection Logs, Workspace Agent Logs, and API keys. These policies
5+
help manage database growth by removing records older than a specified duration.
66

77
## Overview
88

@@ -25,12 +25,13 @@ a YAML configuration file.
2525

2626
### Settings
2727

28-
| Setting | CLI Flag | Environment Variable | Default | Description |
29-
|-----------------|-------------------------------|-----------------------------------|------------------|--------------------------------------------------------------------------|
30-
| Global | `--global-retention` | `CODER_GLOBAL_RETENTION` | `0` (disabled) | Default retention for all data types. Individual settings override this. |
31-
| Audit Logs | `--audit-logs-retention` | `CODER_AUDIT_LOGS_RETENTION` | `0` (use global) | How long to retain Audit Log entries. |
32-
| Connection Logs | `--connection-logs-retention` | `CODER_CONNECTION_LOGS_RETENTION` | `0` (use global) | How long to retain Connection Log entries. |
33-
| API Keys | `--api-keys-retention` | `CODER_API_KEYS_RETENTION` | `7d` | How long to retain expired API keys. |
28+
| Setting | CLI Flag | Environment Variable | Default | Description |
29+
|----------------------|------------------------------------|----------------------------------------|------------------|--------------------------------------------------------------------------|
30+
| Global | `--global-retention` | `CODER_GLOBAL_RETENTION` | `0` (disabled) | Default retention for all data types. Individual settings override this. |
31+
| Audit Logs | `--audit-logs-retention` | `CODER_AUDIT_LOGS_RETENTION` | `0` (use global) | How long to retain Audit Log entries. |
32+
| Connection Logs | `--connection-logs-retention` | `CODER_CONNECTION_LOGS_RETENTION` | `0` (use global) | How long to retain Connection Log entries. |
33+
| API Keys | `--api-keys-retention` | `CODER_API_KEYS_RETENTION` | `7d` | How long to retain expired API keys. |
34+
| Workspace Agent Logs | `--workspace-agent-logs-retention` | `CODER_WORKSPACE_AGENT_LOGS_RETENTION` | `7d` | How long to retain workspace agent logs. |
3435

3536
### Duration Format
3637

@@ -68,6 +69,7 @@ retention:
6869
audit_logs: 365d
6970
connection_logs: 0s
7071
api_keys: 7d
72+
workspace_agent_logs: 7d
7173
```
7274
7375
## How Retention Works
@@ -103,6 +105,16 @@ ago. Active keys are never deleted by the retention policy.
103105
Keeping expired keys for a short period allows Coder to return a more helpful
104106
error message when users attempt to use an expired key.
105107

108+
### Workspace Agent Logs Behavior
109+
110+
Workspace agent logs are retained based on the retention period, but **logs from
111+
the latest build of each workspace are always retained** regardless of age. This
112+
ensures you can always debug issues with active workspaces.
113+
114+
Only logs from non-latest workspace builds that are older than the retention
115+
period are deleted. Setting `--workspace-agent-logs-retention=7d` keeps all logs
116+
from the latest build plus logs from previous builds for up to 7 days.
117+
106118
## Best Practices
107119

108120
### Recommended Starting Configuration
@@ -115,6 +127,7 @@ retention:
115127
audit_logs: 365d
116128
connection_logs: 0s # Use global
117129
api_keys: 7d
130+
workspace_agent_logs: 7d
118131
```
119132

120133
### Compliance Considerations
@@ -160,6 +173,7 @@ retention:
160173
audit_logs: 0s
161174
connection_logs: 0s
162175
api_keys: 0s
176+
workspace_agent_logs: 0s
163177
```
164178

165179
There is no way to disable retention for a specific data type while global

0 commit comments

Comments
 (0)