diff --git a/cli/testdata/coder_list_--output_json.golden b/cli/testdata/coder_list_--output_json.golden index 822998329be5b..ba560a39f59d7 100644 --- a/cli/testdata/coder_list_--output_json.golden +++ b/cli/testdata/coder_list_--output_json.golden @@ -55,7 +55,8 @@ "template_name": "", "template_display_name": "", "template_icon": "" - } + }, + "logs_overflowed": false }, "reason": "initiator", "resources": [], diff --git a/cli/testdata/coder_provisioner_jobs_list_--help.golden b/cli/testdata/coder_provisioner_jobs_list_--help.golden index f380a0334867c..8e22f78e978f2 100644 --- a/cli/testdata/coder_provisioner_jobs_list_--help.golden +++ b/cli/testdata/coder_provisioner_jobs_list_--help.golden @@ -11,7 +11,7 @@ OPTIONS: -O, --org string, $CODER_ORGANIZATION Select which organization (uuid or name) to use. - -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|organization|queue] (default: created at,id,type,template display name,status,queue,tags) + -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|logs overflowed|organization|queue] (default: created at,id,type,template display name,status,queue,tags) Columns to display in table output. -l, --limit int, $CODER_PROVISIONER_JOB_LIST_LIMIT (default: 50) diff --git a/cli/testdata/coder_provisioner_jobs_list_--output_json.golden b/cli/testdata/coder_provisioner_jobs_list_--output_json.golden index e36723765b4df..6ccf672360a55 100644 --- a/cli/testdata/coder_provisioner_jobs_list_--output_json.golden +++ b/cli/testdata/coder_provisioner_jobs_list_--output_json.golden @@ -26,6 +26,7 @@ "template_display_name": "", "template_icon": "" }, + "logs_overflowed": false, "organization_name": "Coder" }, { @@ -57,6 +58,7 @@ "workspace_id": "===========[workspace ID]===========", "workspace_name": "test-workspace" }, + "logs_overflowed": false, "organization_name": "Coder" } ] diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index 7c723994d38d2..7087b5dcc977f 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -15261,6 +15261,9 @@ const docTemplate = `{ "input": { "$ref": "#/definitions/codersdk.ProvisionerJobInput" }, + "logs_overflowed": { + "type": "boolean" + }, "metadata": { "$ref": "#/definitions/codersdk.ProvisionerJobMetadata" }, diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 28a38ffd32d70..3a1443cd76dec 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -13852,6 +13852,9 @@ "input": { "$ref": "#/definitions/codersdk.ProvisionerJobInput" }, + "logs_overflowed": { + "type": "boolean" + }, "metadata": { "$ref": "#/definitions/codersdk.ProvisionerJobMetadata" }, diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 257cbc6e6b142..72489ea92d572 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -4489,6 +4489,22 @@ func (q *querier) UpdateProvisionerJobByID(ctx context.Context, arg database.Upd return q.db.UpdateProvisionerJobByID(ctx, arg) } +func (q *querier) UpdateProvisionerJobLogsLength(ctx context.Context, arg database.UpdateProvisionerJobLogsLengthParams) error { + // Not sure what the rbac should be here, going with this for now + if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerJobs); err != nil { + return err + } + return q.db.UpdateProvisionerJobLogsLength(ctx, arg) +} + +func (q *querier) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg database.UpdateProvisionerJobLogsOverflowedParams) error { + // Not sure what the rbac should be here, going with this for now + if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerJobs); err != nil { + return err + } + return q.db.UpdateProvisionerJobLogsOverflowed(ctx, arg) +} + func (q *querier) UpdateProvisionerJobWithCancelByID(ctx context.Context, arg database.UpdateProvisionerJobWithCancelByIDParams) error { // TODO: Remove this once we have a proper rbac check for provisioner jobs. // Details in https://github.com/coder/coder/issues/16160 diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index dc86d598617fd..14ab09bada09a 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4341,6 +4341,20 @@ func (s *MethodTestSuite) TestSystemFunctions() { UpdatedAt: time.Now(), }).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate) })) + s.Run("UpdateProvisionerJobLogsLength", s.Subtest(func(db database.Store, check *expects) { + j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) + check.Args(database.UpdateProvisionerJobLogsLengthParams{ + ID: j.ID, + LogsLength: 100, + }).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate) + })) + s.Run("UpdateProvisionerJobLogsOverflowed", s.Subtest(func(db database.Store, check *expects) { + j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) + check.Args(database.UpdateProvisionerJobLogsOverflowedParams{ + ID: j.ID, + LogsOverflowed: true, + }).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate) + })) s.Run("InsertProvisionerJob", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) check.Args(database.InsertProvisionerJobParams{ diff --git a/coderd/database/dbfake/dbfake.go b/coderd/database/dbfake/dbfake.go index 99d3c72ab4be3..e7c00255d47c2 100644 --- a/coderd/database/dbfake/dbfake.go +++ b/coderd/database/dbfake/dbfake.go @@ -179,6 +179,7 @@ func (b WorkspaceBuildBuilder) Do() WorkspaceResponse { Input: payload, Tags: map[string]string{}, TraceMetadata: pqtype.NullRawMessage{}, + LogsOverflowed: false, }) require.NoError(b.t, err, "insert job") diff --git a/coderd/database/dbgen/dbgen.go b/coderd/database/dbgen/dbgen.go index 71a86c329a5ad..81d9efd1cd3e3 100644 --- a/coderd/database/dbgen/dbgen.go +++ b/coderd/database/dbgen/dbgen.go @@ -775,6 +775,7 @@ func ProvisionerJob(t testing.TB, db database.Store, ps pubsub.Pubsub, orig data Input: takeFirstSlice(orig.Input, []byte("{}")), Tags: tags, TraceMetadata: pqtype.NullRawMessage{}, + LogsOverflowed: false, }) require.NoError(t, err, "insert job") if ps != nil { diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 811d945ac7da9..3fffb29966735 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -2784,6 +2784,20 @@ func (m queryMetricsStore) UpdateProvisionerJobByID(ctx context.Context, arg dat return err } +func (m queryMetricsStore) UpdateProvisionerJobLogsLength(ctx context.Context, arg database.UpdateProvisionerJobLogsLengthParams) error { + start := time.Now() + r0 := m.s.UpdateProvisionerJobLogsLength(ctx, arg) + m.queryLatencies.WithLabelValues("UpdateProvisionerJobLogsLength").Observe(time.Since(start).Seconds()) + return r0 +} + +func (m queryMetricsStore) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg database.UpdateProvisionerJobLogsOverflowedParams) error { + start := time.Now() + r0 := m.s.UpdateProvisionerJobLogsOverflowed(ctx, arg) + m.queryLatencies.WithLabelValues("UpdateProvisionerJobLogsOverflowed").Observe(time.Since(start).Seconds()) + return r0 +} + func (m queryMetricsStore) UpdateProvisionerJobWithCancelByID(ctx context.Context, arg database.UpdateProvisionerJobWithCancelByIDParams) error { start := time.Now() err := m.s.UpdateProvisionerJobWithCancelByID(ctx, arg) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index b20c3d06209b5..20bc17117e0eb 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -5958,6 +5958,34 @@ func (mr *MockStoreMockRecorder) UpdateProvisionerJobByID(ctx, arg any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateProvisionerJobByID", reflect.TypeOf((*MockStore)(nil).UpdateProvisionerJobByID), ctx, arg) } +// UpdateProvisionerJobLogsLength mocks base method. +func (m *MockStore) UpdateProvisionerJobLogsLength(ctx context.Context, arg database.UpdateProvisionerJobLogsLengthParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateProvisionerJobLogsLength", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateProvisionerJobLogsLength indicates an expected call of UpdateProvisionerJobLogsLength. +func (mr *MockStoreMockRecorder) UpdateProvisionerJobLogsLength(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateProvisionerJobLogsLength", reflect.TypeOf((*MockStore)(nil).UpdateProvisionerJobLogsLength), ctx, arg) +} + +// UpdateProvisionerJobLogsOverflowed mocks base method. +func (m *MockStore) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg database.UpdateProvisionerJobLogsOverflowedParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateProvisionerJobLogsOverflowed", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateProvisionerJobLogsOverflowed indicates an expected call of UpdateProvisionerJobLogsOverflowed. +func (mr *MockStoreMockRecorder) UpdateProvisionerJobLogsOverflowed(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateProvisionerJobLogsOverflowed", reflect.TypeOf((*MockStore)(nil).UpdateProvisionerJobLogsOverflowed), ctx, arg) +} + // UpdateProvisionerJobWithCancelByID mocks base method. func (m *MockStore) UpdateProvisionerJobWithCancelByID(ctx context.Context, arg database.UpdateProvisionerJobWithCancelByIDParams) error { m.ctrl.T.Helper() diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index c6c147e2f0bcb..053b5302d3e38 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -1419,11 +1419,18 @@ CASE WHEN (started_at IS NULL) THEN 'pending'::provisioner_job_status ELSE 'running'::provisioner_job_status END -END) STORED NOT NULL +END) STORED NOT NULL, + logs_length integer DEFAULT 0 NOT NULL, + logs_overflowed boolean DEFAULT false NOT NULL, + CONSTRAINT max_provisioner_logs_length CHECK ((logs_length <= 1048576)) ); COMMENT ON COLUMN provisioner_jobs.job_status IS 'Computed column to track the status of the job.'; +COMMENT ON COLUMN provisioner_jobs.logs_length IS 'Total length of provisioner logs'; + +COMMENT ON COLUMN provisioner_jobs.logs_overflowed IS 'Whether the provisioner logs overflowed in length'; + CREATE TABLE provisioner_keys ( id uuid NOT NULL, created_at timestamp with time zone NOT NULL, diff --git a/coderd/database/errors.go b/coderd/database/errors.go index 66c702de24445..0388ea2cbff49 100644 --- a/coderd/database/errors.go +++ b/coderd/database/errors.go @@ -79,3 +79,11 @@ func IsWorkspaceAgentLogsLimitError(err error) bool { return false } + +func IsProvisionerJobLogsLimitError(err error) bool { + var pqErr *pq.Error + if errors.As(err, &pqErr) { + return pqErr.Constraint == "max_provisioner_logs_length" && pqErr.Table == "provisioner_jobs" + } + return false +} diff --git a/coderd/database/migrations/000355_add_provisioner_logs_overflowed.down.sql b/coderd/database/migrations/000355_add_provisioner_logs_overflowed.down.sql new file mode 100644 index 0000000000000..39f34a2b491ee --- /dev/null +++ b/coderd/database/migrations/000355_add_provisioner_logs_overflowed.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE provisioner_jobs DROP COLUMN logs_length; +ALTER TABLE provisioner_jobs DROP COLUMN logs_overflowed; \ No newline at end of file diff --git a/coderd/database/migrations/000355_add_provisioner_logs_overflowed.up.sql b/coderd/database/migrations/000355_add_provisioner_logs_overflowed.up.sql new file mode 100644 index 0000000000000..80f58cf5c6693 --- /dev/null +++ b/coderd/database/migrations/000355_add_provisioner_logs_overflowed.up.sql @@ -0,0 +1,6 @@ + -- Add logs length tracking and overflow flag, similar to workspace agents + ALTER TABLE provisioner_jobs ADD COLUMN logs_length integer NOT NULL DEFAULT 0 CONSTRAINT max_provisioner_logs_length CHECK (logs_length <= 1048576); + ALTER TABLE provisioner_jobs ADD COLUMN logs_overflowed boolean NOT NULL DEFAULT false; + + COMMENT ON COLUMN provisioner_jobs.logs_length IS 'Total length of provisioner logs'; + COMMENT ON COLUMN provisioner_jobs.logs_overflowed IS 'Whether the provisioner logs overflowed in length'; diff --git a/coderd/database/models.go b/coderd/database/models.go index 8eed09f97b804..8b13c8a8af057 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -3384,6 +3384,10 @@ type ProvisionerJob struct { TraceMetadata pqtype.NullRawMessage `db:"trace_metadata" json:"trace_metadata"` // Computed column to track the status of the job. JobStatus ProvisionerJobStatus `db:"job_status" json:"job_status"` + // Total length of provisioner logs + LogsLength int32 `db:"logs_length" json:"logs_length"` + // Whether the provisioner logs overflowed in length + LogsOverflowed bool `db:"logs_overflowed" json:"logs_overflowed"` } type ProvisionerJobLog struct { diff --git a/coderd/database/querier.go b/coderd/database/querier.go index baa5d8590b1d7..a2c6cda1afc4b 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -593,6 +593,8 @@ type sqlcQuerier interface { UpdatePresetPrebuildStatus(ctx context.Context, arg UpdatePresetPrebuildStatusParams) error UpdateProvisionerDaemonLastSeenAt(ctx context.Context, arg UpdateProvisionerDaemonLastSeenAtParams) error UpdateProvisionerJobByID(ctx context.Context, arg UpdateProvisionerJobByIDParams) error + UpdateProvisionerJobLogsLength(ctx context.Context, arg UpdateProvisionerJobLogsLengthParams) error + UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg UpdateProvisionerJobLogsOverflowedParams) error UpdateProvisionerJobWithCancelByID(ctx context.Context, arg UpdateProvisionerJobWithCancelByIDParams) error UpdateProvisionerJobWithCompleteByID(ctx context.Context, arg UpdateProvisionerJobWithCompleteByIDParams) error UpdateProvisionerJobWithCompleteWithStartedAtByID(ctx context.Context, arg UpdateProvisionerJobWithCompleteWithStartedAtByIDParams) error diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 5c06119e80a75..6033ab728007d 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -8514,6 +8514,44 @@ func (q *sqlQuerier) InsertProvisionerJobLogs(ctx context.Context, arg InsertPro return items, nil } +const updateProvisionerJobLogsLength = `-- name: UpdateProvisionerJobLogsLength :exec +UPDATE + provisioner_jobs +SET + logs_length = logs_length + $2 +WHERE + id = $1 +` + +type UpdateProvisionerJobLogsLengthParams struct { + ID uuid.UUID `db:"id" json:"id"` + LogsLength int32 `db:"logs_length" json:"logs_length"` +} + +func (q *sqlQuerier) UpdateProvisionerJobLogsLength(ctx context.Context, arg UpdateProvisionerJobLogsLengthParams) error { + _, err := q.db.ExecContext(ctx, updateProvisionerJobLogsLength, arg.ID, arg.LogsLength) + return err +} + +const updateProvisionerJobLogsOverflowed = `-- name: UpdateProvisionerJobLogsOverflowed :exec +UPDATE + provisioner_jobs +SET + logs_overflowed = $2 +WHERE + id = $1 +` + +type UpdateProvisionerJobLogsOverflowedParams struct { + ID uuid.UUID `db:"id" json:"id"` + LogsOverflowed bool `db:"logs_overflowed" json:"logs_overflowed"` +} + +func (q *sqlQuerier) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg UpdateProvisionerJobLogsOverflowedParams) error { + _, err := q.db.ExecContext(ctx, updateProvisionerJobLogsOverflowed, arg.ID, arg.LogsOverflowed) + return err +} + const acquireProvisionerJob = `-- name: AcquireProvisionerJob :one UPDATE provisioner_jobs @@ -8543,7 +8581,7 @@ WHERE SKIP LOCKED LIMIT 1 - ) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + ) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed ` type AcquireProvisionerJobParams struct { @@ -8589,13 +8627,15 @@ func (q *sqlQuerier) AcquireProvisionerJob(ctx context.Context, arg AcquireProvi &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, + &i.LogsOverflowed, ) return i, err } const getProvisionerJobByID = `-- name: GetProvisionerJobByID :one SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE @@ -8625,13 +8665,15 @@ func (q *sqlQuerier) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (P &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, + &i.LogsOverflowed, ) return i, err } const getProvisionerJobByIDForUpdate = `-- name: GetProvisionerJobByIDForUpdate :one SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE @@ -8665,6 +8707,8 @@ func (q *sqlQuerier) GetProvisionerJobByIDForUpdate(ctx context.Context, id uuid &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, + &i.LogsOverflowed, ) return i, err } @@ -8708,7 +8752,7 @@ func (q *sqlQuerier) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID const getProvisionerJobsByIDs = `-- name: GetProvisionerJobsByIDs :many SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE @@ -8744,6 +8788,8 @@ func (q *sqlQuerier) GetProvisionerJobsByIDs(ctx context.Context, ids []uuid.UUI &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, + &i.LogsOverflowed, ); err != nil { return nil, err } @@ -8811,7 +8857,7 @@ SELECT -- Step 5: Final SELECT with INNER JOIN provisioner_jobs fj.id, fj.created_at, - pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, + pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, pj.logs_length, pj.logs_overflowed, fj.queue_position, fj.queue_size FROM @@ -8867,6 +8913,8 @@ func (q *sqlQuerier) GetProvisionerJobsByIDsWithQueuePosition(ctx context.Contex &i.ProvisionerJob.ErrorCode, &i.ProvisionerJob.TraceMetadata, &i.ProvisionerJob.JobStatus, + &i.ProvisionerJob.LogsLength, + &i.ProvisionerJob.LogsOverflowed, &i.QueuePosition, &i.QueueSize, ); err != nil { @@ -8909,7 +8957,7 @@ queue_size AS ( SELECT COUNT(*) AS count FROM pending_jobs ) SELECT - pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, + pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, pj.logs_length, pj.logs_overflowed, COALESCE(qp.queue_position, 0) AS queue_position, COALESCE(qs.count, 0) AS queue_size, -- Use subquery to utilize ORDER BY in array_agg since it cannot be @@ -9045,6 +9093,8 @@ func (q *sqlQuerier) GetProvisionerJobsByOrganizationAndStatusWithQueuePositionA &i.ProvisionerJob.ErrorCode, &i.ProvisionerJob.TraceMetadata, &i.ProvisionerJob.JobStatus, + &i.ProvisionerJob.LogsLength, + &i.ProvisionerJob.LogsOverflowed, &i.QueuePosition, &i.QueueSize, pq.Array(&i.AvailableWorkers), @@ -9071,7 +9121,7 @@ func (q *sqlQuerier) GetProvisionerJobsByOrganizationAndStatusWithQueuePositionA } const getProvisionerJobsCreatedAfter = `-- name: GetProvisionerJobsCreatedAfter :many -SELECT id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status FROM provisioner_jobs WHERE created_at > $1 +SELECT id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE created_at > $1 ` func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, createdAt time.Time) ([]ProvisionerJob, error) { @@ -9103,6 +9153,8 @@ func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, created &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, + &i.LogsOverflowed, ); err != nil { return nil, err } @@ -9119,7 +9171,7 @@ func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, created const getProvisionerJobsToBeReaped = `-- name: GetProvisionerJobsToBeReaped :many SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE @@ -9176,6 +9228,8 @@ func (q *sqlQuerier) GetProvisionerJobsToBeReaped(ctx context.Context, arg GetPr &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, + &i.LogsOverflowed, ); err != nil { return nil, err } @@ -9204,10 +9258,11 @@ INSERT INTO "type", "input", tags, - trace_metadata + trace_metadata, + logs_overflowed ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed ` type InsertProvisionerJobParams struct { @@ -9223,6 +9278,7 @@ type InsertProvisionerJobParams struct { Input json.RawMessage `db:"input" json:"input"` Tags StringMap `db:"tags" json:"tags"` TraceMetadata pqtype.NullRawMessage `db:"trace_metadata" json:"trace_metadata"` + LogsOverflowed bool `db:"logs_overflowed" json:"logs_overflowed"` } func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisionerJobParams) (ProvisionerJob, error) { @@ -9239,6 +9295,7 @@ func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisi arg.Input, arg.Tags, arg.TraceMetadata, + arg.LogsOverflowed, ) var i ProvisionerJob err := row.Scan( @@ -9261,6 +9318,8 @@ func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisi &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, + &i.LogsOverflowed, ) return i, err } diff --git a/coderd/database/queries/provisionerjoblogs.sql b/coderd/database/queries/provisionerjoblogs.sql index b98cf471f0d1a..c0ef188bdd382 100644 --- a/coderd/database/queries/provisionerjoblogs.sql +++ b/coderd/database/queries/provisionerjoblogs.sql @@ -19,3 +19,19 @@ SELECT unnest(@level :: log_level [ ]) AS LEVEL, unnest(@stage :: VARCHAR(128) [ ]) AS stage, unnest(@output :: VARCHAR(1024) [ ]) AS output RETURNING *; + +-- name: UpdateProvisionerJobLogsOverflowed :exec +UPDATE + provisioner_jobs +SET + logs_overflowed = $2 +WHERE + id = $1; + +-- name: UpdateProvisionerJobLogsLength :exec +UPDATE + provisioner_jobs +SET + logs_length = logs_length + $2 +WHERE + id = $1; diff --git a/coderd/database/queries/provisionerjobs.sql b/coderd/database/queries/provisionerjobs.sql index fcf348e089def..3ba581646689e 100644 --- a/coderd/database/queries/provisionerjobs.sql +++ b/coderd/database/queries/provisionerjobs.sql @@ -247,10 +247,11 @@ INSERT INTO "type", "input", tags, - trace_metadata + trace_metadata, + logs_overflowed ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING *; + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING *; -- name: UpdateProvisionerJobByID :exec UPDATE diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index 518b48d2fe04b..94173703c467d 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -902,29 +902,93 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest) return nil, xerrors.Errorf("update job: %w", err) } - if len(request.Logs) > 0 { + if len(request.Logs) > 0 && !job.LogsOverflowed { //nolint:exhaustruct // We append to the additional fields below. insertParams := database.InsertProvisionerJobLogsParams{ JobID: parsedID, } + + newLogSize := 0 + overflowedErrorMsg := "Provisioner logs exceeded the max size of 1MB. Will not continue to write provisioner logs for workspace build." + lenErrMsg := len(overflowedErrorMsg) + + var ( + createdAt time.Time + level database.LogLevel + stage string + source database.LogSource + output string + ) + for _, log := range request.Logs { - logLevel, err := convertLogLevel(log.Level) + // Build our log params + level, err = convertLogLevel(log.Level) if err != nil { return nil, xerrors.Errorf("convert log level: %w", err) } - logSource, err := convertLogSource(log.Source) + source, err = convertLogSource(log.Source) if err != nil { return nil, xerrors.Errorf("convert log source: %w", err) } - insertParams.CreatedAt = append(insertParams.CreatedAt, time.UnixMilli(log.CreatedAt)) - insertParams.Level = append(insertParams.Level, logLevel) - insertParams.Stage = append(insertParams.Stage, log.Stage) - insertParams.Source = append(insertParams.Source, logSource) - insertParams.Output = append(insertParams.Output, log.Output) + createdAt = time.UnixMilli(log.CreatedAt) + stage = log.Stage + output = log.Output + + // Check if we would overflow the job logs (not leaving enough room for the error message) + willOverflow := int64(job.LogsLength)+int64(newLogSize)+int64(lenErrMsg)+int64(len(output)) > 1048576 + if willOverflow { + s.Logger.Debug(ctx, "provisioner job logs overflowed 1MB size limit in database", slog.F("job_id", parsedID)) + err = s.Database.UpdateProvisionerJobLogsOverflowed(ctx, database.UpdateProvisionerJobLogsOverflowedParams{ + ID: parsedID, + LogsOverflowed: true, + }) + if err != nil { + s.Logger.Error(ctx, "failed to set logs overflowed flag", slog.F("job_id", parsedID), slog.Error(err)) + } + + level = database.LogLevelWarn + output = overflowedErrorMsg + } + + newLogSize += len(output) + + insertParams.CreatedAt = append(insertParams.CreatedAt, createdAt) + insertParams.Level = append(insertParams.Level, level) + insertParams.Stage = append(insertParams.Stage, stage) + insertParams.Source = append(insertParams.Source, source) + insertParams.Output = append(insertParams.Output, output) s.Logger.Debug(ctx, "job log", slog.F("job_id", parsedID), - slog.F("stage", log.Stage), - slog.F("output", log.Output)) + slog.F("stage", stage), + slog.F("output", output)) + + // Don't write any more logs because there's no room. + if willOverflow { + break + } + } + + err = s.Database.UpdateProvisionerJobLogsLength(ctx, database.UpdateProvisionerJobLogsLengthParams{ + ID: parsedID, + LogsLength: int32(newLogSize), // #nosec G115 - Log output length is limited to 1MB (2^20) which fits in an int32. + }) + if err != nil { + // Even though we do the runtime check for the overflow, we still check for the database error + // as well. + if database.IsProvisionerJobLogsLimitError(err) { + err = s.Database.UpdateProvisionerJobLogsOverflowed(ctx, database.UpdateProvisionerJobLogsOverflowedParams{ + ID: parsedID, + LogsOverflowed: true, + }) + if err != nil { + s.Logger.Error(ctx, "failed to set logs overflowed flag", slog.F("job_id", parsedID), slog.Error(err)) + } + return &proto.UpdateJobResponse{ + Canceled: job.CanceledAt.Valid, + }, nil + } + s.Logger.Error(ctx, "failed to update logs length", slog.F("job_id", parsedID), slog.Error(err)) + return nil, xerrors.Errorf("update logs length: %w", err) } logs, err := s.Database.InsertProvisionerJobLogs(ctx, insertParams) @@ -932,6 +996,7 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest) s.Logger.Error(ctx, "failed to insert job logs", slog.F("job_id", parsedID), slog.Error(err)) return nil, xerrors.Errorf("insert job logs: %w", err) } + // Publish by the lowest log ID inserted so the log stream will fetch // everything from that point. lowestID := logs[0].ID diff --git a/coderd/provisionerdserver/provisionerdserver_test.go b/coderd/provisionerdserver/provisionerdserver_test.go index 66684835650a8..b6f9d82a597e7 100644 --- a/coderd/provisionerdserver/provisionerdserver_test.go +++ b/coderd/provisionerdserver/provisionerdserver_test.go @@ -928,6 +928,141 @@ func TestUpdateJob(t *testing.T) { require.Equal(t, workspaceTags[1].Key, "cat") require.Equal(t, workspaceTags[1].Value, "jinx") }) + + t.Run("LogSizeLimit", func(t *testing.T) { + t.Parallel() + srv, db, _, pd := setup(t, false, &overrides{}) + job := setupJob(t, db, pd.ID, pd.Tags) + + // Create a log message that exceeds the 1MB limit + largeOutput := strings.Repeat("a", 1048577) // 1MB + 1 byte + + _, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: largeOutput, + }}, + }) + require.NoError(t, err) // Should succeed but trigger overflow + + // Verify the overflow flag is set + jobResult, err := db.GetProvisionerJobByID(ctx, job) + require.NoError(t, err) + require.True(t, jobResult.LogsOverflowed) + }) + + t.Run("IncrementalLogSizeOverflow", func(t *testing.T) { + t.Parallel() + srv, db, _, pd := setup(t, false, &overrides{}) + job := setupJob(t, db, pd.ID, pd.Tags) + + // Send logs that together exceed the limit + mediumOutput := strings.Repeat("b", 524289) // Half a MB + 1 byte + + // First log - should succeed + _, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: mediumOutput, + }}, + }) + require.NoError(t, err) + + // Verify overflow flag not yet set + jobResult, err := db.GetProvisionerJobByID(ctx, job) + require.NoError(t, err) + require.False(t, jobResult.LogsOverflowed) + + // Second log - should trigger overflow + _, err = srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: mediumOutput, + }}, + }) + require.NoError(t, err) + + // Verify overflow flag is set + jobResult, err = db.GetProvisionerJobByID(ctx, job) + require.NoError(t, err) + require.True(t, jobResult.LogsOverflowed) + }) + + t.Run("LogSizeTracking", func(t *testing.T) { + t.Parallel() + srv, db, _, pd := setup(t, false, &overrides{}) + job := setupJob(t, db, pd.ID, pd.Tags) + + logOutput := "test log message" + expectedSize := int32(len(logOutput)) // #nosec G115 - Log length is 16. + + _, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: logOutput, + }}, + }) + require.NoError(t, err) + + // Verify the logs_length is correctly tracked + jobResult, err := db.GetProvisionerJobByID(ctx, job) + require.NoError(t, err) + require.Equal(t, expectedSize, jobResult.LogsLength) + require.False(t, jobResult.LogsOverflowed) + }) + + t.Run("LogOverflowStopsProcessing", func(t *testing.T) { + t.Parallel() + srv, db, _, pd := setup(t, false, &overrides{}) + job := setupJob(t, db, pd.ID, pd.Tags) + + // First: trigger overflow + largeOutput := strings.Repeat("a", 1048577) // 1MB + 1 byte + _, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: largeOutput, + }}, + }) + require.NoError(t, err) + + // Get the initial log count + initialLogs, err := db.GetProvisionerLogsAfterID(ctx, database.GetProvisionerLogsAfterIDParams{ + JobID: job, + CreatedAfter: -1, + }) + require.NoError(t, err) + initialCount := len(initialLogs) + + // Second: try to send more logs - should be ignored + _, err = srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: "this should be ignored", + }}, + }) + require.NoError(t, err) + + // Verify no new logs were added + finalLogs, err := db.GetProvisionerLogsAfterID(ctx, database.GetProvisionerLogsAfterIDParams{ + JobID: job, + CreatedAfter: -1, + }) + require.NoError(t, err) + require.Equal(t, initialCount, len(finalLogs)) + }) } func TestFailJob(t *testing.T) { diff --git a/coderd/provisionerjobs.go b/coderd/provisionerjobs.go index 800b2916efef3..e9ab5260988d4 100644 --- a/coderd/provisionerjobs.go +++ b/coderd/provisionerjobs.go @@ -363,6 +363,7 @@ func convertProvisionerJob(pj database.GetProvisionerJobsByIDsWithQueuePositionR Tags: provisionerJob.Tags, QueuePosition: int(pj.QueuePosition), QueueSize: int(pj.QueueSize), + LogsOverflowed: provisionerJob.LogsOverflowed, } // Applying values optional to the struct. if provisionerJob.StartedAt.Valid { diff --git a/coderd/templateversions.go b/coderd/templateversions.go index 2a6e09d94978e..2c02268bba0a9 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -552,6 +552,7 @@ func (api *API) postTemplateVersionDryRun(rw http.ResponseWriter, r *http.Reques Valid: true, RawMessage: metadataRaw, }, + LogsOverflowed: false, }) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ @@ -1646,6 +1647,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht Valid: true, RawMessage: traceMetadataRaw, }, + LogsOverflowed: false, }) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ diff --git a/coderd/wsbuilder/wsbuilder.go b/coderd/wsbuilder/wsbuilder.go index 52567b463baac..73e449ee5bb93 100644 --- a/coderd/wsbuilder/wsbuilder.go +++ b/coderd/wsbuilder/wsbuilder.go @@ -409,6 +409,7 @@ func (b *Builder) buildTx(authFunc func(action policy.Action, object rbac.Object Valid: true, RawMessage: traceMetadataRaw, }, + LogsOverflowed: false, }) if err != nil { return nil, nil, nil, BuildError{http.StatusInternalServerError, "insert provisioner job", err} diff --git a/codersdk/provisionerdaemons.go b/codersdk/provisionerdaemons.go index 5fbda371b8f3f..e36f995f1688e 100644 --- a/codersdk/provisionerdaemons.go +++ b/codersdk/provisionerdaemons.go @@ -188,6 +188,7 @@ type ProvisionerJob struct { Type ProvisionerJobType `json:"type" table:"type"` AvailableWorkers []uuid.UUID `json:"available_workers,omitempty" format:"uuid" table:"available workers"` Metadata ProvisionerJobMetadata `json:"metadata" table:"metadata,recursive_inline"` + LogsOverflowed bool `json:"logs_overflowed" table:"logs overflowed"` } // ProvisionerJobLog represents the provisioner log entry annotated with source and level. diff --git a/docs/reference/api/builds.md b/docs/reference/api/builds.md index fb491405df362..a465575baeaa3 100644 --- a/docs/reference/api/builds.md +++ b/docs/reference/api/builds.md @@ -52,6 +52,7 @@ curl -X GET http://coder-server:8080/api/v2/users/{user}/workspace/{workspacenam "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -289,6 +290,7 @@ curl -X GET http://coder-server:8080/api/v2/workspacebuilds/{workspacebuild} \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1015,6 +1017,7 @@ curl -X GET http://coder-server:8080/api/v2/workspacebuilds/{workspacebuild}/sta "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1325,6 +1328,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaces/{workspace}/builds \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1540,6 +1544,7 @@ Status Code **200** | `»»» error` | string | false | | | | `»»» template_version_id` | string(uuid) | false | | | | `»»» workspace_build_id` | string(uuid) | false | | | +| `»» logs_overflowed` | boolean | false | | | | `»» metadata` | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata) | false | | | | `»»» template_display_name` | string | false | | | | `»»» template_icon` | string | false | | | @@ -1816,6 +1821,7 @@ curl -X POST http://coder-server:8080/api/v2/workspaces/{workspace}/builds \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", diff --git a/docs/reference/api/organizations.md b/docs/reference/api/organizations.md index 497e3f56d4e47..d418a1fcba106 100644 --- a/docs/reference/api/organizations.md +++ b/docs/reference/api/organizations.md @@ -407,6 +407,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/provisi "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -457,6 +458,7 @@ Status Code **200** | `»» error` | string | false | | | | `»» template_version_id` | string(uuid) | false | | | | `»» workspace_build_id` | string(uuid) | false | | | +| `» logs_overflowed` | boolean | false | | | | `» metadata` | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata) | false | | | | `»» template_display_name` | string | false | | | | `»» template_icon` | string | false | | | @@ -534,6 +536,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/provisi "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index 033ef6e196972..dcbd00628e979 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -5906,6 +5906,7 @@ Only certain features set these fields: - FeatureManagedAgentLimit| "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -5943,6 +5944,7 @@ Only certain features set these fields: - FeatureManagedAgentLimit| | `file_id` | string | false | | | | `id` | string | false | | | | `input` | [codersdk.ProvisionerJobInput](#codersdkprovisionerjobinput) | false | | | +| `logs_overflowed` | boolean | false | | | | `metadata` | [codersdk.ProvisionerJobMetadata](#codersdkprovisionerjobmetadata) | false | | | | `organization_id` | string | false | | | | `queue_position` | integer | false | | | @@ -7626,6 +7628,7 @@ Restarts will only happen on weekdays in this list on weeks which line up with W "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -8802,6 +8805,7 @@ If the schedule is empty, the user will be updated to use the default schedule.| "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -9911,6 +9915,7 @@ If the schedule is empty, the user will be updated to use the default schedule.| "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -10642,6 +10647,7 @@ If the schedule is empty, the user will be updated to use the default schedule.| "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", diff --git a/docs/reference/api/templates.md b/docs/reference/api/templates.md index 0db4ef8d04879..ea2e2c50cca7f 100644 --- a/docs/reference/api/templates.md +++ b/docs/reference/api/templates.md @@ -479,6 +479,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/templat "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -577,6 +578,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/templat "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -699,6 +701,7 @@ curl -X POST http://coder-server:8080/api/v2/organizations/{organization}/templa "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1264,6 +1267,7 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1337,6 +1341,7 @@ Status Code **200** | `»»» error` | string | false | | | | `»»» template_version_id` | string(uuid) | false | | | | `»»» workspace_build_id` | string(uuid) | false | | | +| `»» logs_overflowed` | boolean | false | | | | `»» metadata` | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata) | false | | | | `»»» template_display_name` | string | false | | | | `»»» template_icon` | string | false | | | @@ -1543,6 +1548,7 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions/{templ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1616,6 +1622,7 @@ Status Code **200** | `»»» error` | string | false | | | | `»»» template_version_id` | string(uuid) | false | | | | `»»» workspace_build_id` | string(uuid) | false | | | +| `»» logs_overflowed` | boolean | false | | | | `»» metadata` | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata) | false | | | | `»»» template_display_name` | string | false | | | | `»»» template_icon` | string | false | | | @@ -1712,6 +1719,7 @@ curl -X GET http://coder-server:8080/api/v2/templateversions/{templateversion} \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1819,6 +1827,7 @@ curl -X PATCH http://coder-server:8080/api/v2/templateversions/{templateversion} "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -2016,6 +2025,7 @@ curl -X POST http://coder-server:8080/api/v2/templateversions/{templateversion}/ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -2089,6 +2099,7 @@ curl -X GET http://coder-server:8080/api/v2/templateversions/{templateversion}/d "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", diff --git a/docs/reference/api/workspaces.md b/docs/reference/api/workspaces.md index debcb421e02e3..d7187259b5bb6 100644 --- a/docs/reference/api/workspaces.md +++ b/docs/reference/api/workspaces.md @@ -107,6 +107,7 @@ of the template will be used. "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -394,6 +395,7 @@ curl -X GET http://coder-server:8080/api/v2/users/{user}/workspace/{workspacenam "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -706,6 +708,7 @@ of the template will be used. "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -996,6 +999,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaces \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1267,6 +1271,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaces/{workspace} \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1670,6 +1675,7 @@ curl -X PUT http://coder-server:8080/api/v2/workspaces/{workspace}/dormant \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", diff --git a/docs/reference/cli/provisioner_jobs_list.md b/docs/reference/cli/provisioner_jobs_list.md index 07ad02f419bde..a0bff8554d610 100644 --- a/docs/reference/cli/provisioner_jobs_list.md +++ b/docs/reference/cli/provisioner_jobs_list.md @@ -45,10 +45,10 @@ Select which organization (uuid or name) to use. ### -c, --column -| | | -|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Type | [id\|created at\|started at\|completed at\|canceled at\|error\|error code\|status\|worker id\|worker name\|file id\|tags\|queue position\|queue size\|organization id\|template version id\|workspace build id\|type\|available workers\|template version name\|template id\|template name\|template display name\|template icon\|workspace id\|workspace name\|organization\|queue] | -| Default | created at,id,type,template display name,status,queue,tags | +| | | +|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Type | [id\|created at\|started at\|completed at\|canceled at\|error\|error code\|status\|worker id\|worker name\|file id\|tags\|queue position\|queue size\|organization id\|template version id\|workspace build id\|type\|available workers\|template version name\|template id\|template name\|template display name\|template icon\|workspace id\|workspace name\|logs overflowed\|organization\|queue] | +| Default | created at,id,type,template display name,status,queue,tags | Columns to display in table output. diff --git a/enterprise/cli/testdata/coder_provisioner_jobs_list_--help.golden b/enterprise/cli/testdata/coder_provisioner_jobs_list_--help.golden index f380a0334867c..8e22f78e978f2 100644 --- a/enterprise/cli/testdata/coder_provisioner_jobs_list_--help.golden +++ b/enterprise/cli/testdata/coder_provisioner_jobs_list_--help.golden @@ -11,7 +11,7 @@ OPTIONS: -O, --org string, $CODER_ORGANIZATION Select which organization (uuid or name) to use. - -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|organization|queue] (default: created at,id,type,template display name,status,queue,tags) + -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|logs overflowed|organization|queue] (default: created at,id,type,template display name,status,queue,tags) Columns to display in table output. -l, --limit int, $CODER_PROVISIONER_JOB_LIST_LIMIT (default: 50) diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 6165198c6fa23..3bd870d60159f 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -2156,6 +2156,7 @@ export interface ProvisionerJob { readonly type: ProvisionerJobType; readonly available_workers?: readonly string[]; readonly metadata: ProvisionerJobMetadata; + readonly logs_overflowed: boolean; } // From codersdk/provisionerdaemons.go diff --git a/site/src/modules/resources/AgentRow.tsx b/site/src/modules/resources/AgentRow.tsx index 20c551fc73065..ab0e5884c48e9 100644 --- a/site/src/modules/resources/AgentRow.tsx +++ b/site/src/modules/resources/AgentRow.tsx @@ -87,7 +87,8 @@ export const AgentRow: FC = ({ logs.push({ id: -1, level: "error", - output: "Startup logs exceeded the max size of 1MB!", + output: + "Startup logs exceeded the max size of 1MB, and will not continue to be written to the database! Logs will continue to be written to the /tmp/coder-startup-script.log file in the workspace.", created_at: new Date().toISOString(), source_id: "", }); diff --git a/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx b/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx index fcf6f0dbee549..20c929406d32c 100644 --- a/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx +++ b/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx @@ -1,9 +1,9 @@ import { type Interpolation, type Theme, useTheme } from "@emotion/react"; -import type { ProvisionerJobLog } from "api/typesGenerated"; +import type { ProvisionerJobLog, WorkspaceBuild } from "api/typesGenerated"; import type { Line } from "components/Logs/LogLine"; import { DEFAULT_LOG_LINE_SIDE_PADDING, Logs } from "components/Logs/Logs"; import dayjs from "dayjs"; -import { type FC, Fragment, type HTMLAttributes } from "react"; +import { type FC, Fragment, type HTMLAttributes, useMemo } from "react"; import { BODY_FONT_FAMILY, MONOSPACE_FONT_FAMILY } from "theme/constants"; const Language = { @@ -42,15 +42,37 @@ interface WorkspaceBuildLogsProps extends HTMLAttributes { hideTimestamps?: boolean; sticky?: boolean; logs: ProvisionerJobLog[]; + build?: WorkspaceBuild; } export const WorkspaceBuildLogs: FC = ({ hideTimestamps, sticky, logs, + build, ...attrs }) => { const theme = useTheme(); + + const processedLogs = useMemo(() => { + const allLogs = logs || []; + + // Add synthetic overflow message if needed + if (build?.job?.logs_overflowed) { + allLogs.push({ + id: -1, + created_at: new Date().toISOString(), + log_level: "error", + log_source: "provisioner", + output: + "Provisioner logs exceeded the max size of 1MB. Will not continue to write provisioner logs for workspace build.", + stage: "overflow", + }); + } + + return allLogs; + }, [logs, build?.job?.logs_overflowed]); + const groupedLogsByStage = groupLogsByStage(logs); return ( diff --git a/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx b/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx index 6add701c8b688..3a45653557dcc 100644 --- a/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx +++ b/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx @@ -212,7 +212,24 @@ export const WorkspaceBuildPageView: FC = ({ )} - {tabState.value === "build" && } + {build?.job?.logs_overflowed && ( + + Provisioner logs exceeded the max size of 1MB. Will not continue + to write provisioner logs for workspace build. + + )} + + {tabState.value === "build" && ( + + )} {tabState.value !== "build" && selectedAgent && ( )} @@ -261,7 +278,10 @@ const ScrollArea: FC> = (props) => { ); }; -const BuildLogsContent: FC<{ logs?: ProvisionerJobLog[] }> = ({ logs }) => { +const BuildLogsContent: FC<{ + logs?: ProvisionerJobLog[]; + build?: WorkspaceBuild; +}> = ({ logs, build }) => { if (!logs) { return ; } @@ -278,6 +298,7 @@ const BuildLogsContent: FC<{ logs?: ProvisionerJobLog[] }> = ({ logs }) => { }, }} logs={sortLogsByCreatedAt(logs)} + build={build} /> ); }; diff --git a/site/src/testHelpers/entities.ts b/site/src/testHelpers/entities.ts index f91a29cb48412..78dd9e4e8687a 100644 --- a/site/src/testHelpers/entities.ts +++ b/site/src/testHelpers/entities.ts @@ -689,6 +689,7 @@ export const MockProvisionerJob: TypesGen.ProvisionerJob = { template_version_name: "test-version", workspace_name: "test-workspace", }, + logs_overflowed: false, }; export const MockFailedProvisionerJob: TypesGen.ProvisionerJob = {