From 9f8cd500bfaa452d0f32159169b43a2af6ede08c Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Tue, 15 Jul 2025 17:10:33 +0000 Subject: [PATCH 01/21] add logs overflowed field to provisioner jobs and gen models --- coderd/database/dump.sql | 3 ++- ...9_add_provisioner_logs_overflowed.down.sql | 1 + ...349_add_provisioner_logs_overflowed.up.sql | 1 + coderd/database/models.go | 3 ++- coderd/database/queries.sql.go | 27 ++++++++++++------- 5 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 coderd/database/migrations/000349_add_provisioner_logs_overflowed.down.sql create mode 100644 coderd/database/migrations/000349_add_provisioner_logs_overflowed.up.sql diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index 54f984294fa4e..b9da48f2afa12 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -1360,7 +1360,8 @@ CASE WHEN (started_at IS NULL) THEN 'pending'::provisioner_job_status ELSE 'running'::provisioner_job_status END -END) STORED NOT NULL +END) STORED NOT NULL, + logs_overflowed boolean DEFAULT false NOT NULL ); COMMENT ON COLUMN provisioner_jobs.job_status IS 'Computed column to track the status of the job.'; diff --git a/coderd/database/migrations/000349_add_provisioner_logs_overflowed.down.sql b/coderd/database/migrations/000349_add_provisioner_logs_overflowed.down.sql new file mode 100644 index 0000000000000..d20995e0abece --- /dev/null +++ b/coderd/database/migrations/000349_add_provisioner_logs_overflowed.down.sql @@ -0,0 +1 @@ +ALTER TABLE provisioner_jobs DROP COLUMN logs_overflowed; \ No newline at end of file diff --git a/coderd/database/migrations/000349_add_provisioner_logs_overflowed.up.sql b/coderd/database/migrations/000349_add_provisioner_logs_overflowed.up.sql new file mode 100644 index 0000000000000..4770ed721064f --- /dev/null +++ b/coderd/database/migrations/000349_add_provisioner_logs_overflowed.up.sql @@ -0,0 +1 @@ +ALTER TABLE provisioner_jobs ADD COLUMN logs_overflowed boolean NOT NULL DEFAULT false; \ No newline at end of file diff --git a/coderd/database/models.go b/coderd/database/models.go index 749de51118152..8806957145b38 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -3155,7 +3155,8 @@ type ProvisionerJob struct { ErrorCode sql.NullString `db:"error_code" json:"error_code"` TraceMetadata pqtype.NullRawMessage `db:"trace_metadata" json:"trace_metadata"` // Computed column to track the status of the job. - JobStatus ProvisionerJobStatus `db:"job_status" json:"job_status"` + JobStatus ProvisionerJobStatus `db:"job_status" json:"job_status"` + LogsOverflowed bool `db:"logs_overflowed" json:"logs_overflowed"` } type ProvisionerJobLog struct { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 04ded71f1242a..ef92d3d09f565 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -8012,7 +8012,7 @@ WHERE SKIP LOCKED LIMIT 1 - ) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + ) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed ` type AcquireProvisionerJobParams struct { @@ -8058,13 +8058,14 @@ func (q *sqlQuerier) AcquireProvisionerJob(ctx context.Context, arg AcquireProvi &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsOverflowed, ) return i, err } const getProvisionerJobByID = `-- name: GetProvisionerJobByID :one SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed FROM provisioner_jobs WHERE @@ -8094,13 +8095,14 @@ func (q *sqlQuerier) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (P &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsOverflowed, ) return i, err } const getProvisionerJobByIDForUpdate = `-- name: GetProvisionerJobByIDForUpdate :one SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed FROM provisioner_jobs WHERE @@ -8134,6 +8136,7 @@ func (q *sqlQuerier) GetProvisionerJobByIDForUpdate(ctx context.Context, id uuid &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsOverflowed, ) return i, err } @@ -8177,7 +8180,7 @@ func (q *sqlQuerier) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID const getProvisionerJobsByIDs = `-- name: GetProvisionerJobsByIDs :many SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed FROM provisioner_jobs WHERE @@ -8213,6 +8216,7 @@ func (q *sqlQuerier) GetProvisionerJobsByIDs(ctx context.Context, ids []uuid.UUI &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsOverflowed, ); err != nil { return nil, err } @@ -8280,7 +8284,7 @@ SELECT -- Step 5: Final SELECT with INNER JOIN provisioner_jobs fj.id, fj.created_at, - pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, + pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, pj.logs_overflowed, fj.queue_position, fj.queue_size FROM @@ -8336,6 +8340,7 @@ func (q *sqlQuerier) GetProvisionerJobsByIDsWithQueuePosition(ctx context.Contex &i.ProvisionerJob.ErrorCode, &i.ProvisionerJob.TraceMetadata, &i.ProvisionerJob.JobStatus, + &i.ProvisionerJob.LogsOverflowed, &i.QueuePosition, &i.QueueSize, ); err != nil { @@ -8378,7 +8383,7 @@ queue_size AS ( SELECT COUNT(*) AS count FROM pending_jobs ) SELECT - pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, + pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, pj.logs_overflowed, COALESCE(qp.queue_position, 0) AS queue_position, COALESCE(qs.count, 0) AS queue_size, -- Use subquery to utilize ORDER BY in array_agg since it cannot be @@ -8514,6 +8519,7 @@ func (q *sqlQuerier) GetProvisionerJobsByOrganizationAndStatusWithQueuePositionA &i.ProvisionerJob.ErrorCode, &i.ProvisionerJob.TraceMetadata, &i.ProvisionerJob.JobStatus, + &i.ProvisionerJob.LogsOverflowed, &i.QueuePosition, &i.QueueSize, pq.Array(&i.AvailableWorkers), @@ -8540,7 +8546,7 @@ func (q *sqlQuerier) GetProvisionerJobsByOrganizationAndStatusWithQueuePositionA } const getProvisionerJobsCreatedAfter = `-- name: GetProvisionerJobsCreatedAfter :many -SELECT id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status FROM provisioner_jobs WHERE created_at > $1 +SELECT id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed FROM provisioner_jobs WHERE created_at > $1 ` func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, createdAt time.Time) ([]ProvisionerJob, error) { @@ -8572,6 +8578,7 @@ func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, created &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsOverflowed, ); err != nil { return nil, err } @@ -8588,7 +8595,7 @@ func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, created const getProvisionerJobsToBeReaped = `-- name: GetProvisionerJobsToBeReaped :many SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed FROM provisioner_jobs WHERE @@ -8645,6 +8652,7 @@ func (q *sqlQuerier) GetProvisionerJobsToBeReaped(ctx context.Context, arg GetPr &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsOverflowed, ); err != nil { return nil, err } @@ -8676,7 +8684,7 @@ INSERT INTO trace_metadata ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed ` type InsertProvisionerJobParams struct { @@ -8730,6 +8738,7 @@ func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisi &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsOverflowed, ) return i, err } From 68d8e5eee6a6fba413f251751b6926b577450947 Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Tue, 15 Jul 2025 17:26:00 +0000 Subject: [PATCH 02/21] update queries and make gen --- coderd/database/dbauthz/dbauthz.go | 8 ++++ coderd/database/dbmetrics/querymetrics.go | 14 +++++++ coderd/database/dbmock/dbmock.go | 29 +++++++++++++ coderd/database/querier.go | 2 + coderd/database/queries.sql.go | 42 ++++++++++++++++++- .../database/queries/provisionerjoblogs.sql | 16 +++++++ coderd/database/queries/provisionerjobs.sql | 5 ++- 7 files changed, 112 insertions(+), 4 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 55665b4381862..02a69b759ae20 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -2486,6 +2486,10 @@ func (q *querier) GetProvisionerJobByIDForUpdate(ctx context.Context, id uuid.UU return job, nil } +func (q *querier) GetProvisionerJobLogSize(ctx context.Context, jobID uuid.UUID) (interface{}, error) { + panic("not implemented") +} + func (q *querier) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID uuid.UUID) ([]database.ProvisionerJobTiming, error) { _, err := q.GetProvisionerJobByID(ctx, jobID) if err != nil { @@ -4202,6 +4206,10 @@ func (q *querier) RevokeDBCryptKey(ctx context.Context, activeKeyDigest string) return q.db.RevokeDBCryptKey(ctx, activeKeyDigest) } +func (q *querier) SetProvisionerJobLogsOverflowed(ctx context.Context, arg database.SetProvisionerJobLogsOverflowedParams) error { + panic("not implemented") +} + func (q *querier) TryAcquireLock(ctx context.Context, id int64) (bool, error) { return q.db.TryAcquireLock(ctx, id) } diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index b8ae92cd9f270..d1d8a67c2357f 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -1230,6 +1230,13 @@ func (m queryMetricsStore) GetProvisionerJobByIDForUpdate(ctx context.Context, i return r0, r1 } +func (m queryMetricsStore) GetProvisionerJobLogSize(ctx context.Context, jobID uuid.UUID) (interface{}, error) { + start := time.Now() + r0, r1 := m.s.GetProvisionerJobLogSize(ctx, jobID) + m.queryLatencies.WithLabelValues("GetProvisionerJobLogSize").Observe(time.Since(start).Seconds()) + return r0, r1 +} + func (m queryMetricsStore) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID uuid.UUID) ([]database.ProvisionerJobTiming, error) { start := time.Now() r0, r1 := m.s.GetProvisionerJobTimingsByJobID(ctx, jobID) @@ -2602,6 +2609,13 @@ func (m queryMetricsStore) RevokeDBCryptKey(ctx context.Context, activeKeyDigest return r0 } +func (m queryMetricsStore) SetProvisionerJobLogsOverflowed(ctx context.Context, arg database.SetProvisionerJobLogsOverflowedParams) error { + start := time.Now() + r0 := m.s.SetProvisionerJobLogsOverflowed(ctx, arg) + m.queryLatencies.WithLabelValues("SetProvisionerJobLogsOverflowed").Observe(time.Since(start).Seconds()) + return r0 +} + func (m queryMetricsStore) TryAcquireLock(ctx context.Context, pgTryAdvisoryXactLock int64) (bool, error) { start := time.Now() ok, err := m.s.TryAcquireLock(ctx, pgTryAdvisoryXactLock) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index ec9ca45b195e7..47aa7f8fdb0b2 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -2553,6 +2553,21 @@ func (mr *MockStoreMockRecorder) GetProvisionerJobByIDForUpdate(ctx, id any) *go return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProvisionerJobByIDForUpdate", reflect.TypeOf((*MockStore)(nil).GetProvisionerJobByIDForUpdate), ctx, id) } +// GetProvisionerJobLogSize mocks base method. +func (m *MockStore) GetProvisionerJobLogSize(ctx context.Context, jobID uuid.UUID) (any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetProvisionerJobLogSize", ctx, jobID) + ret0, _ := ret[0].(any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetProvisionerJobLogSize indicates an expected call of GetProvisionerJobLogSize. +func (mr *MockStoreMockRecorder) GetProvisionerJobLogSize(ctx, jobID any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProvisionerJobLogSize", reflect.TypeOf((*MockStore)(nil).GetProvisionerJobLogSize), ctx, jobID) +} + // GetProvisionerJobTimingsByJobID mocks base method. func (m *MockStore) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID uuid.UUID) ([]database.ProvisionerJobTiming, error) { m.ctrl.T.Helper() @@ -5549,6 +5564,20 @@ func (mr *MockStoreMockRecorder) RevokeDBCryptKey(ctx, activeKeyDigest any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RevokeDBCryptKey", reflect.TypeOf((*MockStore)(nil).RevokeDBCryptKey), ctx, activeKeyDigest) } +// SetProvisionerJobLogsOverflowed mocks base method. +func (m *MockStore) SetProvisionerJobLogsOverflowed(ctx context.Context, arg database.SetProvisionerJobLogsOverflowedParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetProvisionerJobLogsOverflowed", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetProvisionerJobLogsOverflowed indicates an expected call of SetProvisionerJobLogsOverflowed. +func (mr *MockStoreMockRecorder) SetProvisionerJobLogsOverflowed(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetProvisionerJobLogsOverflowed", reflect.TypeOf((*MockStore)(nil).SetProvisionerJobLogsOverflowed), ctx, arg) +} + // TryAcquireLock mocks base method. func (m *MockStore) TryAcquireLock(ctx context.Context, pgTryAdvisoryXactLock int64) (bool, error) { m.ctrl.T.Helper() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index b83c7415a60c8..302ba6ceb997d 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -282,6 +282,7 @@ type sqlcQuerier interface { // Gets a single provisioner job by ID for update. // This is used to securely reap jobs that have been hung/pending for a long time. GetProvisionerJobByIDForUpdate(ctx context.Context, id uuid.UUID) (ProvisionerJob, error) + GetProvisionerJobLogSize(ctx context.Context, jobID uuid.UUID) (interface{}, error) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID uuid.UUID) ([]ProvisionerJobTiming, error) GetProvisionerJobsByIDs(ctx context.Context, ids []uuid.UUID) ([]ProvisionerJob, error) GetProvisionerJobsByIDsWithQueuePosition(ctx context.Context, arg GetProvisionerJobsByIDsWithQueuePositionParams) ([]GetProvisionerJobsByIDsWithQueuePositionRow, error) @@ -561,6 +562,7 @@ type sqlcQuerier interface { RemoveUserFromAllGroups(ctx context.Context, userID uuid.UUID) error RemoveUserFromGroups(ctx context.Context, arg RemoveUserFromGroupsParams) ([]uuid.UUID, error) RevokeDBCryptKey(ctx context.Context, activeKeyDigest string) error + SetProvisionerJobLogsOverflowed(ctx context.Context, arg SetProvisionerJobLogsOverflowedParams) error // Non blocking lock. Returns true if the lock was acquired, false otherwise. // // This must be called from within a transaction. The lock will be automatically diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index ef92d3d09f565..3367d2ab24df7 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -7878,6 +7878,22 @@ func (q *sqlQuerier) UpsertProvisionerDaemon(ctx context.Context, arg UpsertProv return i, err } +const getProvisionerJobLogSize = `-- name: GetProvisionerJobLogSize :one + SELECT + COALESCE(SUM(LENGTH(output)), 0) AS total_size + FROM + provisioner_job_logs + WHERE + job_id = $1 +` + +func (q *sqlQuerier) GetProvisionerJobLogSize(ctx context.Context, jobID uuid.UUID) (interface{}, error) { + row := q.db.QueryRowContext(ctx, getProvisionerJobLogSize, jobID) + var total_size interface{} + err := row.Scan(&total_size) + return total_size, err +} + const getProvisionerLogsAfterID = `-- name: GetProvisionerLogsAfterID :many SELECT job_id, created_at, source, level, stage, output, id @@ -7985,6 +8001,25 @@ func (q *sqlQuerier) InsertProvisionerJobLogs(ctx context.Context, arg InsertPro return items, nil } +const setProvisionerJobLogsOverflowed = `-- name: SetProvisionerJobLogsOverflowed :exec +UPDATE + provisioner_jobs +SET + logs_overflowed = $2 +WHERE + id = $1 +` + +type SetProvisionerJobLogsOverflowedParams struct { + ID uuid.UUID `db:"id" json:"id"` + LogsOverflowed bool `db:"logs_overflowed" json:"logs_overflowed"` +} + +func (q *sqlQuerier) SetProvisionerJobLogsOverflowed(ctx context.Context, arg SetProvisionerJobLogsOverflowedParams) error { + _, err := q.db.ExecContext(ctx, setProvisionerJobLogsOverflowed, arg.ID, arg.LogsOverflowed) + return err +} + const acquireProvisionerJob = `-- name: AcquireProvisionerJob :one UPDATE provisioner_jobs @@ -8681,10 +8716,11 @@ INSERT INTO "type", "input", tags, - trace_metadata + trace_metadata, + logs_overflowed ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed ` type InsertProvisionerJobParams struct { @@ -8700,6 +8736,7 @@ type InsertProvisionerJobParams struct { Input json.RawMessage `db:"input" json:"input"` Tags StringMap `db:"tags" json:"tags"` TraceMetadata pqtype.NullRawMessage `db:"trace_metadata" json:"trace_metadata"` + LogsOverflowed bool `db:"logs_overflowed" json:"logs_overflowed"` } func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisionerJobParams) (ProvisionerJob, error) { @@ -8716,6 +8753,7 @@ func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisi arg.Input, arg.Tags, arg.TraceMetadata, + arg.LogsOverflowed, ) var i ProvisionerJob err := row.Scan( diff --git a/coderd/database/queries/provisionerjoblogs.sql b/coderd/database/queries/provisionerjoblogs.sql index b98cf471f0d1a..50148cfa250fb 100644 --- a/coderd/database/queries/provisionerjoblogs.sql +++ b/coderd/database/queries/provisionerjoblogs.sql @@ -8,6 +8,14 @@ WHERE AND ( id > @created_after ) ORDER BY id ASC; + +-- name: GetProvisionerJobLogSize :one + SELECT + COALESCE(SUM(LENGTH(output)), 0) AS total_size + FROM + provisioner_job_logs + WHERE + job_id = @job_id; -- name: InsertProvisionerJobLogs :many INSERT INTO @@ -19,3 +27,11 @@ SELECT unnest(@level :: log_level [ ]) AS LEVEL, unnest(@stage :: VARCHAR(128) [ ]) AS stage, unnest(@output :: VARCHAR(1024) [ ]) AS output RETURNING *; + +-- name: SetProvisionerJobLogsOverflowed :exec +UPDATE + provisioner_jobs +SET + logs_overflowed = $2 +WHERE + id = $1; diff --git a/coderd/database/queries/provisionerjobs.sql b/coderd/database/queries/provisionerjobs.sql index f3902ba2ddd38..7b40409f135d6 100644 --- a/coderd/database/queries/provisionerjobs.sql +++ b/coderd/database/queries/provisionerjobs.sql @@ -245,10 +245,11 @@ INSERT INTO "type", "input", tags, - trace_metadata + trace_metadata, + logs_overflowed ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING *; + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING *; -- name: UpdateProvisionerJobByID :exec UPDATE From 476c6fffe1961bae9150fdbc8c9ec911c8b9c385 Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Tue, 15 Jul 2025 17:43:01 +0000 Subject: [PATCH 03/21] add logs length field as well with constraint --- coderd/database/dump.sql | 8 +++++- ...9_add_provisioner_logs_overflowed.down.sql | 1 + ...349_add_provisioner_logs_overflowed.up.sql | 7 ++++- coderd/database/models.go | 7 +++-- coderd/database/queries.sql.go | 27 ++++++++++++------- 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/coderd/database/dump.sql b/coderd/database/dump.sql index b9da48f2afa12..a0b9c07e2e52b 100644 --- a/coderd/database/dump.sql +++ b/coderd/database/dump.sql @@ -1361,11 +1361,17 @@ CASE ELSE 'running'::provisioner_job_status END END) STORED NOT NULL, - logs_overflowed boolean DEFAULT false NOT NULL + logs_length integer DEFAULT 0 NOT NULL, + logs_overflowed boolean DEFAULT false NOT NULL, + CONSTRAINT max_provisioner_logs_length CHECK ((logs_length <= 1048576)) ); COMMENT ON COLUMN provisioner_jobs.job_status IS 'Computed column to track the status of the job.'; +COMMENT ON COLUMN provisioner_jobs.logs_length IS 'Total length of provisioner logs'; + +COMMENT ON COLUMN provisioner_jobs.logs_overflowed IS 'Whether the provisioner logs overflowed in length'; + CREATE TABLE provisioner_keys ( id uuid NOT NULL, created_at timestamp with time zone NOT NULL, diff --git a/coderd/database/migrations/000349_add_provisioner_logs_overflowed.down.sql b/coderd/database/migrations/000349_add_provisioner_logs_overflowed.down.sql index d20995e0abece..39f34a2b491ee 100644 --- a/coderd/database/migrations/000349_add_provisioner_logs_overflowed.down.sql +++ b/coderd/database/migrations/000349_add_provisioner_logs_overflowed.down.sql @@ -1 +1,2 @@ +ALTER TABLE provisioner_jobs DROP COLUMN logs_length; ALTER TABLE provisioner_jobs DROP COLUMN logs_overflowed; \ No newline at end of file diff --git a/coderd/database/migrations/000349_add_provisioner_logs_overflowed.up.sql b/coderd/database/migrations/000349_add_provisioner_logs_overflowed.up.sql index 4770ed721064f..6f47ac790628a 100644 --- a/coderd/database/migrations/000349_add_provisioner_logs_overflowed.up.sql +++ b/coderd/database/migrations/000349_add_provisioner_logs_overflowed.up.sql @@ -1 +1,6 @@ -ALTER TABLE provisioner_jobs ADD COLUMN logs_overflowed boolean NOT NULL DEFAULT false; \ No newline at end of file + -- Add logs length tracking and overflow flag, similar to workspace agents + ALTER TABLE provisioner_jobs ADD COLUMN logs_length integer NOT NULL DEFAULT 0 CONSTRAINT max_provisioner_logs_length CHECK (logs_length <= 1048576); + ALTER TABLE provisioner_jobs ADD COLUMN logs_overflowed boolean NOT NULL DEFAULT false; + + COMMENT ON COLUMN provisioner_jobs.logs_length IS 'Total length of provisioner logs'; + COMMENT ON COLUMN provisioner_jobs.logs_overflowed IS 'Whether the provisioner logs overflowed in length'; \ No newline at end of file diff --git a/coderd/database/models.go b/coderd/database/models.go index 8806957145b38..e7efe46b431f4 100644 --- a/coderd/database/models.go +++ b/coderd/database/models.go @@ -3155,8 +3155,11 @@ type ProvisionerJob struct { ErrorCode sql.NullString `db:"error_code" json:"error_code"` TraceMetadata pqtype.NullRawMessage `db:"trace_metadata" json:"trace_metadata"` // Computed column to track the status of the job. - JobStatus ProvisionerJobStatus `db:"job_status" json:"job_status"` - LogsOverflowed bool `db:"logs_overflowed" json:"logs_overflowed"` + JobStatus ProvisionerJobStatus `db:"job_status" json:"job_status"` + // Total length of provisioner logs + LogsLength int32 `db:"logs_length" json:"logs_length"` + // Whether the provisioner logs overflowed in length + LogsOverflowed bool `db:"logs_overflowed" json:"logs_overflowed"` } type ProvisionerJobLog struct { diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 3367d2ab24df7..af7d3ffdd3e87 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -8047,7 +8047,7 @@ WHERE SKIP LOCKED LIMIT 1 - ) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed + ) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed ` type AcquireProvisionerJobParams struct { @@ -8093,6 +8093,7 @@ func (q *sqlQuerier) AcquireProvisionerJob(ctx context.Context, arg AcquireProvi &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, &i.LogsOverflowed, ) return i, err @@ -8100,7 +8101,7 @@ func (q *sqlQuerier) AcquireProvisionerJob(ctx context.Context, arg AcquireProvi const getProvisionerJobByID = `-- name: GetProvisionerJobByID :one SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE @@ -8130,6 +8131,7 @@ func (q *sqlQuerier) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (P &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, &i.LogsOverflowed, ) return i, err @@ -8137,7 +8139,7 @@ func (q *sqlQuerier) GetProvisionerJobByID(ctx context.Context, id uuid.UUID) (P const getProvisionerJobByIDForUpdate = `-- name: GetProvisionerJobByIDForUpdate :one SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE @@ -8171,6 +8173,7 @@ func (q *sqlQuerier) GetProvisionerJobByIDForUpdate(ctx context.Context, id uuid &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, &i.LogsOverflowed, ) return i, err @@ -8215,7 +8218,7 @@ func (q *sqlQuerier) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID const getProvisionerJobsByIDs = `-- name: GetProvisionerJobsByIDs :many SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE @@ -8251,6 +8254,7 @@ func (q *sqlQuerier) GetProvisionerJobsByIDs(ctx context.Context, ids []uuid.UUI &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, &i.LogsOverflowed, ); err != nil { return nil, err @@ -8319,7 +8323,7 @@ SELECT -- Step 5: Final SELECT with INNER JOIN provisioner_jobs fj.id, fj.created_at, - pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, pj.logs_overflowed, + pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, pj.logs_length, pj.logs_overflowed, fj.queue_position, fj.queue_size FROM @@ -8375,6 +8379,7 @@ func (q *sqlQuerier) GetProvisionerJobsByIDsWithQueuePosition(ctx context.Contex &i.ProvisionerJob.ErrorCode, &i.ProvisionerJob.TraceMetadata, &i.ProvisionerJob.JobStatus, + &i.ProvisionerJob.LogsLength, &i.ProvisionerJob.LogsOverflowed, &i.QueuePosition, &i.QueueSize, @@ -8418,7 +8423,7 @@ queue_size AS ( SELECT COUNT(*) AS count FROM pending_jobs ) SELECT - pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, pj.logs_overflowed, + pj.id, pj.created_at, pj.updated_at, pj.started_at, pj.canceled_at, pj.completed_at, pj.error, pj.organization_id, pj.initiator_id, pj.provisioner, pj.storage_method, pj.type, pj.input, pj.worker_id, pj.file_id, pj.tags, pj.error_code, pj.trace_metadata, pj.job_status, pj.logs_length, pj.logs_overflowed, COALESCE(qp.queue_position, 0) AS queue_position, COALESCE(qs.count, 0) AS queue_size, -- Use subquery to utilize ORDER BY in array_agg since it cannot be @@ -8554,6 +8559,7 @@ func (q *sqlQuerier) GetProvisionerJobsByOrganizationAndStatusWithQueuePositionA &i.ProvisionerJob.ErrorCode, &i.ProvisionerJob.TraceMetadata, &i.ProvisionerJob.JobStatus, + &i.ProvisionerJob.LogsLength, &i.ProvisionerJob.LogsOverflowed, &i.QueuePosition, &i.QueueSize, @@ -8581,7 +8587,7 @@ func (q *sqlQuerier) GetProvisionerJobsByOrganizationAndStatusWithQueuePositionA } const getProvisionerJobsCreatedAfter = `-- name: GetProvisionerJobsCreatedAfter :many -SELECT id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed FROM provisioner_jobs WHERE created_at > $1 +SELECT id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE created_at > $1 ` func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, createdAt time.Time) ([]ProvisionerJob, error) { @@ -8613,6 +8619,7 @@ func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, created &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, &i.LogsOverflowed, ); err != nil { return nil, err @@ -8630,7 +8637,7 @@ func (q *sqlQuerier) GetProvisionerJobsCreatedAfter(ctx context.Context, created const getProvisionerJobsToBeReaped = `-- name: GetProvisionerJobsToBeReaped :many SELECT - id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed + id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed FROM provisioner_jobs WHERE @@ -8687,6 +8694,7 @@ func (q *sqlQuerier) GetProvisionerJobsToBeReaped(ctx context.Context, arg GetPr &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, &i.LogsOverflowed, ); err != nil { return nil, err @@ -8720,7 +8728,7 @@ INSERT INTO logs_overflowed ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_overflowed + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING id, created_at, updated_at, started_at, canceled_at, completed_at, error, organization_id, initiator_id, provisioner, storage_method, type, input, worker_id, file_id, tags, error_code, trace_metadata, job_status, logs_length, logs_overflowed ` type InsertProvisionerJobParams struct { @@ -8776,6 +8784,7 @@ func (q *sqlQuerier) InsertProvisionerJob(ctx context.Context, arg InsertProvisi &i.ErrorCode, &i.TraceMetadata, &i.JobStatus, + &i.LogsLength, &i.LogsOverflowed, ) return i, err From c8de63396510102c5132e8a33c399bde368ed8dd Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Tue, 15 Jul 2025 18:12:38 +0000 Subject: [PATCH 04/21] update queries --- coderd/database/dbauthz/dbauthz.go | 12 ++++-- coderd/database/dbmetrics/querymetrics.go | 21 ++++++---- coderd/database/dbmock/dbmock.go | 42 ++++++++++++------- coderd/database/querier.go | 3 +- coderd/database/queries.sql.go | 27 ++++++++++-- .../database/queries/provisionerjoblogs.sql | 10 ++++- 6 files changed, 84 insertions(+), 31 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 02a69b759ae20..b38cfaa337b60 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -4206,10 +4206,6 @@ func (q *querier) RevokeDBCryptKey(ctx context.Context, activeKeyDigest string) return q.db.RevokeDBCryptKey(ctx, activeKeyDigest) } -func (q *querier) SetProvisionerJobLogsOverflowed(ctx context.Context, arg database.SetProvisionerJobLogsOverflowedParams) error { - panic("not implemented") -} - func (q *querier) TryAcquireLock(ctx context.Context, id int64) (bool, error) { return q.db.TryAcquireLock(ctx, id) } @@ -4439,6 +4435,14 @@ func (q *querier) UpdateProvisionerJobByID(ctx context.Context, arg database.Upd return q.db.UpdateProvisionerJobByID(ctx, arg) } +func (q *querier) UpdateProvisionerJobLogsLength(ctx context.Context, arg database.UpdateProvisionerJobLogsLengthParams) error { + panic("not implemented") +} + +func (q *querier) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg database.UpdateProvisionerJobLogsOverflowedParams) error { + panic("not implemented") +} + func (q *querier) UpdateProvisionerJobWithCancelByID(ctx context.Context, arg database.UpdateProvisionerJobWithCancelByIDParams) error { // TODO: Remove this once we have a proper rbac check for provisioner jobs. // Details in https://github.com/coder/coder/issues/16160 diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index d1d8a67c2357f..d8bb23d9631cc 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -2609,13 +2609,6 @@ func (m queryMetricsStore) RevokeDBCryptKey(ctx context.Context, activeKeyDigest return r0 } -func (m queryMetricsStore) SetProvisionerJobLogsOverflowed(ctx context.Context, arg database.SetProvisionerJobLogsOverflowedParams) error { - start := time.Now() - r0 := m.s.SetProvisionerJobLogsOverflowed(ctx, arg) - m.queryLatencies.WithLabelValues("SetProvisionerJobLogsOverflowed").Observe(time.Since(start).Seconds()) - return r0 -} - func (m queryMetricsStore) TryAcquireLock(ctx context.Context, pgTryAdvisoryXactLock int64) (bool, error) { start := time.Now() ok, err := m.s.TryAcquireLock(ctx, pgTryAdvisoryXactLock) @@ -2777,6 +2770,20 @@ func (m queryMetricsStore) UpdateProvisionerJobByID(ctx context.Context, arg dat return err } +func (m queryMetricsStore) UpdateProvisionerJobLogsLength(ctx context.Context, arg database.UpdateProvisionerJobLogsLengthParams) error { + start := time.Now() + r0 := m.s.UpdateProvisionerJobLogsLength(ctx, arg) + m.queryLatencies.WithLabelValues("UpdateProvisionerJobLogsLength").Observe(time.Since(start).Seconds()) + return r0 +} + +func (m queryMetricsStore) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg database.UpdateProvisionerJobLogsOverflowedParams) error { + start := time.Now() + r0 := m.s.UpdateProvisionerJobLogsOverflowed(ctx, arg) + m.queryLatencies.WithLabelValues("UpdateProvisionerJobLogsOverflowed").Observe(time.Since(start).Seconds()) + return r0 +} + func (m queryMetricsStore) UpdateProvisionerJobWithCancelByID(ctx context.Context, arg database.UpdateProvisionerJobWithCancelByIDParams) error { start := time.Now() err := m.s.UpdateProvisionerJobWithCancelByID(ctx, arg) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 47aa7f8fdb0b2..88eb5dc1760eb 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -5564,20 +5564,6 @@ func (mr *MockStoreMockRecorder) RevokeDBCryptKey(ctx, activeKeyDigest any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RevokeDBCryptKey", reflect.TypeOf((*MockStore)(nil).RevokeDBCryptKey), ctx, activeKeyDigest) } -// SetProvisionerJobLogsOverflowed mocks base method. -func (m *MockStore) SetProvisionerJobLogsOverflowed(ctx context.Context, arg database.SetProvisionerJobLogsOverflowedParams) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetProvisionerJobLogsOverflowed", ctx, arg) - ret0, _ := ret[0].(error) - return ret0 -} - -// SetProvisionerJobLogsOverflowed indicates an expected call of SetProvisionerJobLogsOverflowed. -func (mr *MockStoreMockRecorder) SetProvisionerJobLogsOverflowed(ctx, arg any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetProvisionerJobLogsOverflowed", reflect.TypeOf((*MockStore)(nil).SetProvisionerJobLogsOverflowed), ctx, arg) -} - // TryAcquireLock mocks base method. func (m *MockStore) TryAcquireLock(ctx context.Context, pgTryAdvisoryXactLock int64) (bool, error) { m.ctrl.T.Helper() @@ -5913,6 +5899,34 @@ func (mr *MockStoreMockRecorder) UpdateProvisionerJobByID(ctx, arg any) *gomock. return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateProvisionerJobByID", reflect.TypeOf((*MockStore)(nil).UpdateProvisionerJobByID), ctx, arg) } +// UpdateProvisionerJobLogsLength mocks base method. +func (m *MockStore) UpdateProvisionerJobLogsLength(ctx context.Context, arg database.UpdateProvisionerJobLogsLengthParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateProvisionerJobLogsLength", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateProvisionerJobLogsLength indicates an expected call of UpdateProvisionerJobLogsLength. +func (mr *MockStoreMockRecorder) UpdateProvisionerJobLogsLength(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateProvisionerJobLogsLength", reflect.TypeOf((*MockStore)(nil).UpdateProvisionerJobLogsLength), ctx, arg) +} + +// UpdateProvisionerJobLogsOverflowed mocks base method. +func (m *MockStore) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg database.UpdateProvisionerJobLogsOverflowedParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateProvisionerJobLogsOverflowed", ctx, arg) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateProvisionerJobLogsOverflowed indicates an expected call of UpdateProvisionerJobLogsOverflowed. +func (mr *MockStoreMockRecorder) UpdateProvisionerJobLogsOverflowed(ctx, arg any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateProvisionerJobLogsOverflowed", reflect.TypeOf((*MockStore)(nil).UpdateProvisionerJobLogsOverflowed), ctx, arg) +} + // UpdateProvisionerJobWithCancelByID mocks base method. func (m *MockStore) UpdateProvisionerJobWithCancelByID(ctx context.Context, arg database.UpdateProvisionerJobWithCancelByIDParams) error { m.ctrl.T.Helper() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 302ba6ceb997d..881e7adcd15ae 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -562,7 +562,6 @@ type sqlcQuerier interface { RemoveUserFromAllGroups(ctx context.Context, userID uuid.UUID) error RemoveUserFromGroups(ctx context.Context, arg RemoveUserFromGroupsParams) ([]uuid.UUID, error) RevokeDBCryptKey(ctx context.Context, activeKeyDigest string) error - SetProvisionerJobLogsOverflowed(ctx context.Context, arg SetProvisionerJobLogsOverflowedParams) error // Non blocking lock. Returns true if the lock was acquired, false otherwise. // // This must be called from within a transaction. The lock will be automatically @@ -591,6 +590,8 @@ type sqlcQuerier interface { UpdatePresetPrebuildStatus(ctx context.Context, arg UpdatePresetPrebuildStatusParams) error UpdateProvisionerDaemonLastSeenAt(ctx context.Context, arg UpdateProvisionerDaemonLastSeenAtParams) error UpdateProvisionerJobByID(ctx context.Context, arg UpdateProvisionerJobByIDParams) error + UpdateProvisionerJobLogsLength(ctx context.Context, arg UpdateProvisionerJobLogsLengthParams) error + UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg UpdateProvisionerJobLogsOverflowedParams) error UpdateProvisionerJobWithCancelByID(ctx context.Context, arg UpdateProvisionerJobWithCancelByIDParams) error UpdateProvisionerJobWithCompleteByID(ctx context.Context, arg UpdateProvisionerJobWithCompleteByIDParams) error UpdateProvisionerJobWithCompleteWithStartedAtByID(ctx context.Context, arg UpdateProvisionerJobWithCompleteWithStartedAtByIDParams) error diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index af7d3ffdd3e87..9ef40811ccef4 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -8001,7 +8001,26 @@ func (q *sqlQuerier) InsertProvisionerJobLogs(ctx context.Context, arg InsertPro return items, nil } -const setProvisionerJobLogsOverflowed = `-- name: SetProvisionerJobLogsOverflowed :exec +const updateProvisionerJobLogsLength = `-- name: UpdateProvisionerJobLogsLength :exec +UPDATE + provisioner_jobs +SET + logs_length = logs_length + $2 +WHERE + id = $1 +` + +type UpdateProvisionerJobLogsLengthParams struct { + ID uuid.UUID `db:"id" json:"id"` + LogsLength int32 `db:"logs_length" json:"logs_length"` +} + +func (q *sqlQuerier) UpdateProvisionerJobLogsLength(ctx context.Context, arg UpdateProvisionerJobLogsLengthParams) error { + _, err := q.db.ExecContext(ctx, updateProvisionerJobLogsLength, arg.ID, arg.LogsLength) + return err +} + +const updateProvisionerJobLogsOverflowed = `-- name: UpdateProvisionerJobLogsOverflowed :exec UPDATE provisioner_jobs SET @@ -8010,13 +8029,13 @@ WHERE id = $1 ` -type SetProvisionerJobLogsOverflowedParams struct { +type UpdateProvisionerJobLogsOverflowedParams struct { ID uuid.UUID `db:"id" json:"id"` LogsOverflowed bool `db:"logs_overflowed" json:"logs_overflowed"` } -func (q *sqlQuerier) SetProvisionerJobLogsOverflowed(ctx context.Context, arg SetProvisionerJobLogsOverflowedParams) error { - _, err := q.db.ExecContext(ctx, setProvisionerJobLogsOverflowed, arg.ID, arg.LogsOverflowed) +func (q *sqlQuerier) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg UpdateProvisionerJobLogsOverflowedParams) error { + _, err := q.db.ExecContext(ctx, updateProvisionerJobLogsOverflowed, arg.ID, arg.LogsOverflowed) return err } diff --git a/coderd/database/queries/provisionerjoblogs.sql b/coderd/database/queries/provisionerjoblogs.sql index 50148cfa250fb..715bd2280a3e4 100644 --- a/coderd/database/queries/provisionerjoblogs.sql +++ b/coderd/database/queries/provisionerjoblogs.sql @@ -28,10 +28,18 @@ SELECT unnest(@stage :: VARCHAR(128) [ ]) AS stage, unnest(@output :: VARCHAR(1024) [ ]) AS output RETURNING *; --- name: SetProvisionerJobLogsOverflowed :exec +-- name: UpdateProvisionerJobLogsOverflowed :exec UPDATE provisioner_jobs SET logs_overflowed = $2 WHERE id = $1; + +-- name: UpdateProvisionerJobLogsLength :exec +UPDATE + provisioner_jobs +SET + logs_length = logs_length + $2 +WHERE + id = $1; From 908c8886b308876bb35b574ef6c86df42dd6f15e Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Tue, 15 Jul 2025 18:35:50 +0000 Subject: [PATCH 05/21] handle log size overflow on insertion --- coderd/database/errors.go | 8 ++++++ .../provisionerdserver/provisionerdserver.go | 26 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/coderd/database/errors.go b/coderd/database/errors.go index 66c702de24445..0388ea2cbff49 100644 --- a/coderd/database/errors.go +++ b/coderd/database/errors.go @@ -79,3 +79,11 @@ func IsWorkspaceAgentLogsLimitError(err error) bool { return false } + +func IsProvisionerJobLogsLimitError(err error) bool { + var pqErr *pq.Error + if errors.As(err, &pqErr) { + return pqErr.Constraint == "max_provisioner_logs_length" && pqErr.Table == "provisioner_jobs" + } + return false +} diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index f545169c93b31..48e711fecadf3 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -927,6 +927,32 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest) slog.F("output", log.Output)) } + newLogSize := 0 + for _, log := range request.Logs { + newLogSize += len(log.Output) + } + + err = s.Database.UpdateProvisionerJobLogsLength(ctx, database.UpdateProvisionerJobLogsLengthParams{ + ID: parsedID, + LogsLength: int32(newLogSize), // #nosec G115 - Log output length is limited to 1MB (2^20) which fits in an int32. + }) + if err != nil { + if database.IsProvisionerJobLogsLimitError(err) { + err = s.Database.UpdateProvisionerJobLogsOverflowed(ctx, database.UpdateProvisionerJobLogsOverflowedParams{ + ID: parsedID, + LogsOverflowed: true, + }) + if err != nil { + s.Logger.Error(ctx, "failed to set logs overflowed flag", slog.F("job_id", parsedID), slog.Error(err)) + } + return &proto.UpdateJobResponse{ + Canceled: job.CanceledAt.Valid, + }, nil + } + s.Logger.Error(ctx, "failed to update logs length", slog.F("job_id", parsedID), slog.Error(err)) + return nil, xerrors.Errorf("update logs length: %w", err) + } + logs, err := s.Database.InsertProvisionerJobLogs(ctx, insertParams) if err != nil { s.Logger.Error(ctx, "failed to insert job logs", slog.F("job_id", parsedID), slog.Error(err)) From c709a9e641d0c0ea623c0d622c319912df7f6888 Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Tue, 15 Jul 2025 18:47:47 +0000 Subject: [PATCH 06/21] update sdk --- coderd/provisionerjobs.go | 1 + codersdk/provisionerdaemons.go | 1 + 2 files changed, 2 insertions(+) diff --git a/coderd/provisionerjobs.go b/coderd/provisionerjobs.go index 800b2916efef3..e9ab5260988d4 100644 --- a/coderd/provisionerjobs.go +++ b/coderd/provisionerjobs.go @@ -363,6 +363,7 @@ func convertProvisionerJob(pj database.GetProvisionerJobsByIDsWithQueuePositionR Tags: provisionerJob.Tags, QueuePosition: int(pj.QueuePosition), QueueSize: int(pj.QueueSize), + LogsOverflowed: provisionerJob.LogsOverflowed, } // Applying values optional to the struct. if provisionerJob.StartedAt.Valid { diff --git a/codersdk/provisionerdaemons.go b/codersdk/provisionerdaemons.go index 5fbda371b8f3f..e36f995f1688e 100644 --- a/codersdk/provisionerdaemons.go +++ b/codersdk/provisionerdaemons.go @@ -188,6 +188,7 @@ type ProvisionerJob struct { Type ProvisionerJobType `json:"type" table:"type"` AvailableWorkers []uuid.UUID `json:"available_workers,omitempty" format:"uuid" table:"available workers"` Metadata ProvisionerJobMetadata `json:"metadata" table:"metadata,recursive_inline"` + LogsOverflowed bool `json:"logs_overflowed" table:"logs overflowed"` } // ProvisionerJobLog represents the provisioner log entry annotated with source and level. From f32ac68252f19cba1b28362c4aeab828bf57eb39 Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Tue, 15 Jul 2025 19:23:20 +0000 Subject: [PATCH 07/21] implement dbauthz stubs --- coderd/database/dbauthz/dbauthz.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index b38cfaa337b60..8347d4ce46263 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -2487,7 +2487,11 @@ func (q *querier) GetProvisionerJobByIDForUpdate(ctx context.Context, id uuid.UU } func (q *querier) GetProvisionerJobLogSize(ctx context.Context, jobID uuid.UUID) (interface{}, error) { - panic("not implemented") + _, err := q.GetProvisionerJobByID(ctx, jobID) + if err != nil { + return nil, err + } + return q.db.GetProvisionerJobLogSize(ctx, jobID) } func (q *querier) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID uuid.UUID) ([]database.ProvisionerJobTiming, error) { @@ -4436,11 +4440,19 @@ func (q *querier) UpdateProvisionerJobByID(ctx context.Context, arg database.Upd } func (q *querier) UpdateProvisionerJobLogsLength(ctx context.Context, arg database.UpdateProvisionerJobLogsLengthParams) error { - panic("not implemented") + // Not sure what the rbac should be here, going with this for now + if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerJobs); err != nil { + return err + } + return q.db.UpdateProvisionerJobLogsLength(ctx, arg) } func (q *querier) UpdateProvisionerJobLogsOverflowed(ctx context.Context, arg database.UpdateProvisionerJobLogsOverflowedParams) error { - panic("not implemented") + // Not sure what the rbac should be here, going with this for now + if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourceProvisionerJobs); err != nil { + return err + } + return q.db.UpdateProvisionerJobLogsOverflowed(ctx, arg) } func (q *querier) UpdateProvisionerJobWithCancelByID(ctx context.Context, arg database.UpdateProvisionerJobWithCancelByIDParams) error { From 29ad77c3aeaedf3d61fc82f17e20b3d9718bacdc Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Tue, 15 Jul 2025 19:48:13 +0000 Subject: [PATCH 08/21] couple tests --- .../provisionerdserver_test.go | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/coderd/provisionerdserver/provisionerdserver_test.go b/coderd/provisionerdserver/provisionerdserver_test.go index 66684835650a8..b16c33ba524b9 100644 --- a/coderd/provisionerdserver/provisionerdserver_test.go +++ b/coderd/provisionerdserver/provisionerdserver_test.go @@ -928,6 +928,96 @@ func TestUpdateJob(t *testing.T) { require.Equal(t, workspaceTags[1].Key, "cat") require.Equal(t, workspaceTags[1].Value, "jinx") }) + + t.Run("LogSizeLimit", func(t *testing.T) { + t.Parallel() + srv, db, _, pd := setup(t, false, &overrides{}) + job := setupJob(t, db, pd.ID, pd.Tags) + + // Create a log message that exceeds the 1MB limit + largeOutput := strings.Repeat("a", 1048577) // 1MB + 1 byte + + _, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: largeOutput, + }}, + }) + require.NoError(t, err) // Should succeed but trigger overflow + + // Verify the overflow flag is set + jobResult, err := db.GetProvisionerJobByID(ctx, job) + require.NoError(t, err) + require.True(t, jobResult.LogsOverflowed) + }) + + t.Run("IncrementalLogSizeOverflow", func(t *testing.T) { + t.Parallel() + srv, db, _, pd := setup(t, false, &overrides{}) + job := setupJob(t, db, pd.ID, pd.Tags) + + // Send logs that together exceed the limit + mediumOutput := strings.Repeat("b", 524289) // Half a MB + 1 byte + + // First log - should succeed + _, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: mediumOutput, + }}, + }) + require.NoError(t, err) + + // Verify overflow flag not yet set + jobResult, err := db.GetProvisionerJobByID(ctx, job) + require.NoError(t, err) + require.False(t, jobResult.LogsOverflowed) + + // Second log - should trigger overflow + _, err = srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: mediumOutput, + }}, + }) + require.NoError(t, err) + + // Verify overflow flag is set + jobResult, err = db.GetProvisionerJobByID(ctx, job) + require.NoError(t, err) + require.True(t, jobResult.LogsOverflowed) + }) + + t.Run("LogSizeTracking", func(t *testing.T) { + t.Parallel() + srv, db, _, pd := setup(t, false, &overrides{}) + job := setupJob(t, db, pd.ID, pd.Tags) + + logOutput := "test log message" + expectedSize := int32(len(logOutput)) + + _, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: logOutput, + }}, + }) + require.NoError(t, err) + + // Verify the logs_length is correctly tracked + jobResult, err := db.GetProvisionerJobByID(ctx, job) + require.NoError(t, err) + require.Equal(t, expectedSize, jobResult.LogsLength) + require.False(t, jobResult.LogsOverflowed) + }) } func TestFailJob(t *testing.T) { From f6134aa94cdfd4df3e3465f3bfc0d48bf9b0406b Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Tue, 15 Jul 2025 21:40:55 +0000 Subject: [PATCH 09/21] initial frontend updates --- .../provisionerdserver/provisionerdserver.go | 6 +++ .../provisionerdserver_test.go | 45 +++++++++++++++++++ site/src/api/typesGenerated.ts | 1 + .../WorkspaceBuildLogs/WorkspaceBuildLogs.tsx | 25 ++++++++++- .../WorkspaceBuildPageView.tsx | 25 ++++++++++- site/src/testHelpers/entities.ts | 1 + 6 files changed, 99 insertions(+), 4 deletions(-) diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index 48e711fecadf3..59ac2a5b6b3c6 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -903,6 +903,12 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest) } if len(request.Logs) > 0 { + if job.LogsOverflowed { + return &proto.UpdateJobResponse{ + Canceled: job.CanceledAt.Valid, + }, nil + } + //nolint:exhaustruct // We append to the additional fields below. insertParams := database.InsertProvisionerJobLogsParams{ JobID: parsedID, diff --git a/coderd/provisionerdserver/provisionerdserver_test.go b/coderd/provisionerdserver/provisionerdserver_test.go index b16c33ba524b9..f8aa701b354b1 100644 --- a/coderd/provisionerdserver/provisionerdserver_test.go +++ b/coderd/provisionerdserver/provisionerdserver_test.go @@ -1018,6 +1018,51 @@ func TestUpdateJob(t *testing.T) { require.Equal(t, expectedSize, jobResult.LogsLength) require.False(t, jobResult.LogsOverflowed) }) + + t.Run("LogOverflowStopsProcessing", func(t *testing.T) { + t.Parallel() + srv, db, _, pd := setup(t, false, &overrides{}) + job := setupJob(t, db, pd.ID, pd.Tags) + + // First: trigger overflow + largeOutput := strings.Repeat("a", 1048577) // 1MB + 1 byte + _, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: largeOutput, + }}, + }) + require.NoError(t, err) + + // Get the initial log count + initialLogs, err := db.GetProvisionerLogsAfterID(ctx, database.GetProvisionerLogsAfterIDParams{ + JobID: job, + CreatedAfter: -1, + }) + require.NoError(t, err) + initialCount := len(initialLogs) + + // Second: try to send more logs - should be ignored + _, err = srv.UpdateJob(ctx, &proto.UpdateJobRequest{ + JobId: job.String(), + Logs: []*proto.Log{{ + Source: proto.LogSource_PROVISIONER, + Level: sdkproto.LogLevel_INFO, + Output: "this should be ignored", + }}, + }) + require.NoError(t, err) + + // Verify no new logs were added + finalLogs, err := db.GetProvisionerLogsAfterID(ctx, database.GetProvisionerLogsAfterIDParams{ + JobID: job, + CreatedAfter: -1, + }) + require.NoError(t, err) + require.Equal(t, initialCount, len(finalLogs)) + }) } func TestFailJob(t *testing.T) { diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 53dc919df2df3..f01d09e4ac5e3 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -2041,6 +2041,7 @@ export interface ProvisionerJob { readonly type: ProvisionerJobType; readonly available_workers?: readonly string[]; readonly metadata: ProvisionerJobMetadata; + readonly logs_overflowed: boolean; } // From codersdk/provisionerdaemons.go diff --git a/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx b/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx index fcf6f0dbee549..2715af8848e41 100644 --- a/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx +++ b/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx @@ -1,9 +1,9 @@ import { type Interpolation, type Theme, useTheme } from "@emotion/react"; -import type { ProvisionerJobLog } from "api/typesGenerated"; +import type { ProvisionerJobLog, WorkspaceBuild } from "api/typesGenerated"; import type { Line } from "components/Logs/LogLine"; import { DEFAULT_LOG_LINE_SIDE_PADDING, Logs } from "components/Logs/Logs"; import dayjs from "dayjs"; -import { type FC, Fragment, type HTMLAttributes } from "react"; +import { type FC, Fragment, type HTMLAttributes, useMemo } from "react"; import { BODY_FONT_FAMILY, MONOSPACE_FONT_FAMILY } from "theme/constants"; const Language = { @@ -42,15 +42,36 @@ interface WorkspaceBuildLogsProps extends HTMLAttributes { hideTimestamps?: boolean; sticky?: boolean; logs: ProvisionerJobLog[]; + build?: WorkspaceBuild; } export const WorkspaceBuildLogs: FC = ({ hideTimestamps, sticky, logs, + build, ...attrs }) => { const theme = useTheme(); + + const processedLogs = useMemo(() => { + const allLogs = logs || []; + + // Add synthetic overflow message if needed + if (build?.job?.logs_overflowed) { + allLogs.push({ + id: -1, + created_at: new Date().toISOString(), + log_level: "error", + log_source: "provisioner", + output: "Provisioner logs exceeded the max size of 1MB!", + stage: "overflow", + }); + } + + return allLogs; + }, [logs, build?.job?.logs_overflowed]); + const groupedLogsByStage = groupLogsByStage(logs); return ( diff --git a/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx b/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx index 6add701c8b688..f20086c4799c4 100644 --- a/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx +++ b/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx @@ -212,7 +212,24 @@ export const WorkspaceBuildPageView: FC = ({ )} - {tabState.value === "build" && } + {build?.job?.logs_overflowed && ( + + Provisioner logs exceeded the maximum size of 1MB and were + truncated. + + )} + + {tabState.value === "build" && ( + + )} {tabState.value !== "build" && selectedAgent && ( )} @@ -261,7 +278,10 @@ const ScrollArea: FC> = (props) => { ); }; -const BuildLogsContent: FC<{ logs?: ProvisionerJobLog[] }> = ({ logs }) => { +const BuildLogsContent: FC<{ + logs?: ProvisionerJobLog[]; + build?: WorkspaceBuild; +}> = ({ logs, build }) => { if (!logs) { return ; } @@ -278,6 +298,7 @@ const BuildLogsContent: FC<{ logs?: ProvisionerJobLog[] }> = ({ logs }) => { }, }} logs={sortLogsByCreatedAt(logs)} + build={build} /> ); }; diff --git a/site/src/testHelpers/entities.ts b/site/src/testHelpers/entities.ts index 22dc47ae2390f..ae449ccb7673d 100644 --- a/site/src/testHelpers/entities.ts +++ b/site/src/testHelpers/entities.ts @@ -689,6 +689,7 @@ export const MockProvisionerJob: TypesGen.ProvisionerJob = { template_version_name: "test-version", workspace_name: "test-workspace", }, + logs_overflowed: false, }; export const MockFailedProvisionerJob: TypesGen.ProvisionerJob = { From cc469f8b27f669c4a74b05c455c25523b1a924c5 Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Mon, 21 Jul 2025 15:25:10 +0000 Subject: [PATCH 10/21] runtime check possible overflow --- .../provisionerdserver/provisionerdserver.go | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index 59ac2a5b6b3c6..f50e133e219c0 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -902,12 +902,7 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest) return nil, xerrors.Errorf("update job: %w", err) } - if len(request.Logs) > 0 { - if job.LogsOverflowed { - return &proto.UpdateJobResponse{ - Canceled: job.CanceledAt.Valid, - }, nil - } + if len(request.Logs) > 0 && !job.LogsOverflowed { //nolint:exhaustruct // We append to the additional fields below. insertParams := database.InsertProvisionerJobLogsParams{ @@ -938,11 +933,28 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest) newLogSize += len(log.Output) } + willOverflow := int64(job.LogsLength)+int64(newLogSize) > 1048576 + if willOverflow { + err = s.Database.UpdateProvisionerJobLogsOverflowed(ctx, database.UpdateProvisionerJobLogsOverflowedParams{ + ID: parsedID, + LogsOverflowed: true, + }) + if err != nil { + s.Logger.Error(ctx, "failed to set logs overflowed flag", slog.F("job_id", parsedID), slog.Error(err)) + } + return &proto.UpdateJobResponse{ + Canceled: job.CanceledAt.Valid, + }, nil + } + err = s.Database.UpdateProvisionerJobLogsLength(ctx, database.UpdateProvisionerJobLogsLengthParams{ ID: parsedID, LogsLength: int32(newLogSize), // #nosec G115 - Log output length is limited to 1MB (2^20) which fits in an int32. }) if err != nil { + + // Even though we do the runtime check for the overflow, we still check for the database error + // as well. if database.IsProvisionerJobLogsLimitError(err) { err = s.Database.UpdateProvisionerJobLogsOverflowed(ctx, database.UpdateProvisionerJobLogsOverflowedParams{ ID: parsedID, @@ -964,6 +976,7 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest) s.Logger.Error(ctx, "failed to insert job logs", slog.F("job_id", parsedID), slog.Error(err)) return nil, xerrors.Errorf("insert job logs: %w", err) } + // Publish by the lowest log ID inserted so the log stream will fetch // everything from that point. lowestID := logs[0].ID From 29297d6e570a15b158709e8ff42c26c8fb506a03 Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Mon, 21 Jul 2025 15:30:11 +0000 Subject: [PATCH 11/21] update log message to be less confusing --- site/src/modules/resources/AgentRow.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/src/modules/resources/AgentRow.tsx b/site/src/modules/resources/AgentRow.tsx index 3d0888f7872b1..f3df105577ee2 100644 --- a/site/src/modules/resources/AgentRow.tsx +++ b/site/src/modules/resources/AgentRow.tsx @@ -89,7 +89,7 @@ export const AgentRow: FC = ({ logs.push({ id: -1, level: "error", - output: "Startup logs exceeded the max size of 1MB!", + output: "Startup logs exceeded the max size of 1MB, and will not continue to be written to the database! Logs will continue to be written to the /tmp/coder-startup-script.log file in the workspace.", created_at: new Date().toISOString(), source_id: "", }); From 1398491621a6ac6caeacb33e702e57a6a9b59ba7 Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Mon, 21 Jul 2025 16:04:33 +0000 Subject: [PATCH 12/21] lint fixes --- coderd/database/dbfake/dbfake.go | 1 + coderd/database/dbgen/dbgen.go | 1 + coderd/provisionerdserver/provisionerdserver.go | 2 -- coderd/provisionerdserver/provisionerdserver_test.go | 2 +- coderd/templateversions.go | 2 ++ coderd/wsbuilder/wsbuilder.go | 1 + site/src/modules/resources/AgentRow.tsx | 3 ++- 7 files changed, 8 insertions(+), 4 deletions(-) diff --git a/coderd/database/dbfake/dbfake.go b/coderd/database/dbfake/dbfake.go index 98e98122e74e5..ac6d76f7a806f 100644 --- a/coderd/database/dbfake/dbfake.go +++ b/coderd/database/dbfake/dbfake.go @@ -179,6 +179,7 @@ func (b WorkspaceBuildBuilder) Do() WorkspaceResponse { Input: payload, Tags: map[string]string{}, TraceMetadata: pqtype.NullRawMessage{}, + LogsOverflowed: false, }) require.NoError(b.t, err, "insert job") diff --git a/coderd/database/dbgen/dbgen.go b/coderd/database/dbgen/dbgen.go index d5693afe98826..d27439c6ed3fb 100644 --- a/coderd/database/dbgen/dbgen.go +++ b/coderd/database/dbgen/dbgen.go @@ -774,6 +774,7 @@ func ProvisionerJob(t testing.TB, db database.Store, ps pubsub.Pubsub, orig data Input: takeFirstSlice(orig.Input, []byte("{}")), Tags: tags, TraceMetadata: pqtype.NullRawMessage{}, + LogsOverflowed: false, }) require.NoError(t, err, "insert job") if ps != nil { diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index f50e133e219c0..32d303e5d5b90 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -903,7 +903,6 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest) } if len(request.Logs) > 0 && !job.LogsOverflowed { - //nolint:exhaustruct // We append to the additional fields below. insertParams := database.InsertProvisionerJobLogsParams{ JobID: parsedID, @@ -952,7 +951,6 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest) LogsLength: int32(newLogSize), // #nosec G115 - Log output length is limited to 1MB (2^20) which fits in an int32. }) if err != nil { - // Even though we do the runtime check for the overflow, we still check for the database error // as well. if database.IsProvisionerJobLogsLimitError(err) { diff --git a/coderd/provisionerdserver/provisionerdserver_test.go b/coderd/provisionerdserver/provisionerdserver_test.go index f8aa701b354b1..b6f9d82a597e7 100644 --- a/coderd/provisionerdserver/provisionerdserver_test.go +++ b/coderd/provisionerdserver/provisionerdserver_test.go @@ -1000,7 +1000,7 @@ func TestUpdateJob(t *testing.T) { job := setupJob(t, db, pd.ID, pd.Tags) logOutput := "test log message" - expectedSize := int32(len(logOutput)) + expectedSize := int32(len(logOutput)) // #nosec G115 - Log length is 16. _, err := srv.UpdateJob(ctx, &proto.UpdateJobRequest{ JobId: job.String(), diff --git a/coderd/templateversions.go b/coderd/templateversions.go index de069b5ca4723..353ac1452402b 100644 --- a/coderd/templateversions.go +++ b/coderd/templateversions.go @@ -551,6 +551,7 @@ func (api *API) postTemplateVersionDryRun(rw http.ResponseWriter, r *http.Reques Valid: true, RawMessage: metadataRaw, }, + LogsOverflowed: false, }) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ @@ -1645,6 +1646,7 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht Valid: true, RawMessage: traceMetadataRaw, }, + LogsOverflowed: false, }) if err != nil { httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ diff --git a/coderd/wsbuilder/wsbuilder.go b/coderd/wsbuilder/wsbuilder.go index 90ea02e966a09..a2afc523aae76 100644 --- a/coderd/wsbuilder/wsbuilder.go +++ b/coderd/wsbuilder/wsbuilder.go @@ -387,6 +387,7 @@ func (b *Builder) buildTx(authFunc func(action policy.Action, object rbac.Object Valid: true, RawMessage: traceMetadataRaw, }, + LogsOverflowed: false, }) if err != nil { return nil, nil, nil, BuildError{http.StatusInternalServerError, "insert provisioner job", err} diff --git a/site/src/modules/resources/AgentRow.tsx b/site/src/modules/resources/AgentRow.tsx index 3ecfd15623c3b..5fb785c9d9b63 100644 --- a/site/src/modules/resources/AgentRow.tsx +++ b/site/src/modules/resources/AgentRow.tsx @@ -87,7 +87,8 @@ export const AgentRow: FC = ({ logs.push({ id: -1, level: "error", - output: "Startup logs exceeded the max size of 1MB, and will not continue to be written to the database! Logs will continue to be written to the /tmp/coder-startup-script.log file in the workspace.", + output: + "Startup logs exceeded the max size of 1MB, and will not continue to be written to the database! Logs will continue to be written to the /tmp/coder-startup-script.log file in the workspace.", created_at: new Date().toISOString(), source_id: "", }); From 630e2ac43204b683dfadad2fab9cfd8c1a8a1955 Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Mon, 21 Jul 2025 16:10:34 +0000 Subject: [PATCH 13/21] rename migrations after conflicting merge --- ...d.down.sql => 000350_add_provisioner_logs_overflowed.down.sql} | 0 ...lowed.up.sql => 000350_add_provisioner_logs_overflowed.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename coderd/database/migrations/{000349_add_provisioner_logs_overflowed.down.sql => 000350_add_provisioner_logs_overflowed.down.sql} (100%) rename coderd/database/migrations/{000349_add_provisioner_logs_overflowed.up.sql => 000350_add_provisioner_logs_overflowed.up.sql} (100%) diff --git a/coderd/database/migrations/000349_add_provisioner_logs_overflowed.down.sql b/coderd/database/migrations/000350_add_provisioner_logs_overflowed.down.sql similarity index 100% rename from coderd/database/migrations/000349_add_provisioner_logs_overflowed.down.sql rename to coderd/database/migrations/000350_add_provisioner_logs_overflowed.down.sql diff --git a/coderd/database/migrations/000349_add_provisioner_logs_overflowed.up.sql b/coderd/database/migrations/000350_add_provisioner_logs_overflowed.up.sql similarity index 100% rename from coderd/database/migrations/000349_add_provisioner_logs_overflowed.up.sql rename to coderd/database/migrations/000350_add_provisioner_logs_overflowed.up.sql From e44cd47dbf0793b40c4ac5aefa37c14d0a979449 Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Mon, 21 Jul 2025 16:49:54 +0000 Subject: [PATCH 14/21] run make gen after migration --- cli/testdata/coder_list_--output_json.golden | 3 ++- .../coder_provisioner_jobs_list_--help.golden | 2 +- .../coder_provisioner_jobs_list_--output_json.golden | 2 ++ coderd/apidoc/docs.go | 3 +++ coderd/apidoc/swagger.json | 3 +++ docs/reference/api/builds.md | 6 ++++++ docs/reference/api/organizations.md | 3 +++ docs/reference/api/schemas.md | 6 ++++++ docs/reference/api/templates.md | 11 +++++++++++ docs/reference/api/workspaces.md | 6 ++++++ docs/reference/cli/provisioner_jobs_list.md | 8 ++++---- .../coder_provisioner_jobs_list_--help.golden | 2 +- 12 files changed, 48 insertions(+), 7 deletions(-) diff --git a/cli/testdata/coder_list_--output_json.golden b/cli/testdata/coder_list_--output_json.golden index 51c2887cd1e4a..75ac7ac86ce5a 100644 --- a/cli/testdata/coder_list_--output_json.golden +++ b/cli/testdata/coder_list_--output_json.golden @@ -55,7 +55,8 @@ "template_name": "", "template_display_name": "", "template_icon": "" - } + }, + "logs_overflowed": false }, "reason": "initiator", "resources": [], diff --git a/cli/testdata/coder_provisioner_jobs_list_--help.golden b/cli/testdata/coder_provisioner_jobs_list_--help.golden index f380a0334867c..8e22f78e978f2 100644 --- a/cli/testdata/coder_provisioner_jobs_list_--help.golden +++ b/cli/testdata/coder_provisioner_jobs_list_--help.golden @@ -11,7 +11,7 @@ OPTIONS: -O, --org string, $CODER_ORGANIZATION Select which organization (uuid or name) to use. - -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|organization|queue] (default: created at,id,type,template display name,status,queue,tags) + -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|logs overflowed|organization|queue] (default: created at,id,type,template display name,status,queue,tags) Columns to display in table output. -l, --limit int, $CODER_PROVISIONER_JOB_LIST_LIMIT (default: 50) diff --git a/cli/testdata/coder_provisioner_jobs_list_--output_json.golden b/cli/testdata/coder_provisioner_jobs_list_--output_json.golden index e36723765b4df..6ccf672360a55 100644 --- a/cli/testdata/coder_provisioner_jobs_list_--output_json.golden +++ b/cli/testdata/coder_provisioner_jobs_list_--output_json.golden @@ -26,6 +26,7 @@ "template_display_name": "", "template_icon": "" }, + "logs_overflowed": false, "organization_name": "Coder" }, { @@ -57,6 +58,7 @@ "workspace_id": "===========[workspace ID]===========", "workspace_name": "test-workspace" }, + "logs_overflowed": false, "organization_name": "Coder" } ] diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index 3618ed8610f5a..f23ce588a179e 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -15191,6 +15191,9 @@ const docTemplate = `{ "input": { "$ref": "#/definitions/codersdk.ProvisionerJobInput" }, + "logs_overflowed": { + "type": "boolean" + }, "metadata": { "$ref": "#/definitions/codersdk.ProvisionerJobMetadata" }, diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 11d403e75aad7..698df599f7c1a 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -13783,6 +13783,9 @@ "input": { "$ref": "#/definitions/codersdk.ProvisionerJobInput" }, + "logs_overflowed": { + "type": "boolean" + }, "metadata": { "$ref": "#/definitions/codersdk.ProvisionerJobMetadata" }, diff --git a/docs/reference/api/builds.md b/docs/reference/api/builds.md index 686f19316a8c0..d98a51bf3d534 100644 --- a/docs/reference/api/builds.md +++ b/docs/reference/api/builds.md @@ -52,6 +52,7 @@ curl -X GET http://coder-server:8080/api/v2/users/{user}/workspace/{workspacenam "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -289,6 +290,7 @@ curl -X GET http://coder-server:8080/api/v2/workspacebuilds/{workspacebuild} \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1015,6 +1017,7 @@ curl -X GET http://coder-server:8080/api/v2/workspacebuilds/{workspacebuild}/sta "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1325,6 +1328,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaces/{workspace}/builds \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1540,6 +1544,7 @@ Status Code **200** | `»»» error` | string | false | | | | `»»» template_version_id` | string(uuid) | false | | | | `»»» workspace_build_id` | string(uuid) | false | | | +| `»» logs_overflowed` | boolean | false | | | | `»» metadata` | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata) | false | | | | `»»» template_display_name` | string | false | | | | `»»» template_icon` | string | false | | | @@ -1815,6 +1820,7 @@ curl -X POST http://coder-server:8080/api/v2/workspaces/{workspace}/builds \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", diff --git a/docs/reference/api/organizations.md b/docs/reference/api/organizations.md index 497e3f56d4e47..d418a1fcba106 100644 --- a/docs/reference/api/organizations.md +++ b/docs/reference/api/organizations.md @@ -407,6 +407,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/provisi "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -457,6 +458,7 @@ Status Code **200** | `»» error` | string | false | | | | `»» template_version_id` | string(uuid) | false | | | | `»» workspace_build_id` | string(uuid) | false | | | +| `» logs_overflowed` | boolean | false | | | | `» metadata` | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata) | false | | | | `»» template_display_name` | string | false | | | | `»» template_icon` | string | false | | | @@ -534,6 +536,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/provisi "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", diff --git a/docs/reference/api/schemas.md b/docs/reference/api/schemas.md index 2abcb2b3204f2..94d95180cd6b9 100644 --- a/docs/reference/api/schemas.md +++ b/docs/reference/api/schemas.md @@ -5853,6 +5853,7 @@ Only certain features set these fields: - FeatureManagedAgentLimit| "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -5890,6 +5891,7 @@ Only certain features set these fields: - FeatureManagedAgentLimit| | `file_id` | string | false | | | | `id` | string | false | | | | `input` | [codersdk.ProvisionerJobInput](#codersdkprovisionerjobinput) | false | | | +| `logs_overflowed` | boolean | false | | | | `metadata` | [codersdk.ProvisionerJobMetadata](#codersdkprovisionerjobmetadata) | false | | | | `organization_id` | string | false | | | | `queue_position` | integer | false | | | @@ -7571,6 +7573,7 @@ Restarts will only happen on weekdays in this list on weeks which line up with W "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -8747,6 +8750,7 @@ If the schedule is empty, the user will be updated to use the default schedule.| "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -9856,6 +9860,7 @@ If the schedule is empty, the user will be updated to use the default schedule.| "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -10587,6 +10592,7 @@ If the schedule is empty, the user will be updated to use the default schedule.| "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", diff --git a/docs/reference/api/templates.md b/docs/reference/api/templates.md index 4c21b3644be2d..bfb4abf493fa6 100644 --- a/docs/reference/api/templates.md +++ b/docs/reference/api/templates.md @@ -472,6 +472,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/templat "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -570,6 +571,7 @@ curl -X GET http://coder-server:8080/api/v2/organizations/{organization}/templat "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -692,6 +694,7 @@ curl -X POST http://coder-server:8080/api/v2/organizations/{organization}/templa "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1251,6 +1254,7 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1324,6 +1328,7 @@ Status Code **200** | `»»» error` | string | false | | | | `»»» template_version_id` | string(uuid) | false | | | | `»»» workspace_build_id` | string(uuid) | false | | | +| `»» logs_overflowed` | boolean | false | | | | `»» metadata` | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata) | false | | | | `»»» template_display_name` | string | false | | | | `»»» template_icon` | string | false | | | @@ -1530,6 +1535,7 @@ curl -X GET http://coder-server:8080/api/v2/templates/{template}/versions/{templ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1603,6 +1609,7 @@ Status Code **200** | `»»» error` | string | false | | | | `»»» template_version_id` | string(uuid) | false | | | | `»»» workspace_build_id` | string(uuid) | false | | | +| `»» logs_overflowed` | boolean | false | | | | `»» metadata` | [codersdk.ProvisionerJobMetadata](schemas.md#codersdkprovisionerjobmetadata) | false | | | | `»»» template_display_name` | string | false | | | | `»»» template_icon` | string | false | | | @@ -1699,6 +1706,7 @@ curl -X GET http://coder-server:8080/api/v2/templateversions/{templateversion} \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1806,6 +1814,7 @@ curl -X PATCH http://coder-server:8080/api/v2/templateversions/{templateversion} "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -2003,6 +2012,7 @@ curl -X POST http://coder-server:8080/api/v2/templateversions/{templateversion}/ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -2076,6 +2086,7 @@ curl -X GET http://coder-server:8080/api/v2/templateversions/{templateversion}/d "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", diff --git a/docs/reference/api/workspaces.md b/docs/reference/api/workspaces.md index debcb421e02e3..d7187259b5bb6 100644 --- a/docs/reference/api/workspaces.md +++ b/docs/reference/api/workspaces.md @@ -107,6 +107,7 @@ of the template will be used. "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -394,6 +395,7 @@ curl -X GET http://coder-server:8080/api/v2/users/{user}/workspace/{workspacenam "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -706,6 +708,7 @@ of the template will be used. "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -996,6 +999,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaces \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1267,6 +1271,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaces/{workspace} \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", @@ -1670,6 +1675,7 @@ curl -X PUT http://coder-server:8080/api/v2/workspaces/{workspace}/dormant \ "template_version_id": "0ba39c92-1f1b-4c32-aa3e-9925d7713eb1", "workspace_build_id": "badaf2eb-96c5-4050-9f1d-db2d39ca5478" }, + "logs_overflowed": true, "metadata": { "template_display_name": "string", "template_icon": "string", diff --git a/docs/reference/cli/provisioner_jobs_list.md b/docs/reference/cli/provisioner_jobs_list.md index 07ad02f419bde..a0bff8554d610 100644 --- a/docs/reference/cli/provisioner_jobs_list.md +++ b/docs/reference/cli/provisioner_jobs_list.md @@ -45,10 +45,10 @@ Select which organization (uuid or name) to use. ### -c, --column -| | | -|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Type | [id\|created at\|started at\|completed at\|canceled at\|error\|error code\|status\|worker id\|worker name\|file id\|tags\|queue position\|queue size\|organization id\|template version id\|workspace build id\|type\|available workers\|template version name\|template id\|template name\|template display name\|template icon\|workspace id\|workspace name\|organization\|queue] | -| Default | created at,id,type,template display name,status,queue,tags | +| | | +|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Type | [id\|created at\|started at\|completed at\|canceled at\|error\|error code\|status\|worker id\|worker name\|file id\|tags\|queue position\|queue size\|organization id\|template version id\|workspace build id\|type\|available workers\|template version name\|template id\|template name\|template display name\|template icon\|workspace id\|workspace name\|logs overflowed\|organization\|queue] | +| Default | created at,id,type,template display name,status,queue,tags | Columns to display in table output. diff --git a/enterprise/cli/testdata/coder_provisioner_jobs_list_--help.golden b/enterprise/cli/testdata/coder_provisioner_jobs_list_--help.golden index f380a0334867c..8e22f78e978f2 100644 --- a/enterprise/cli/testdata/coder_provisioner_jobs_list_--help.golden +++ b/enterprise/cli/testdata/coder_provisioner_jobs_list_--help.golden @@ -11,7 +11,7 @@ OPTIONS: -O, --org string, $CODER_ORGANIZATION Select which organization (uuid or name) to use. - -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|organization|queue] (default: created at,id,type,template display name,status,queue,tags) + -c, --column [id|created at|started at|completed at|canceled at|error|error code|status|worker id|worker name|file id|tags|queue position|queue size|organization id|template version id|workspace build id|type|available workers|template version name|template id|template name|template display name|template icon|workspace id|workspace name|logs overflowed|organization|queue] (default: created at,id,type,template display name,status,queue,tags) Columns to display in table output. -l, --limit int, $CODER_PROVISIONER_JOB_LIST_LIMIT (default: 50) From 61a58f55a03a606f9215b64f9a6883d38872f59c Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Wed, 30 Jul 2025 15:37:15 +0000 Subject: [PATCH 15/21] adjust migration numbers --- ...d.down.sql => 000353_add_provisioner_logs_overflowed.down.sql} | 0 ...lowed.up.sql => 000353_add_provisioner_logs_overflowed.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename coderd/database/migrations/{000350_add_provisioner_logs_overflowed.down.sql => 000353_add_provisioner_logs_overflowed.down.sql} (100%) rename coderd/database/migrations/{000350_add_provisioner_logs_overflowed.up.sql => 000353_add_provisioner_logs_overflowed.up.sql} (100%) diff --git a/coderd/database/migrations/000350_add_provisioner_logs_overflowed.down.sql b/coderd/database/migrations/000353_add_provisioner_logs_overflowed.down.sql similarity index 100% rename from coderd/database/migrations/000350_add_provisioner_logs_overflowed.down.sql rename to coderd/database/migrations/000353_add_provisioner_logs_overflowed.down.sql diff --git a/coderd/database/migrations/000350_add_provisioner_logs_overflowed.up.sql b/coderd/database/migrations/000353_add_provisioner_logs_overflowed.up.sql similarity index 100% rename from coderd/database/migrations/000350_add_provisioner_logs_overflowed.up.sql rename to coderd/database/migrations/000353_add_provisioner_logs_overflowed.up.sql From e7f2ec6b6d5a204370bf047f08092f3098376b6f Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Wed, 30 Jul 2025 15:55:42 +0000 Subject: [PATCH 16/21] update dbauthz and querymetrics for removed query --- coderd/database/dbauthz/dbauthz.go | 8 -------- coderd/database/dbmetrics/querymetrics.go | 7 ------- coderd/database/dbmock/dbmock.go | 15 --------------- coderd/database/querier.go | 1 - coderd/database/queries.sql.go | 16 ---------------- coderd/database/queries/provisionerjoblogs.sql | 8 -------- 6 files changed, 55 deletions(-) diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 3059a91908eb8..72489ea92d572 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -2552,14 +2552,6 @@ func (q *querier) GetProvisionerJobByIDForUpdate(ctx context.Context, id uuid.UU return job, nil } -func (q *querier) GetProvisionerJobLogSize(ctx context.Context, jobID uuid.UUID) (interface{}, error) { - _, err := q.GetProvisionerJobByID(ctx, jobID) - if err != nil { - return nil, err - } - return q.db.GetProvisionerJobLogSize(ctx, jobID) -} - func (q *querier) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID uuid.UUID) ([]database.ProvisionerJobTiming, error) { _, err := q.GetProvisionerJobByID(ctx, jobID) if err != nil { diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 5f738693db1b5..3fffb29966735 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -1258,13 +1258,6 @@ func (m queryMetricsStore) GetProvisionerJobByIDForUpdate(ctx context.Context, i return r0, r1 } -func (m queryMetricsStore) GetProvisionerJobLogSize(ctx context.Context, jobID uuid.UUID) (interface{}, error) { - start := time.Now() - r0, r1 := m.s.GetProvisionerJobLogSize(ctx, jobID) - m.queryLatencies.WithLabelValues("GetProvisionerJobLogSize").Observe(time.Since(start).Seconds()) - return r0, r1 -} - func (m queryMetricsStore) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID uuid.UUID) ([]database.ProvisionerJobTiming, error) { start := time.Now() r0, r1 := m.s.GetProvisionerJobTimingsByJobID(ctx, jobID) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 20b026f49098a..20bc17117e0eb 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -2642,21 +2642,6 @@ func (mr *MockStoreMockRecorder) GetProvisionerJobByIDForUpdate(ctx, id any) *go return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProvisionerJobByIDForUpdate", reflect.TypeOf((*MockStore)(nil).GetProvisionerJobByIDForUpdate), ctx, id) } -// GetProvisionerJobLogSize mocks base method. -func (m *MockStore) GetProvisionerJobLogSize(ctx context.Context, jobID uuid.UUID) (any, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetProvisionerJobLogSize", ctx, jobID) - ret0, _ := ret[0].(any) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetProvisionerJobLogSize indicates an expected call of GetProvisionerJobLogSize. -func (mr *MockStoreMockRecorder) GetProvisionerJobLogSize(ctx, jobID any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProvisionerJobLogSize", reflect.TypeOf((*MockStore)(nil).GetProvisionerJobLogSize), ctx, jobID) -} - // GetProvisionerJobTimingsByJobID mocks base method. func (m *MockStore) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID uuid.UUID) ([]database.ProvisionerJobTiming, error) { m.ctrl.T.Helper() diff --git a/coderd/database/querier.go b/coderd/database/querier.go index f4f486161da7c..a2c6cda1afc4b 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -287,7 +287,6 @@ type sqlcQuerier interface { // Gets a single provisioner job by ID for update. // This is used to securely reap jobs that have been hung/pending for a long time. GetProvisionerJobByIDForUpdate(ctx context.Context, id uuid.UUID) (ProvisionerJob, error) - GetProvisionerJobLogSize(ctx context.Context, jobID uuid.UUID) (interface{}, error) GetProvisionerJobTimingsByJobID(ctx context.Context, jobID uuid.UUID) ([]ProvisionerJobTiming, error) GetProvisionerJobsByIDs(ctx context.Context, ids []uuid.UUID) ([]ProvisionerJob, error) GetProvisionerJobsByIDsWithQueuePosition(ctx context.Context, arg GetProvisionerJobsByIDsWithQueuePositionParams) ([]GetProvisionerJobsByIDsWithQueuePositionRow, error) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index ff4667481893e..1b8358ce8d856 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -8407,22 +8407,6 @@ func (q *sqlQuerier) UpsertProvisionerDaemon(ctx context.Context, arg UpsertProv return i, err } -const getProvisionerJobLogSize = `-- name: GetProvisionerJobLogSize :one - SELECT - COALESCE(SUM(LENGTH(output)), 0) AS total_size - FROM - provisioner_job_logs - WHERE - job_id = $1 -` - -func (q *sqlQuerier) GetProvisionerJobLogSize(ctx context.Context, jobID uuid.UUID) (interface{}, error) { - row := q.db.QueryRowContext(ctx, getProvisionerJobLogSize, jobID) - var total_size interface{} - err := row.Scan(&total_size) - return total_size, err -} - const getProvisionerLogsAfterID = `-- name: GetProvisionerLogsAfterID :many SELECT job_id, created_at, source, level, stage, output, id diff --git a/coderd/database/queries/provisionerjoblogs.sql b/coderd/database/queries/provisionerjoblogs.sql index 715bd2280a3e4..c0ef188bdd382 100644 --- a/coderd/database/queries/provisionerjoblogs.sql +++ b/coderd/database/queries/provisionerjoblogs.sql @@ -8,14 +8,6 @@ WHERE AND ( id > @created_after ) ORDER BY id ASC; - --- name: GetProvisionerJobLogSize :one - SELECT - COALESCE(SUM(LENGTH(output)), 0) AS total_size - FROM - provisioner_job_logs - WHERE - job_id = @job_id; -- name: InsertProvisionerJobLogs :many INSERT INTO From 4a2a7f825c8cc8d075d64b457d6bb6abb559c024 Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Wed, 30 Jul 2025 20:33:53 +0000 Subject: [PATCH 17/21] add rbac tests for provisioner job logs length and overflowed queries --- coderd/database/dbauthz/dbauthz_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index bcf0caa95c365..d0075977143b8 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -4339,6 +4339,20 @@ func (s *MethodTestSuite) TestSystemFunctions() { UpdatedAt: time.Now(), }).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate) })) + s.Run("UpdateProvisionerJobLogsLength", s.Subtest(func(db database.Store, check *expects) { + j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) + check.Args(database.UpdateProvisionerJobLogsLengthParams{ + ID: j.ID, + LogsLength: 100, + }).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate) + })) + s.Run("UpdateProvisionerJobLogsOverflowed", s.Subtest(func(db database.Store, check *expects) { + j := dbgen.ProvisionerJob(s.T(), db, nil, database.ProvisionerJob{}) + check.Args(database.UpdateProvisionerJobLogsOverflowedParams{ + ID: j.ID, + LogsOverflowed: true, + }).Asserts(rbac.ResourceProvisionerJobs, policy.ActionUpdate) + })) s.Run("InsertProvisionerJob", s.Subtest(func(db database.Store, check *expects) { dbtestutil.DisableForeignKeysAndTriggers(s.T(), db) check.Args(database.InsertProvisionerJobParams{ From 8115e4aa31d75dd65a578717ec93b1a05ca326d7 Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Wed, 30 Jul 2025 22:22:52 +0000 Subject: [PATCH 18/21] reshuffling --- ...353_add_provisioner_logs_overflowed.up.sql | 2 +- .../provisionerdserver/provisionerdserver.go | 74 ++++++++++++------- .../WorkspaceBuildLogs/WorkspaceBuildLogs.tsx | 3 +- .../WorkspaceBuildPageView.tsx | 4 +- 4 files changed, 53 insertions(+), 30 deletions(-) diff --git a/coderd/database/migrations/000353_add_provisioner_logs_overflowed.up.sql b/coderd/database/migrations/000353_add_provisioner_logs_overflowed.up.sql index 6f47ac790628a..80f58cf5c6693 100644 --- a/coderd/database/migrations/000353_add_provisioner_logs_overflowed.up.sql +++ b/coderd/database/migrations/000353_add_provisioner_logs_overflowed.up.sql @@ -3,4 +3,4 @@ ALTER TABLE provisioner_jobs ADD COLUMN logs_overflowed boolean NOT NULL DEFAULT false; COMMENT ON COLUMN provisioner_jobs.logs_length IS 'Total length of provisioner logs'; - COMMENT ON COLUMN provisioner_jobs.logs_overflowed IS 'Whether the provisioner logs overflowed in length'; \ No newline at end of file + COMMENT ON COLUMN provisioner_jobs.logs_overflowed IS 'Whether the provisioner logs overflowed in length'; diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index ab99e9dad8936..ca959729d34e2 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -907,43 +907,65 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest) insertParams := database.InsertProvisionerJobLogsParams{ JobID: parsedID, } + + newLogSize := 0 + overflowedErrorMsg := "Provisioner logs exceeded the max size of 1MB. Will not continue to write provisioner logs for workspace build." + lenErrMsg := len(overflowedErrorMsg) + + var ( + createdAt time.Time + level database.LogLevel + stage string + source database.LogSource + output string + ) + for _, log := range request.Logs { - logLevel, err := convertLogLevel(log.Level) + // Build our log params + level, err = convertLogLevel(log.Level) if err != nil { return nil, xerrors.Errorf("convert log level: %w", err) } - logSource, err := convertLogSource(log.Source) + source, err = convertLogSource(log.Source) if err != nil { return nil, xerrors.Errorf("convert log source: %w", err) } - insertParams.CreatedAt = append(insertParams.CreatedAt, time.UnixMilli(log.CreatedAt)) - insertParams.Level = append(insertParams.Level, logLevel) - insertParams.Stage = append(insertParams.Stage, log.Stage) - insertParams.Source = append(insertParams.Source, logSource) - insertParams.Output = append(insertParams.Output, log.Output) + createdAt = time.UnixMilli(log.CreatedAt) + stage = log.Stage + output = log.Output + + // Check if we would overflow the job logs (not leaving enough room for the error message) + willOverflow := int64(job.LogsLength)+int64(newLogSize)+int64(lenErrMsg)+int64(len(output)) > 1048576 + if willOverflow { + s.Logger.Debug(ctx, "provisioner job logs overflowed 1MB size limit in database.", slog.F("job_id", parsedID)) + err = s.Database.UpdateProvisionerJobLogsOverflowed(ctx, database.UpdateProvisionerJobLogsOverflowedParams{ + ID: parsedID, + LogsOverflowed: true, + }) + if err != nil { + s.Logger.Error(ctx, "failed to set logs overflowed flag", slog.F("job_id", parsedID), slog.Error(err)) + } + + level = database.LogLevelWarn + output = overflowedErrorMsg + } + + newLogSize += len(output) + + insertParams.CreatedAt = append(insertParams.CreatedAt, createdAt) + insertParams.Level = append(insertParams.Level, level) + insertParams.Stage = append(insertParams.Stage, stage) + insertParams.Source = append(insertParams.Source, source) + insertParams.Output = append(insertParams.Output, output) s.Logger.Debug(ctx, "job log", slog.F("job_id", parsedID), - slog.F("stage", log.Stage), - slog.F("output", log.Output)) - } + slog.F("stage", stage), + slog.F("output", output)) - newLogSize := 0 - for _, log := range request.Logs { - newLogSize += len(log.Output) - } - - willOverflow := int64(job.LogsLength)+int64(newLogSize) > 1048576 - if willOverflow { - err = s.Database.UpdateProvisionerJobLogsOverflowed(ctx, database.UpdateProvisionerJobLogsOverflowedParams{ - ID: parsedID, - LogsOverflowed: true, - }) - if err != nil { - s.Logger.Error(ctx, "failed to set logs overflowed flag", slog.F("job_id", parsedID), slog.Error(err)) + // Don't write any more logs because there's no room. + if willOverflow { + break } - return &proto.UpdateJobResponse{ - Canceled: job.CanceledAt.Valid, - }, nil } err = s.Database.UpdateProvisionerJobLogsLength(ctx, database.UpdateProvisionerJobLogsLengthParams{ diff --git a/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx b/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx index 2715af8848e41..20c929406d32c 100644 --- a/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx +++ b/site/src/modules/workspaces/WorkspaceBuildLogs/WorkspaceBuildLogs.tsx @@ -64,7 +64,8 @@ export const WorkspaceBuildLogs: FC = ({ created_at: new Date().toISOString(), log_level: "error", log_source: "provisioner", - output: "Provisioner logs exceeded the max size of 1MB!", + output: + "Provisioner logs exceeded the max size of 1MB. Will not continue to write provisioner logs for workspace build.", stage: "overflow", }); } diff --git a/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx b/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx index f20086c4799c4..3a45653557dcc 100644 --- a/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx +++ b/site/src/pages/WorkspaceBuildPage/WorkspaceBuildPageView.tsx @@ -222,8 +222,8 @@ export const WorkspaceBuildPageView: FC = ({ borderBottom: `1px solid ${theme.palette.divider}`, }} > - Provisioner logs exceeded the maximum size of 1MB and were - truncated. + Provisioner logs exceeded the max size of 1MB. Will not continue + to write provisioner logs for workspace build. )} From c09f0f6cd0fbd3527121b99d198a350ed1261e65 Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Wed, 30 Jul 2025 22:27:46 +0000 Subject: [PATCH 19/21] update migration numbers --- ...d.down.sql => 000354_add_provisioner_logs_overflowed.down.sql} | 0 ...lowed.up.sql => 000354_add_provisioner_logs_overflowed.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename coderd/database/migrations/{000353_add_provisioner_logs_overflowed.down.sql => 000354_add_provisioner_logs_overflowed.down.sql} (100%) rename coderd/database/migrations/{000353_add_provisioner_logs_overflowed.up.sql => 000354_add_provisioner_logs_overflowed.up.sql} (100%) diff --git a/coderd/database/migrations/000353_add_provisioner_logs_overflowed.down.sql b/coderd/database/migrations/000354_add_provisioner_logs_overflowed.down.sql similarity index 100% rename from coderd/database/migrations/000353_add_provisioner_logs_overflowed.down.sql rename to coderd/database/migrations/000354_add_provisioner_logs_overflowed.down.sql diff --git a/coderd/database/migrations/000353_add_provisioner_logs_overflowed.up.sql b/coderd/database/migrations/000354_add_provisioner_logs_overflowed.up.sql similarity index 100% rename from coderd/database/migrations/000353_add_provisioner_logs_overflowed.up.sql rename to coderd/database/migrations/000354_add_provisioner_logs_overflowed.up.sql From 1ab1347d43016061815b389b4975d6d06292d76c Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Wed, 30 Jul 2025 22:29:58 +0000 Subject: [PATCH 20/21] remove period from slog log --- coderd/provisionerdserver/provisionerdserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coderd/provisionerdserver/provisionerdserver.go b/coderd/provisionerdserver/provisionerdserver.go index ca959729d34e2..94173703c467d 100644 --- a/coderd/provisionerdserver/provisionerdserver.go +++ b/coderd/provisionerdserver/provisionerdserver.go @@ -937,7 +937,7 @@ func (s *server) UpdateJob(ctx context.Context, request *proto.UpdateJobRequest) // Check if we would overflow the job logs (not leaving enough room for the error message) willOverflow := int64(job.LogsLength)+int64(newLogSize)+int64(lenErrMsg)+int64(len(output)) > 1048576 if willOverflow { - s.Logger.Debug(ctx, "provisioner job logs overflowed 1MB size limit in database.", slog.F("job_id", parsedID)) + s.Logger.Debug(ctx, "provisioner job logs overflowed 1MB size limit in database", slog.F("job_id", parsedID)) err = s.Database.UpdateProvisionerJobLogsOverflowed(ctx, database.UpdateProvisionerJobLogsOverflowedParams{ ID: parsedID, LogsOverflowed: true, From a03a956c8074a6e11cb46d0f67bad3745fe84c76 Mon Sep 17 00:00:00 2001 From: Benjamin Peinhardt Date: Wed, 30 Jul 2025 23:12:33 +0000 Subject: [PATCH 21/21] migration rename --- ...d.down.sql => 000355_add_provisioner_logs_overflowed.down.sql} | 0 ...lowed.up.sql => 000355_add_provisioner_logs_overflowed.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename coderd/database/migrations/{000354_add_provisioner_logs_overflowed.down.sql => 000355_add_provisioner_logs_overflowed.down.sql} (100%) rename coderd/database/migrations/{000354_add_provisioner_logs_overflowed.up.sql => 000355_add_provisioner_logs_overflowed.up.sql} (100%) diff --git a/coderd/database/migrations/000354_add_provisioner_logs_overflowed.down.sql b/coderd/database/migrations/000355_add_provisioner_logs_overflowed.down.sql similarity index 100% rename from coderd/database/migrations/000354_add_provisioner_logs_overflowed.down.sql rename to coderd/database/migrations/000355_add_provisioner_logs_overflowed.down.sql diff --git a/coderd/database/migrations/000354_add_provisioner_logs_overflowed.up.sql b/coderd/database/migrations/000355_add_provisioner_logs_overflowed.up.sql similarity index 100% rename from coderd/database/migrations/000354_add_provisioner_logs_overflowed.up.sql rename to coderd/database/migrations/000355_add_provisioner_logs_overflowed.up.sql