Fix: Use Check IDs instead of timestamp to process checks

This commit is contained in:
Alex Holliday committed 2026-09-17 11:57:19 -07:00
1 parent 9102335d3e
commit fa0c0abcac
13 files changed
+491 -56

No files matched your search

@@ -9,6 +9,7 @@ import type {
PageSpeedChecksResult,
UptimeChecksResult,
} from "@/domain/checks/check.type.js";
import { PendingCheck } from "@/domain/jobs/job.type.js";
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
import { CheckFilter, DateRange } from "@/types/query.js";
@@ -43,7 +44,7 @@ export interface IChecksRepository {
): Promise<UptimeChecksResult | HardwareChecksResult | PageSpeedChecksResult | DockerChecksResult>;
findDockerContainerChecks(monitorId: string, containerName: string, dateRange: DateRange): Promise<Omit<DockerContainerStats, "restartsInRange">>;
findSummaryByTeamId(teamId: string, dateRange: DateRange): Promise<ChecksSummary>;
findUnevaluatedByMonitorId(monitorId: string, since: number): Promise<Check[]>;
findUnevaluatedByMonitorId(monitorId: string, pending: PendingCheck[]): Promise<Check[]>;
getDailyStatusBuckets(monitorIds: string[], days: number, timezone: string): Promise<DailyCheckBucket[]>;
// update
@@ -33,6 +33,7 @@ import {
getDockerTotalChecks,
getDockerUpChecks,
} from "@/domain/checks/check.docker.aggregation.js";
import { PendingCheck } from "@/domain/jobs/job.type.js";
const SERVICE_NAME = "ChecksRepository";
@@ -360,10 +361,17 @@ class MongoChecksRepository implements IChecksRepository {
};
};
findUnevaluatedByMonitorId = async (monitorId: string, since: number) => {
findUnevaluatedByMonitorId = async (monitorId: string, pending: PendingCheck[]) => {
if (pending.length === 0) return [];
const ids = pending.map((entry) => new mongoose.Types.ObjectId(entry.checkId));
const times = pending.map((entry) => entry.createdAt);
const docs = await CheckModel.find({
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
createdAt: { $gt: new Date(since) },
createdAt: { $gte: new Date(Math.min(...times)), $lte: new Date(Math.max(...times)) },
_id: { $in: ids },
})
.sort({ createdAt: 1 })
.lean<CheckDocument[]>();
@@ -12,7 +12,6 @@ const SERVICE_NAME = "checkService";
export interface ICheckService {
createChecks(checks: Check[]): Promise<Check[]>;
toLastEvaluatedAt(check: Check): number;
toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
getChecksByMonitor(params: {
@@ -55,10 +54,6 @@ export class CheckService implements ICheckService {
return this.checksRepository.createChecks(checks);
};
toLastEvaluatedAt = (check: Check) => {
return new Date(check.createdAt).getTime();
};
toCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
+12
View File
@@ -47,6 +47,18 @@ const JobSchema = new Schema<JobDocument>(
default: null,
},
// Pending checks, inserted but not yet evaluated
pendingChecks: {
type: [
{
checkId: { type: String, required: true },
createdAt: { type: Number, required: true },
},
],
default: [],
_id: false,
},
// Observability
runCount: {
type: Number,
@@ -1,4 +1,4 @@
import { Job, JobSeed, JobType } from "@/domain/jobs/job.type.js";
import { Job, JobSeed, JobType, PendingCheck } from "@/domain/jobs/job.type.js";
export type JobPageQuery = {
page?: number;
@@ -32,9 +32,12 @@ export interface IJobsRepository {
recordOneShot(id: string, now: number): Promise<boolean>;
// ********************
// Hand off to evaluator // This is accomplished by creating an "evaluation" type job
// Hand off to evaluator // This is accomplished pushing newly inserted Check IDs onto the evaluation row
// ********************
upsertEvaluate(monitorId: string, now: number): Promise<boolean>;
upsertEvaluate(monitorId: string, pending: PendingCheck[], now: number): Promise<boolean>;
// Removes processed checks from the evaluate row.
pullEvaluated(id: string, checkIds: string[]): Promise<boolean>;
// ********************
// Job CRUD
+32 -14
View File
@@ -1,6 +1,6 @@
import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js";
import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js";
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId, PendingCheck } from "@/domain/jobs/job.type.js";
// Cap for rearm interval, never want to wait more than 15s.
const REARM_JITTER_MAX_MS = 15_000;
@@ -18,6 +18,7 @@ class MongoJobsRepository implements IJobsRepository {
intervalMs: doc.intervalMs,
lockedBy: doc.lockedBy,
lockedUntil: doc.lockedUntil,
pendingChecks: doc.pendingChecks ?? [],
runCount: doc.runCount,
failCount: doc.failCount,
lastFinishedAt: doc.lastFinishedAt,
@@ -146,24 +147,27 @@ class MongoJobsRepository implements IJobsRepository {
_id: id,
lockedBy: this.workerId, // Only the worker that claimed can record success
},
{
$set: {
nextScheduledAt: PARKED,
lockedBy: null, // Remove lock
lockedUntil: null,
lastFinishedAt: now,
failCount: 0,
lastFailReason: null,
[
{
$set: {
// Re-arm if more checks were pushed while this run held the lease; otherwise park
nextScheduledAt: {
$cond: [{ $gt: [{ $size: { $ifNull: ["$pendingChecks", []] } }, 0] }, now, PARKED],
},
lockedBy: null, // Remove lock
lockedUntil: null,
lastFinishedAt: now,
failCount: 0,
lastFailReason: null,
runCount: { $add: ["$runCount", 1] },
},
},
$inc: {
runCount: 1, // Increment run count
},
}
]
);
return res.modifiedCount === 1;
};
upsertEvaluate = async (monitorId: string, now: number) => {
upsertEvaluate = async (monitorId: string, pending: PendingCheck[], now: number) => {
const filter = { type: "evaluate", refId: monitorId };
const update = {
$setOnInsert: {
@@ -181,6 +185,9 @@ class MongoJobsRepository implements IJobsRepository {
$min: {
nextScheduledAt: now, // no op if there is already a pending evaluation that is older
},
$push: {
pendingChecks: { $each: pending }, // accumulate pending checks
},
};
try {
const res = await JobModel.updateOne(filter, update, { upsert: true });
@@ -195,6 +202,17 @@ class MongoJobsRepository implements IJobsRepository {
throw error;
}
};
pullEvaluated = async (id: string, checkIds: string[]) => {
if (checkIds.length === 0) return true;
const res = await JobModel.updateOne(
{ _id: id, lockedBy: this.workerId }, // Only the worker that claimed can pull checks
{ $pull: { pendingChecks: { checkId: { $in: checkIds } } } }
);
return res.modifiedCount === 1;
};
upsertJob = async (job: JobSeed) => {
const res = await JobModel.updateOne(
{ _id: job.id },
+8
View File
@@ -4,6 +4,11 @@ export const LOCK_MS = 60_000;
export const BACKOFF_MS = 5_000;
export const PARKED = Number.MAX_SAFE_INTEGER;
export type PendingCheck = {
checkId: string;
createdAt: number; // epoch ms, copied from Check.createdAt
};
// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null: cleanup, egress) are just the type.
export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`);
@@ -21,6 +26,9 @@ export type Job = {
lockedBy: string | null; // workerId holding the lease
lockedUntil: number | null; // epoch ms; lease expiry => reclaimable after a crash
// Evaluate these rows only. These are checks that are inserted but not yet evaluated
pendingChecks: PendingCheck[];
// observability (replaces what getMetrics/getJobs)
runCount: number;
failCount: number;