Fix: Use Check IDs instead of timestamp to process checks

server/src/domain/checks/check.repository.interface.ts

@@ -9,6 +9,7 @@ import type {
 	PageSpeedChecksResult,
 	UptimeChecksResult,
 } from "@/domain/checks/check.type.js";
+import { PendingCheck } from "@/domain/jobs/job.type.js";
 import type { MonitorType } from "@/domain/monitors/monitor.type.js";
 import { CheckFilter, DateRange } from "@/types/query.js";
 
@@ -43,7 +44,7 @@ export interface IChecksRepository {
 	): Promise<UptimeChecksResult | HardwareChecksResult | PageSpeedChecksResult | DockerChecksResult>;
 	findDockerContainerChecks(monitorId: string, containerName: string, dateRange: DateRange): Promise<Omit<DockerContainerStats, "restartsInRange">>;
 	findSummaryByTeamId(teamId: string, dateRange: DateRange): Promise<ChecksSummary>;
-	findUnevaluatedByMonitorId(monitorId: string, since: number): Promise<Check[]>;
+	findUnevaluatedByMonitorId(monitorId: string, pending: PendingCheck[]): Promise<Check[]>;
 	getDailyStatusBuckets(monitorIds: string[], days: number, timezone: string): Promise<DailyCheckBucket[]>;
 
 	// update

server/src/domain/checks/check.repository.mongo.ts

@@ -33,6 +33,7 @@ import {
 	getDockerTotalChecks,
 	getDockerUpChecks,
 } from "@/domain/checks/check.docker.aggregation.js";
+import { PendingCheck } from "@/domain/jobs/job.type.js";
 
 const SERVICE_NAME = "ChecksRepository";
 
@@ -360,10 +361,17 @@ class MongoChecksRepository implements IChecksRepository {
 		};
 	};
 
-	findUnevaluatedByMonitorId = async (monitorId: string, since: number) => {
+	findUnevaluatedByMonitorId = async (monitorId: string, pending: PendingCheck[]) => {
+		if (pending.length === 0) return [];
+
+		const ids = pending.map((entry) => new mongoose.Types.ObjectId(entry.checkId));
+
+		const times = pending.map((entry) => entry.createdAt);
+
 		const docs = await CheckModel.find({
 			"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
-			createdAt: { $gt: new Date(since) },
+			createdAt: { $gte: new Date(Math.min(...times)), $lte: new Date(Math.max(...times)) },
+			_id: { $in: ids },
 		})
 			.sort({ createdAt: 1 })
 			.lean<CheckDocument[]>();

server/src/domain/checks/check.service.ts

@@ -12,7 +12,6 @@ const SERVICE_NAME = "checkService";
 
 export interface ICheckService {
 	createChecks(checks: Check[]): Promise<Check[]>;
-	toLastEvaluatedAt(check: Check): number;
 	toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
 	toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
 	getChecksByMonitor(params: {
@@ -55,10 +54,6 @@ export class CheckService implements ICheckService {
 		return this.checksRepository.createChecks(checks);
 	};
 
-	toLastEvaluatedAt = (check: Check) => {
-		return new Date(check.createdAt).getTime();
-	};
-
 	toCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
 		const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
 

server/src/domain/jobs/job.model.ts

@@ -47,6 +47,18 @@ const JobSchema = new Schema<JobDocument>(
 			default: null,
 		},
 
+		// Pending checks, inserted but not yet evaluated
+		pendingChecks: {
+			type: [
+				{
+					checkId: { type: String, required: true },
+					createdAt: { type: Number, required: true },
+				},
+			],
+			default: [],
+			_id: false,
+		},
+
 		// Observability
 		runCount: {
 			type: Number,

server/src/domain/jobs/job.repository.interface.ts

@@ -1,4 +1,4 @@
-import { Job, JobSeed, JobType } from "@/domain/jobs/job.type.js";
+import { Job, JobSeed, JobType, PendingCheck } from "@/domain/jobs/job.type.js";
 
 export type JobPageQuery = {
 	page?: number;
@@ -32,9 +32,12 @@ export interface IJobsRepository {
 	recordOneShot(id: string, now: number): Promise<boolean>;
 
 	// ********************
-	// Hand off to evaluator // This is accomplished by creating an "evaluation" type job
+	// Hand off to evaluator // This is accomplished pushing newly inserted Check IDs onto the evaluation row
 	// ********************
-	upsertEvaluate(monitorId: string, now: number): Promise<boolean>;
+	upsertEvaluate(monitorId: string, pending: PendingCheck[], now: number): Promise<boolean>;
+
+	// Removes processed checks from the evaluate row.
+	pullEvaluated(id: string, checkIds: string[]): Promise<boolean>;
 
 	// ********************
 	// Job CRUD

server/src/domain/jobs/job.repository.mongo.ts

@@ -1,6 +1,6 @@
 import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js";
 import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
-import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js";
+import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId, PendingCheck } from "@/domain/jobs/job.type.js";
 
 // Cap for rearm interval, never want to wait more than 15s.
 const REARM_JITTER_MAX_MS = 15_000;
@@ -18,6 +18,7 @@ class MongoJobsRepository implements IJobsRepository {
 			intervalMs: doc.intervalMs,
 			lockedBy: doc.lockedBy,
 			lockedUntil: doc.lockedUntil,
+			pendingChecks: doc.pendingChecks ?? [],
 			runCount: doc.runCount,
 			failCount: doc.failCount,
 			lastFinishedAt: doc.lastFinishedAt,
@@ -146,24 +147,27 @@ class MongoJobsRepository implements IJobsRepository {
 				_id: id,
 				lockedBy: this.workerId, // Only the worker that claimed can record success
 			},
-			{
-				$set: {
-					nextScheduledAt: PARKED,
-					lockedBy: null, // Remove lock
-					lockedUntil: null,
-					lastFinishedAt: now,
-					failCount: 0,
-					lastFailReason: null,
+			[
+				{
+					$set: {
+						// Re-arm if more checks were pushed while this run held the lease; otherwise park
+						nextScheduledAt: {
+							$cond: [{ $gt: [{ $size: { $ifNull: ["$pendingChecks", []] } }, 0] }, now, PARKED],
+						},
+						lockedBy: null, // Remove lock
+						lockedUntil: null,
+						lastFinishedAt: now,
+						failCount: 0,
+						lastFailReason: null,
+						runCount: { $add: ["$runCount", 1] },
+					},
 				},
-				$inc: {
-					runCount: 1, // Increment run count
-				},
-			}
+			]
 		);
 		return res.modifiedCount === 1;
 	};
 
-	upsertEvaluate = async (monitorId: string, now: number) => {
+	upsertEvaluate = async (monitorId: string, pending: PendingCheck[], now: number) => {
 		const filter = { type: "evaluate", refId: monitorId };
 		const update = {
 			$setOnInsert: {
@@ -181,6 +185,9 @@ class MongoJobsRepository implements IJobsRepository {
 			$min: {
 				nextScheduledAt: now, // no op if there is already a pending evaluation that is older
 			},
+			$push: {
+				pendingChecks: { $each: pending }, // accumulate pending checks
+			},
 		};
 		try {
 			const res = await JobModel.updateOne(filter, update, { upsert: true });
@@ -195,6 +202,17 @@ class MongoJobsRepository implements IJobsRepository {
 			throw error;
 		}
 	};
+
+	pullEvaluated = async (id: string, checkIds: string[]) => {
+		if (checkIds.length === 0) return true;
+
+		const res = await JobModel.updateOne(
+			{ _id: id, lockedBy: this.workerId }, // Only the worker that claimed can pull checks
+			{ $pull: { pendingChecks: { checkId: { $in: checkIds } } } }
+		);
+		return res.modifiedCount === 1;
+	};
+
 	upsertJob = async (job: JobSeed) => {
 		const res = await JobModel.updateOne(
 			{ _id: job.id },

server/src/domain/jobs/job.type.ts

@@ -4,6 +4,11 @@ export const LOCK_MS = 60_000;
 export const BACKOFF_MS = 5_000;
 export const PARKED = Number.MAX_SAFE_INTEGER;
 
+export type PendingCheck = {
+	checkId: string;
+	createdAt: number; // epoch ms, copied from Check.createdAt
+};
+
 // Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null: cleanup, egress) are just the type.
 export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`);
 
@@ -21,6 +26,9 @@ export type Job = {
 	lockedBy: string | null; // workerId holding the lease
 	lockedUntil: number | null; // epoch ms; lease expiry => reclaimable after a crash
 
+	// Evaluate these rows only.  These are checks that are inserted but not yet evaluated
+	pendingChecks: PendingCheck[];
+
 	// observability (replaces what getMetrics/getJobs)
 	runCount: number;
 	failCount: number;

server/src/service/bufferService.ts

@@ -7,12 +7,14 @@ import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js"
 import { ICheckService } from "../domain/checks/check.service.js";
 import { DockerLog } from "@/domain/docker/docker-log.type.js";
 import { IDockerLogsService } from "@/domain/docker/docker-log.service.js";
+import { PendingCheck } from "@/domain/jobs/job.type.js";
 const SERVICE_NAME = "BufferService";
 
 export interface IBufferService {
 	addToBuffer(check: Check): void;
 	addGeoCheckToBuffer(geoCheck: GeoCheck): void;
 	addDockerLogToBuffer(dockerLog: DockerLog): void;
+	ingestChecks(checks: Check[]): Promise<void>;
 	scheduleNextFlush(): void;
 	flushBuffer(): Promise<void>;
 	flushGeoBuffer(): Promise<void>;
@@ -99,6 +101,25 @@ export class BufferService implements IBufferService {
 		}
 	}
 
+	ingestChecks = async (checks: Check[]) => {
+		if (checks.length === 0) return;
+		await this.checksService.createChecks(checks);
+
+		const byMonitor = new Map<string, PendingCheck[]>();
+		for (const check of checks) {
+			const pending = byMonitor.get(check.metadata.monitorId) ?? [];
+			pending.push({ checkId: check.id, createdAt: new Date(check.createdAt).getTime() });
+			byMonitor.set(check.metadata.monitorId, pending);
+		}
+
+		const now = Date.now();
+		const ops: Promise<boolean>[] = [];
+		for (const [monitorId, pendingChecks] of byMonitor) {
+			ops.push(this.jobsRepository.upsertEvaluate(monitorId, pendingChecks, now));
+		}
+		await Promise.all(ops);
+	};
+
 	scheduleNextFlush() {
 		if (this.bufferTimer) {
 			clearTimeout(this.bufferTimer);
@@ -121,6 +142,7 @@ export class BufferService implements IBufferService {
 			}
 		}, this.BUFFER_TIMEOUT);
 	}
+
 	async flushBuffer() {
 		if (this.buffer.length === 0) {
 			return;
@@ -134,11 +156,7 @@ export class BufferService implements IBufferService {
 				service: this.SERVICE_NAME,
 				method: "flushBuffer",
 			});
-			await this.checksService.createChecks(batch);
-			// Need to evaluate checks when they are flushed
-			const monitorIds = [...new Set(batch.map((check) => check.metadata.monitorId))];
-			const now = Date.now();
-			await Promise.all(monitorIds.map((monitorId) => this.jobsRepository.upsertEvaluate(monitorId, now)));
+			await this.ingestChecks(batch);
 		} catch (error: unknown) {
 			this.logger.error({
 				message: error instanceof Error ? error.message : "Unknown error",

server/src/worker/worker.db-queue.ts

@@ -120,19 +120,29 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
 	// ********************
 
 	private runEvaluate = async (job: Job) => {
-		if (!job.refId) return;
+		if (!job.refId || job.pendingChecks.length === 0) return;
+		const pendingCheckIds = job.pendingChecks.map((entry) => entry.checkId);
+
 		const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
-		if (!monitor) return;
-		const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, monitor.lastEvaluatedAt);
+		if (!monitor) {
+			await this.jobsRepository.pullEvaluated(job.id, pendingCheckIds);
+			return;
+		}
+		const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, job.pendingChecks);
 
 		let current = monitor;
 		for (const check of checks) {
 			const status = this.checkService.toStatusResponse(check);
 			const evaluation = await this.checkEvaluator.evaluate(status, check, current);
 			await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
-			await this.monitorsRepository.updateById(job.refId, current.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) });
+			await this.jobsRepository.pullEvaluated(job.id, [check.id]);
 			current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
 		}
+
+		// Clean up ids potentially orphaned retention cleanup
+		const found = new Set(checks.map((check) => check.id));
+		const missing = pendingCheckIds.filter((checkId) => !found.has(checkId));
+		if (missing.length > 0) await this.jobsRepository.pullEvaluated(job.id, missing);
 	};
 
 	private runGeoCheck = async (job: Job) => {