Fix: Use Check IDs instead of timestamp to process checks
This commit is contained in:
1 parent
9102335d3e
commit
fa0c0abcac
13 files changed
+491
-56
No files matched your search
@@ -9,6 +9,7 @@ import type {
|
||||
PageSpeedChecksResult,
|
||||
UptimeChecksResult,
|
||||
} from "@/domain/checks/check.type.js";
|
||||
import { PendingCheck } from "@/domain/jobs/job.type.js";
|
||||
import type { MonitorType } from "@/domain/monitors/monitor.type.js";
|
||||
import { CheckFilter, DateRange } from "@/types/query.js";
|
||||
|
||||
@@ -43,7 +44,7 @@ export interface IChecksRepository {
|
||||
): Promise<UptimeChecksResult | HardwareChecksResult | PageSpeedChecksResult | DockerChecksResult>;
|
||||
findDockerContainerChecks(monitorId: string, containerName: string, dateRange: DateRange): Promise<Omit<DockerContainerStats, "restartsInRange">>;
|
||||
findSummaryByTeamId(teamId: string, dateRange: DateRange): Promise<ChecksSummary>;
|
||||
findUnevaluatedByMonitorId(monitorId: string, since: number): Promise<Check[]>;
|
||||
findUnevaluatedByMonitorId(monitorId: string, pending: PendingCheck[]): Promise<Check[]>;
|
||||
getDailyStatusBuckets(monitorIds: string[], days: number, timezone: string): Promise<DailyCheckBucket[]>;
|
||||
|
||||
// update
|
||||
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
getDockerTotalChecks,
|
||||
getDockerUpChecks,
|
||||
} from "@/domain/checks/check.docker.aggregation.js";
|
||||
import { PendingCheck } from "@/domain/jobs/job.type.js";
|
||||
|
||||
const SERVICE_NAME = "ChecksRepository";
|
||||
|
||||
@@ -360,10 +361,17 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
};
|
||||
};
|
||||
|
||||
findUnevaluatedByMonitorId = async (monitorId: string, since: number) => {
|
||||
findUnevaluatedByMonitorId = async (monitorId: string, pending: PendingCheck[]) => {
|
||||
if (pending.length === 0) return [];
|
||||
|
||||
const ids = pending.map((entry) => new mongoose.Types.ObjectId(entry.checkId));
|
||||
|
||||
const times = pending.map((entry) => entry.createdAt);
|
||||
|
||||
const docs = await CheckModel.find({
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
createdAt: { $gt: new Date(since) },
|
||||
createdAt: { $gte: new Date(Math.min(...times)), $lte: new Date(Math.max(...times)) },
|
||||
_id: { $in: ids },
|
||||
})
|
||||
.sort({ createdAt: 1 })
|
||||
.lean<CheckDocument[]>();
|
||||
|
||||
@@ -12,7 +12,6 @@ const SERVICE_NAME = "checkService";
|
||||
|
||||
export interface ICheckService {
|
||||
createChecks(checks: Check[]): Promise<Check[]>;
|
||||
toLastEvaluatedAt(check: Check): number;
|
||||
toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
|
||||
toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
|
||||
getChecksByMonitor(params: {
|
||||
@@ -55,10 +54,6 @@ export class CheckService implements ICheckService {
|
||||
return this.checksRepository.createChecks(checks);
|
||||
};
|
||||
|
||||
toLastEvaluatedAt = (check: Check) => {
|
||||
return new Date(check.createdAt).getTime();
|
||||
};
|
||||
|
||||
toCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
|
||||
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
|
||||
|
||||
|
||||
@@ -47,6 +47,18 @@ const JobSchema = new Schema<JobDocument>(
|
||||
default: null,
|
||||
},
|
||||
|
||||
// Pending checks, inserted but not yet evaluated
|
||||
pendingChecks: {
|
||||
type: [
|
||||
{
|
||||
checkId: { type: String, required: true },
|
||||
createdAt: { type: Number, required: true },
|
||||
},
|
||||
],
|
||||
default: [],
|
||||
_id: false,
|
||||
},
|
||||
|
||||
// Observability
|
||||
runCount: {
|
||||
type: Number,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Job, JobSeed, JobType } from "@/domain/jobs/job.type.js";
|
||||
import { Job, JobSeed, JobType, PendingCheck } from "@/domain/jobs/job.type.js";
|
||||
|
||||
export type JobPageQuery = {
|
||||
page?: number;
|
||||
@@ -32,9 +32,12 @@ export interface IJobsRepository {
|
||||
recordOneShot(id: string, now: number): Promise<boolean>;
|
||||
|
||||
// ********************
|
||||
// Hand off to evaluator // This is accomplished by creating an "evaluation" type job
|
||||
// Hand off to evaluator // This is accomplished pushing newly inserted Check IDs onto the evaluation row
|
||||
// ********************
|
||||
upsertEvaluate(monitorId: string, now: number): Promise<boolean>;
|
||||
upsertEvaluate(monitorId: string, pending: PendingCheck[], now: number): Promise<boolean>;
|
||||
|
||||
// Removes processed checks from the evaluate row.
|
||||
pullEvaluated(id: string, checkIds: string[]): Promise<boolean>;
|
||||
|
||||
// ********************
|
||||
// Job CRUD
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js";
|
||||
import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
|
||||
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js";
|
||||
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId, PendingCheck } from "@/domain/jobs/job.type.js";
|
||||
|
||||
// Cap for rearm interval, never want to wait more than 15s.
|
||||
const REARM_JITTER_MAX_MS = 15_000;
|
||||
@@ -18,6 +18,7 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
intervalMs: doc.intervalMs,
|
||||
lockedBy: doc.lockedBy,
|
||||
lockedUntil: doc.lockedUntil,
|
||||
pendingChecks: doc.pendingChecks ?? [],
|
||||
runCount: doc.runCount,
|
||||
failCount: doc.failCount,
|
||||
lastFinishedAt: doc.lastFinishedAt,
|
||||
@@ -146,24 +147,27 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
_id: id,
|
||||
lockedBy: this.workerId, // Only the worker that claimed can record success
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
nextScheduledAt: PARKED,
|
||||
lockedBy: null, // Remove lock
|
||||
lockedUntil: null,
|
||||
lastFinishedAt: now,
|
||||
failCount: 0,
|
||||
lastFailReason: null,
|
||||
[
|
||||
{
|
||||
$set: {
|
||||
// Re-arm if more checks were pushed while this run held the lease; otherwise park
|
||||
nextScheduledAt: {
|
||||
$cond: [{ $gt: [{ $size: { $ifNull: ["$pendingChecks", []] } }, 0] }, now, PARKED],
|
||||
},
|
||||
lockedBy: null, // Remove lock
|
||||
lockedUntil: null,
|
||||
lastFinishedAt: now,
|
||||
failCount: 0,
|
||||
lastFailReason: null,
|
||||
runCount: { $add: ["$runCount", 1] },
|
||||
},
|
||||
},
|
||||
$inc: {
|
||||
runCount: 1, // Increment run count
|
||||
},
|
||||
}
|
||||
]
|
||||
);
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
upsertEvaluate = async (monitorId: string, now: number) => {
|
||||
upsertEvaluate = async (monitorId: string, pending: PendingCheck[], now: number) => {
|
||||
const filter = { type: "evaluate", refId: monitorId };
|
||||
const update = {
|
||||
$setOnInsert: {
|
||||
@@ -181,6 +185,9 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
$min: {
|
||||
nextScheduledAt: now, // no op if there is already a pending evaluation that is older
|
||||
},
|
||||
$push: {
|
||||
pendingChecks: { $each: pending }, // accumulate pending checks
|
||||
},
|
||||
};
|
||||
try {
|
||||
const res = await JobModel.updateOne(filter, update, { upsert: true });
|
||||
@@ -195,6 +202,17 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
pullEvaluated = async (id: string, checkIds: string[]) => {
|
||||
if (checkIds.length === 0) return true;
|
||||
|
||||
const res = await JobModel.updateOne(
|
||||
{ _id: id, lockedBy: this.workerId }, // Only the worker that claimed can pull checks
|
||||
{ $pull: { pendingChecks: { checkId: { $in: checkIds } } } }
|
||||
);
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
upsertJob = async (job: JobSeed) => {
|
||||
const res = await JobModel.updateOne(
|
||||
{ _id: job.id },
|
||||
|
||||
@@ -4,6 +4,11 @@ export const LOCK_MS = 60_000;
|
||||
export const BACKOFF_MS = 5_000;
|
||||
export const PARKED = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
export type PendingCheck = {
|
||||
checkId: string;
|
||||
createdAt: number; // epoch ms, copied from Check.createdAt
|
||||
};
|
||||
|
||||
// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null: cleanup, egress) are just the type.
|
||||
export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`);
|
||||
|
||||
@@ -21,6 +26,9 @@ export type Job = {
|
||||
lockedBy: string | null; // workerId holding the lease
|
||||
lockedUntil: number | null; // epoch ms; lease expiry => reclaimable after a crash
|
||||
|
||||
// Evaluate these rows only. These are checks that are inserted but not yet evaluated
|
||||
pendingChecks: PendingCheck[];
|
||||
|
||||
// observability (replaces what getMetrics/getJobs)
|
||||
runCount: number;
|
||||
failCount: number;
|
||||
|
||||
Reference in new issue
Block a user