bug fixes

This commit is contained in:
Alex Holliday committed 2026-09-17 12:30:56 -07:00
1 parent fa0c0abcac
commit f299098548
9 files changed
+118 -26

No files matched your search

@@ -366,11 +366,17 @@ class MongoChecksRepository implements IChecksRepository {
const ids = pending.map((entry) => new mongoose.Types.ObjectId(entry.checkId));
const times = pending.map((entry) => entry.createdAt);
// Bound the query to the pending time range so the monitorId + createdAt index is used
let from = Number.POSITIVE_INFINITY;
let to = Number.NEGATIVE_INFINITY;
for (const entry of pending) {
if (entry.createdAt < from) from = entry.createdAt;
if (entry.createdAt > to) to = entry.createdAt;
}
const docs = await CheckModel.find({
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
createdAt: { $gte: new Date(Math.min(...times)), $lte: new Date(Math.max(...times)) },
createdAt: { $gte: new Date(from), $lte: new Date(to) },
_id: { $in: ids },
})
.sort({ createdAt: 1 })
@@ -36,7 +36,7 @@ export interface IJobsRepository {
// ********************
upsertEvaluate(monitorId: string, pending: PendingCheck[], now: number): Promise<boolean>;
// Removes processed checks from the evaluate row.
// Removes processed checks from the evaluate row. Returns false when this worker no longer holds the lease.
pullEvaluated(id: string, checkIds: string[]): Promise<boolean>;
// ********************
@@ -185,8 +185,8 @@ class MongoJobsRepository implements IJobsRepository {
$min: {
nextScheduledAt: now, // no op if there is already a pending evaluation that is older
},
$push: {
pendingChecks: { $each: pending }, // accumulate pending checks
$addToSet: {
pendingChecks: { $each: pending }, // accumulate pending checks; a retried arm never duplicates an entry
},
};
try {
@@ -203,6 +203,7 @@ class MongoJobsRepository implements IJobsRepository {
}
};
// Returns whether this worker still holds the lease. false means another worker has claimed the row and the caller must stop.
pullEvaluated = async (id: string, checkIds: string[]) => {
if (checkIds.length === 0) return true;
@@ -210,7 +211,7 @@ class MongoJobsRepository implements IJobsRepository {
{ _id: id, lockedBy: this.workerId }, // Only the worker that claimed can pull checks
{ $pull: { pendingChecks: { checkId: { $in: checkIds } } } }
);
return res.modifiedCount === 1;
return res.matchedCount === 1;
};
upsertJob = async (job: JobSeed) => {
+21 -5
View File
@@ -36,6 +36,9 @@ export class BufferService implements IBufferService {
private dockerLogsService: IDockerLogsService;
private jobsRepository: IJobsRepository;
// Checks that are stored but whose evaluate row could not be armed. Retried on the next flush so they are never unevaluated.
private unarmed: Map<string, PendingCheck[]> = new Map();
constructor(
logger: ILogger,
checkService: ICheckService,
@@ -102,20 +105,33 @@ export class BufferService implements IBufferService {
}
ingestChecks = async (checks: Check[]) => {
if (checks.length === 0) return;
await this.checksService.createChecks(checks);
if (checks.length > 0) await this.checksService.createChecks(checks);
const byMonitor = new Map<string, PendingCheck[]>();
// Group by monitor, starting from any entries a previous flush failed to arm
const byMonitor = this.unarmed;
this.unarmed = new Map();
for (const check of checks) {
const pending = byMonitor.get(check.metadata.monitorId) ?? [];
pending.push({ checkId: check.id, createdAt: new Date(check.createdAt).getTime() });
byMonitor.set(check.metadata.monitorId, pending);
}
if (byMonitor.size === 0) return;
const now = Date.now();
const ops: Promise<boolean>[] = [];
for (const [monitorId, pendingChecks] of byMonitor) {
ops.push(this.jobsRepository.upsertEvaluate(monitorId, pendingChecks, now));
ops.push(
this.jobsRepository.upsertEvaluate(monitorId, pendingChecks, now).catch((error: unknown) => {
// The checks are already stored, so a failed arm is retried on the next flush rather than dropped
this.unarmed.set(monitorId, pendingChecks);
this.logger.error({
message: `Could not arm evaluation for monitor ${monitorId}, retrying on next flush: ${error instanceof Error ? error.message : String(error)}`,
service: this.SERVICE_NAME,
method: "ingestChecks",
});
return false;
})
);
}
await Promise.all(ops);
};
@@ -144,7 +160,7 @@ export class BufferService implements IBufferService {
}
async flushBuffer() {
if (this.buffer.length === 0) {
if (this.buffer.length === 0 && this.unarmed.size === 0) {
return;
}
// Take the batch first so a write that drains it can't be appended to mid-flush
+15 -3
View File
@@ -135,14 +135,26 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
const status = this.checkService.toStatusResponse(check);
const evaluation = await this.checkEvaluator.evaluate(status, check, current);
await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
await this.jobsRepository.pullEvaluated(job.id, [check.id]);
const leaseHeld = await this.jobsRepository.pullEvaluated(job.id, [check.id]);
if (!leaseHeld) {
// Another worker has claimed this row and is evaluating the remaining ids; stop here so nothing is applied twice
this.logger.warn({ message: `Lost lease on ${job.id} after check ${check.id}, stopping`, service: SERVICE_NAME, method: "runEvaluate" });
return;
}
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
}
// Clean up ids potentially orphaned retention cleanup
// Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever
const found = new Set(checks.map((check) => check.id));
const missing = pendingCheckIds.filter((checkId) => !found.has(checkId));
if (missing.length > 0) await this.jobsRepository.pullEvaluated(job.id, missing);
if (missing.length > 0) {
this.logger.warn({
message: `Dropping ${missing.length} pending checks with no stored check: ${missing.join(", ")}`,
service: SERVICE_NAME,
method: "runEvaluate",
});
await this.jobsRepository.pullEvaluated(job.id, missing);
}
};
private runGeoCheck = async (job: Job) => {