bug fixes
This commit is contained in:
1 parent
fa0c0abcac
commit
f299098548
9 files changed
+118
-26
No files matched your search
@@ -366,11 +366,17 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
|
||||
const ids = pending.map((entry) => new mongoose.Types.ObjectId(entry.checkId));
|
||||
|
||||
const times = pending.map((entry) => entry.createdAt);
|
||||
// Bound the query to the pending time range so the monitorId + createdAt index is used
|
||||
let from = Number.POSITIVE_INFINITY;
|
||||
let to = Number.NEGATIVE_INFINITY;
|
||||
for (const entry of pending) {
|
||||
if (entry.createdAt < from) from = entry.createdAt;
|
||||
if (entry.createdAt > to) to = entry.createdAt;
|
||||
}
|
||||
|
||||
const docs = await CheckModel.find({
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
createdAt: { $gte: new Date(Math.min(...times)), $lte: new Date(Math.max(...times)) },
|
||||
createdAt: { $gte: new Date(from), $lte: new Date(to) },
|
||||
_id: { $in: ids },
|
||||
})
|
||||
.sort({ createdAt: 1 })
|
||||
|
||||
@@ -36,7 +36,7 @@ export interface IJobsRepository {
|
||||
// ********************
|
||||
upsertEvaluate(monitorId: string, pending: PendingCheck[], now: number): Promise<boolean>;
|
||||
|
||||
// Removes processed checks from the evaluate row.
|
||||
// Removes processed checks from the evaluate row. Returns false when this worker no longer holds the lease.
|
||||
pullEvaluated(id: string, checkIds: string[]): Promise<boolean>;
|
||||
|
||||
// ********************
|
||||
|
||||
@@ -185,8 +185,8 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
$min: {
|
||||
nextScheduledAt: now, // no op if there is already a pending evaluation that is older
|
||||
},
|
||||
$push: {
|
||||
pendingChecks: { $each: pending }, // accumulate pending checks
|
||||
$addToSet: {
|
||||
pendingChecks: { $each: pending }, // accumulate pending checks; a retried arm never duplicates an entry
|
||||
},
|
||||
};
|
||||
try {
|
||||
@@ -203,6 +203,7 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
}
|
||||
};
|
||||
|
||||
// Returns whether this worker still holds the lease. false means another worker has claimed the row and the caller must stop.
|
||||
pullEvaluated = async (id: string, checkIds: string[]) => {
|
||||
if (checkIds.length === 0) return true;
|
||||
|
||||
@@ -210,7 +211,7 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
{ _id: id, lockedBy: this.workerId }, // Only the worker that claimed can pull checks
|
||||
{ $pull: { pendingChecks: { checkId: { $in: checkIds } } } }
|
||||
);
|
||||
return res.modifiedCount === 1;
|
||||
return res.matchedCount === 1;
|
||||
};
|
||||
|
||||
upsertJob = async (job: JobSeed) => {
|
||||
|
||||
@@ -36,6 +36,9 @@ export class BufferService implements IBufferService {
|
||||
private dockerLogsService: IDockerLogsService;
|
||||
private jobsRepository: IJobsRepository;
|
||||
|
||||
// Checks that are stored but whose evaluate row could not be armed. Retried on the next flush so they are never unevaluated.
|
||||
private unarmed: Map<string, PendingCheck[]> = new Map();
|
||||
|
||||
constructor(
|
||||
logger: ILogger,
|
||||
checkService: ICheckService,
|
||||
@@ -102,20 +105,33 @@ export class BufferService implements IBufferService {
|
||||
}
|
||||
|
||||
ingestChecks = async (checks: Check[]) => {
|
||||
if (checks.length === 0) return;
|
||||
await this.checksService.createChecks(checks);
|
||||
if (checks.length > 0) await this.checksService.createChecks(checks);
|
||||
|
||||
const byMonitor = new Map<string, PendingCheck[]>();
|
||||
// Group by monitor, starting from any entries a previous flush failed to arm
|
||||
const byMonitor = this.unarmed;
|
||||
this.unarmed = new Map();
|
||||
for (const check of checks) {
|
||||
const pending = byMonitor.get(check.metadata.monitorId) ?? [];
|
||||
pending.push({ checkId: check.id, createdAt: new Date(check.createdAt).getTime() });
|
||||
byMonitor.set(check.metadata.monitorId, pending);
|
||||
}
|
||||
if (byMonitor.size === 0) return;
|
||||
|
||||
const now = Date.now();
|
||||
const ops: Promise<boolean>[] = [];
|
||||
for (const [monitorId, pendingChecks] of byMonitor) {
|
||||
ops.push(this.jobsRepository.upsertEvaluate(monitorId, pendingChecks, now));
|
||||
ops.push(
|
||||
this.jobsRepository.upsertEvaluate(monitorId, pendingChecks, now).catch((error: unknown) => {
|
||||
// The checks are already stored, so a failed arm is retried on the next flush rather than dropped
|
||||
this.unarmed.set(monitorId, pendingChecks);
|
||||
this.logger.error({
|
||||
message: `Could not arm evaluation for monitor ${monitorId}, retrying on next flush: ${error instanceof Error ? error.message : String(error)}`,
|
||||
service: this.SERVICE_NAME,
|
||||
method: "ingestChecks",
|
||||
});
|
||||
return false;
|
||||
})
|
||||
);
|
||||
}
|
||||
await Promise.all(ops);
|
||||
};
|
||||
@@ -144,7 +160,7 @@ export class BufferService implements IBufferService {
|
||||
}
|
||||
|
||||
async flushBuffer() {
|
||||
if (this.buffer.length === 0) {
|
||||
if (this.buffer.length === 0 && this.unarmed.size === 0) {
|
||||
return;
|
||||
}
|
||||
// Take the batch first so a write that drains it can't be appended to mid-flush
|
||||
|
||||
@@ -135,14 +135,26 @@ export class DBQueueWorker extends JobScheduler implements IQueueWorker {
|
||||
const status = this.checkService.toStatusResponse(check);
|
||||
const evaluation = await this.checkEvaluator.evaluate(status, check, current);
|
||||
await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
|
||||
await this.jobsRepository.pullEvaluated(job.id, [check.id]);
|
||||
const leaseHeld = await this.jobsRepository.pullEvaluated(job.id, [check.id]);
|
||||
if (!leaseHeld) {
|
||||
// Another worker has claimed this row and is evaluating the remaining ids; stop here so nothing is applied twice
|
||||
this.logger.warn({ message: `Lost lease on ${job.id} after check ${check.id}, stopping`, service: SERVICE_NAME, method: "runEvaluate" });
|
||||
return;
|
||||
}
|
||||
current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
|
||||
}
|
||||
|
||||
// Clean up ids potentially orphaned retention cleanup
|
||||
// Ids whose check was not returned (deleted by retention cleanup) would otherwise sit on the row forever
|
||||
const found = new Set(checks.map((check) => check.id));
|
||||
const missing = pendingCheckIds.filter((checkId) => !found.has(checkId));
|
||||
if (missing.length > 0) await this.jobsRepository.pullEvaluated(job.id, missing);
|
||||
if (missing.length > 0) {
|
||||
this.logger.warn({
|
||||
message: `Dropping ${missing.length} pending checks with no stored check: ${missing.join(", ")}`,
|
||||
service: SERVICE_NAME,
|
||||
method: "runEvaluate",
|
||||
});
|
||||
await this.jobsRepository.pullEvaluated(job.id, missing);
|
||||
}
|
||||
};
|
||||
|
||||
private runGeoCheck = async (job: Job) => {
|
||||
|
||||
Reference in new issue
Block a user