rearm jobs on maintenance window pause/delete
This commit is contained in:
1 parent
6aa9360b98
commit
a54fdd89ab
6 files changed
+80
-3
No files matched your search
@@ -338,6 +338,7 @@ export const initializeServices = async ({
|
||||
const maintenanceWindowService = new MaintenanceWindowService({
|
||||
monitorsRepository,
|
||||
maintenanceWindowsRepository,
|
||||
jobsRepository,
|
||||
});
|
||||
const monitorService = new MonitorService({
|
||||
worker,
|
||||
|
||||
@@ -52,6 +52,9 @@ export interface IJobsRepository {
|
||||
// Edit job schedule
|
||||
updateScheduleById(refId: string, type: JobType, intervalMs: number | null): Promise<boolean>;
|
||||
|
||||
// Re-arm the check/geo-check rows. Add jitter to prevent herd
|
||||
markMonitorsDue(monitorIds: string[], now: number): Promise<number>;
|
||||
|
||||
// Delete, drop all rows
|
||||
deleteById(refId: string): Promise<boolean>;
|
||||
|
||||
|
||||
@@ -2,6 +2,9 @@ import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.inte
|
||||
import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
|
||||
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js";
|
||||
|
||||
// Cap for rearm interval, never want to wait more than 15s.
|
||||
const REARM_JITTER_MAX_MS = 15_000;
|
||||
|
||||
class MongoJobsRepository implements IJobsRepository {
|
||||
constructor(private readonly workerId: string) {}
|
||||
|
||||
@@ -249,6 +252,21 @@ class MongoJobsRepository implements IJobsRepository {
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
markMonitorsDue = async (monitorIds: string[], now: number) => {
|
||||
if (monitorIds.length === 0) return 0;
|
||||
const res = await JobModel.updateMany({ refId: { $in: monitorIds }, type: { $in: ["check", "geo-check"] } }, [
|
||||
{
|
||||
// Spread rearmed jobs, but never for more than rearm jitter max
|
||||
$set: {
|
||||
nextScheduledAt: {
|
||||
$add: [now, { $floor: { $multiply: [{ $rand: {} }, { $min: ["$intervalMs", REARM_JITTER_MAX_MS] }] } }],
|
||||
},
|
||||
},
|
||||
},
|
||||
]);
|
||||
return res.modifiedCount;
|
||||
};
|
||||
|
||||
deleteById = async (refId: string) => {
|
||||
const res = await JobModel.deleteMany({ refId }); // Delete all jobs for monitor
|
||||
return res.deletedCount > 0;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
|
||||
import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
|
||||
import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
|
||||
import type { DurationUnit, MaintenanceWindow } from "@/domain/maintenance-windows/maintenance-window.type.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { isWindowActive } from "@/utils/maintenanceWindow.js";
|
||||
@@ -40,16 +41,20 @@ export class MaintenanceWindowService implements IMaintenanceWindowService {
|
||||
static SERVICE_NAME = SERVICE_NAME;
|
||||
private monitorsRepository: IMonitorsRepository;
|
||||
private maintenanceWindowsRepository: IMaintenanceWindowsRepository;
|
||||
private jobsRepository: IJobsRepository;
|
||||
|
||||
constructor({
|
||||
monitorsRepository,
|
||||
maintenanceWindowsRepository,
|
||||
jobsRepository,
|
||||
}: {
|
||||
monitorsRepository: IMonitorsRepository;
|
||||
maintenanceWindowsRepository: IMaintenanceWindowsRepository;
|
||||
jobsRepository: IJobsRepository;
|
||||
}) {
|
||||
this.monitorsRepository = monitorsRepository;
|
||||
this.maintenanceWindowsRepository = maintenanceWindowsRepository;
|
||||
this.jobsRepository = jobsRepository;
|
||||
}
|
||||
|
||||
get serviceName() {
|
||||
@@ -71,6 +76,8 @@ export class MaintenanceWindowService implements IMaintenanceWindowService {
|
||||
const toInitializing = monitorIds.filter((monitorId) => !stillCovered.has(monitorId));
|
||||
if (toInitializing.length > 0) {
|
||||
await this.monitorsRepository.updateByIds(toInitializing, teamId, { status: "initializing" }, ["paused"]);
|
||||
// Rearm jobs to run soon (now + jitter to avoid herding)
|
||||
await this.jobsRepository.markMonitorsDue(toInitializing, now.getTime());
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Reference in new issue
Block a user