Merge branch 'develop' into feat/customcss-status-page

server/src/api/controllers/queueController.ts

@@ -1,4 +1,4 @@
-import { IWorker } from "@/worker/worker.interface.js";
+import { IJobScheduler } from "@/worker/worker.interface.js";
 import { getQueueJobsQueryValidation } from "@/api/validation/queueValidation.js";
 import { Request, Response, NextFunction } from "express";
 
@@ -13,10 +13,7 @@ export interface IJobQueueController {
 
 class JobQueueController implements IJobQueueController {
 	static SERVICE_NAME = SERVICE_NAME;
-	private worker: IWorker;
-	constructor(worker: IWorker) {
-		this.worker = worker;
-	}
+	constructor(private scheduler: IJobScheduler) {}
 
 	get serviceName() {
 		return JobQueueController.SERVICE_NAME;
@@ -24,7 +21,7 @@ class JobQueueController implements IJobQueueController {
 
 	getMetrics = async (req: Request, res: Response, next: NextFunction) => {
 		try {
-			const metrics = await this.worker.getMetrics();
+			const metrics = await this.scheduler.getMetrics();
 			res.status(200).json({
 				success: true,
 				msg: "Queue metrics fetched successfully",
@@ -38,7 +35,7 @@ class JobQueueController implements IJobQueueController {
 	getJobs = async (req: Request, res: Response, next: NextFunction) => {
 		try {
 			const pagination = getQueueJobsQueryValidation.parse(req.query);
-			const { jobs, count } = await this.worker.getJobs(pagination);
+			const { jobs, count } = await this.scheduler.getJobs(pagination);
 			return res.status(200).json({
 				success: true,
 				msg: "Queue jobs fetched successfully",
@@ -52,8 +49,8 @@ class JobQueueController implements IJobQueueController {
 	getAllMetrics = async (req: Request, res: Response, next: NextFunction) => {
 		try {
 			const pagination = getQueueJobsQueryValidation.parse(req.query);
-			const { jobs, count } = await this.worker.getJobs(pagination);
-			const metrics = await this.worker.getMetrics();
+			const { jobs, count } = await this.scheduler.getJobs(pagination);
+			const metrics = await this.scheduler.getMetrics();
 			return res.status(200).json({
 				success: true,
 				msg: "Queue metrics fetched successfully",
@@ -66,7 +63,7 @@ class JobQueueController implements IJobQueueController {
 
 	flushQueue = async (req: Request, res: Response, next: NextFunction) => {
 		try {
-			const result = await this.worker.flushQueues();
+			const result = await this.scheduler.flushQueues();
 			return res.status(200).json({
 				success: true,
 				msg: "Queue flushed successfully",

server/src/api/middleware/handleErrors.ts

@@ -1,12 +1,12 @@
 import type { NextFunction, Request, Response } from "express";
 import { MulterError } from "multer";
-import { logger } from "@/utils/logger.js";
+import type { ILogger } from "@/utils/logger.js";
 import { AppError } from "@/utils/AppError.js";
 
 // Multer throws MulterError, send 413 Payload Too Large / 400
 const multerErrorStatus = (error: MulterError): number => (error.code === "LIMIT_FILE_SIZE" ? 413 : 400);
 
-const handleErrors = (error: unknown, req: Request, res: Response, _next: NextFunction) => {
+const handleErrors = (logger: ILogger) => (error: unknown, req: Request, res: Response, _next: NextFunction) => {
 	const status = error instanceof MulterError ? multerErrorStatus(error) : error instanceof AppError ? error.status || 500 : 500;
 	const message = error instanceof AppError ? error.message : error instanceof Error ? error.message : "Server error";
 	const service = error instanceof AppError ? error.service : error instanceof MulterError ? "uploadMiddleware" : "unknownService";

server/src/api/validation/monitorValidation.ts

@@ -3,6 +3,7 @@ import { booleanCoercion, dnsHostnameRegex, dnsServerValidation } from "./shared
 import { GeoContinents } from "@/domain/geo-checks/geo-check.type.js";
 import {
 	DnsRecordTypes,
+	HttpMethods,
 	HttpStatusCodeSet,
 	MonitorMatchMethods,
 	MonitorStatuses,
@@ -70,6 +71,16 @@ const refineStrategyType = (body: { type?: string; strategy?: string }, ctx: z.R
 	}
 };
 
+const refineHeadMatching = (body: { method?: string; useAdvancedMatching?: boolean; jsonPath?: string }, ctx: z.RefinementCtx) => {
+	if (body.method === "HEAD" && (body.useAdvancedMatching === true || (body.jsonPath ?? "") !== "")) {
+		ctx.addIssue({
+			code: "custom",
+			path: ["method"],
+			message: "HEAD requests have no response body, so they cannot use advanced matching or a JSON path",
+		});
+	}
+};
+
 export const createMonitorBodyValidation = z
 	.object({
 		_id: z.string().optional(),
@@ -95,6 +106,7 @@ export const createMonitorBodyValidation = z
 		jsonPath: z.union([z.string(), z.literal("")]).optional(),
 		expectedValue: z.union([z.string(), z.literal("")]).optional(),
 		matchMethod: z.union([z.enum(MonitorMatchMethods), z.literal("")]).optional(),
+		method: z.enum(HttpMethods).optional(),
 		gameId: z.union([z.string(), z.literal("")]).optional(),
 		grpcServiceName: z.union([z.string(), z.literal("")]).default(""),
 		strategy: z.enum(PageSpeedStrategies).optional(),
@@ -107,7 +119,8 @@ export const createMonitorBodyValidation = z
 		dnsRecordType: z.enum(DnsRecordTypes).optional(),
 	})
 	.superRefine(refineDnsHostname)
-	.superRefine(refineStrategyType);
+	.superRefine(refineStrategyType)
+	.superRefine(refineHeadMatching);
 
 export const editMonitorBodyValidation = z
 	.object({
@@ -127,6 +140,7 @@ export const editMonitorBodyValidation = z
 		jsonPath: z.union([z.string(), z.literal("")]).optional(),
 		expectedValue: z.union([z.string(), z.literal("")]).optional(),
 		matchMethod: z.union([z.enum(MonitorMatchMethods), z.literal("")]).optional(),
+		method: z.enum(HttpMethods).optional(),
 		port: z.number().min(1).max(65535).optional(),
 		cpuAlertThreshold: z.number().optional(),
 		memoryAlertThreshold: z.number().optional(),
@@ -144,7 +158,8 @@ export const editMonitorBodyValidation = z
 		dnsRecordType: z.enum(DnsRecordTypes).optional(),
 	})
 	.superRefine(refineDnsHostname)
-	.superRefine(refineStrategyType);
+	.superRefine(refineStrategyType)
+	.superRefine(refineHeadMatching);
 
 export const pauseMonitorParamValidation = z.object({
 	monitorId: z.string().min(1, "Monitor ID is required"),
@@ -184,6 +199,7 @@ const importedMonitorSchema = z
 		jsonPath: z.union([z.string(), z.literal("")]).optional(),
 		expectedValue: z.union([z.string(), z.literal("")]).optional(),
 		matchMethod: z.union([z.enum(MonitorMatchMethods), z.literal("")]).optional(),
+		method: z.enum(HttpMethods).optional().default("GET"),
 		url: z.string().min(1, "URL is required"),
 		port: z.number().optional(),
 		isActive: z.boolean().default(true),
@@ -215,7 +231,8 @@ const importedMonitorSchema = z
 		updatedAt: z.string().optional(),
 	})
 	.superRefine(refineDnsHostname)
-	.superRefine(refineStrategyType);
+	.superRefine(refineStrategyType)
+	.superRefine(refineHeadMatching);
 
 export const importMonitorsBodyValidation = z.object({
 	monitors: z.array(importedMonitorSchema).min(1, "At least one monitor is required"),
@@ -251,6 +268,7 @@ export const monitorResponseSchema = z
 		jsonPath: z.string().optional(),
 		expectedValue: z.string().optional(),
 		matchMethod: z.enum(MonitorMatchMethods).optional(),
+		method: z.enum(HttpMethods),
 		notifications: z.array(z.string()),
 		tags: z.array(z.string()),
 		customUpCodes: z.array(httpStatusCode).optional(),
@@ -272,5 +290,6 @@ export const monitorResponseSchema = z
 		userId: z.string(),
 		createdAt: z.string(),
 		updatedAt: z.string(),
+		lastEvaluatedAt: z.number(),
 	})
 	.passthrough();

server/src/app.ts

@@ -9,21 +9,21 @@ import { handleErrors } from "@/api/middleware/handleErrors.js";
 import { generalApiLimiter } from "@/api/middleware/rateLimiter.js";
 import { sanitizeBody, sanitizeQuery } from "@/api/middleware/sanitization.js";
 import { setupRoutes } from "@/config/routes.js";
-import { InitializedServices } from "@/config/services.js";
 import { InitializedControllers } from "@/config/controllers.js";
 import { EnvConfig } from "@/domain/app-settings/app-settings.service.js";
 import { createStatusPageCorsOrigin } from "@/api/middleware/statusPageCorsOrigin.js";
 import { isPublicStatusPageApiPath } from "@/api/middleware/statusPagePublicApiPath.js";
 import { createStatusPageDocumentCsp } from "@/api/middleware/statusPageDocumentCsp.js";
+import { ApiServices } from "@/config/services.api.js";
 
 export const createApp = ({
-	services,
+	apiServices: services,
 	controllers,
 	envSettings,
 	frontendPath,
 	openApiSpec,
 }: {
-	services: InitializedServices;
+	apiServices: ApiServices;
 	controllers: InitializedControllers;
 	envSettings: EnvConfig;
 	frontendPath: string;
@@ -115,6 +115,6 @@ export const createApp = ({
 	app.get("*", (req, res) => {
 		res.sendFile(path.join(frontendPath, "index.html"));
 	});
-	app.use(handleErrors);
+	app.use(handleErrors(services.logger));
 	return app;
 };

server/src/config/controllers.ts

@@ -12,7 +12,7 @@ import NotificationController from "../api/controllers/notificationController.js
 import TagsController from "../api/controllers/tagController.js";
 import DiagnosticController from "../api/controllers/diagnosticController.js";
 import IncidentController from "../api/controllers/incidentController.js";
-import type { InitializedServices } from "@/config/services.js";
+import { ApiServices } from "@/config/services.api.js";
 
 export interface InitializedControllers {
 	authController: AuthController;
@@ -30,21 +30,21 @@ export interface InitializedControllers {
 	diagnosticController: DiagnosticController;
 	incidentController: IncidentController;
 }
-export const initializeControllers = (services: InitializedServices): InitializedControllers => {
+export const initializeControllers = (apiServices: ApiServices): InitializedControllers => {
 	return {
-		authController: new AuthController(services.userService),
-		monitorController: new MonitorController(services.monitorService, services.notificationsService),
-		settingsController: new SettingsController(services.settingsService, services.emailService),
-		checkController: new CheckController(services.checkService),
-		geoCheckController: new GeoCheckController(services.geoChecksService),
-		inviteController: new InviteController(services.inviteService),
-		maintenanceWindowController: new MaintenanceWindowController(services.maintenanceWindowService),
-		queueController: new QueueController(services.worker),
-		logController: new LogController(services.logger),
-		statusPageController: new StatusPageController(services.statusPageService, services.monitorsRepository, services.settingsService),
-		notificationController: new NotificationController(services.notificationsService, services.monitorsRepository),
-		tagController: new TagsController(services.tagsService),
-		diagnosticController: new DiagnosticController(services.diagnosticService),
-		incidentController: new IncidentController(services.incidentService),
+		authController: new AuthController(apiServices.userService),
+		monitorController: new MonitorController(apiServices.monitorService, apiServices.notificationsService),
+		settingsController: new SettingsController(apiServices.settingsService, apiServices.emailService),
+		checkController: new CheckController(apiServices.checkService),
+		geoCheckController: new GeoCheckController(apiServices.geoChecksService),
+		inviteController: new InviteController(apiServices.inviteService),
+		maintenanceWindowController: new MaintenanceWindowController(apiServices.maintenanceWindowService),
+		queueController: new QueueController(apiServices.worker),
+		logController: new LogController(apiServices.logger),
+		statusPageController: new StatusPageController(apiServices.statusPageService, apiServices.monitorsRepository, apiServices.settingsService),
+		notificationController: new NotificationController(apiServices.notificationsService, apiServices.monitorsRepository),
+		tagController: new TagsController(apiServices.tagsService),
+		diagnosticController: new DiagnosticController(apiServices.diagnosticService),
+		incidentController: new IncidentController(apiServices.incidentService),
 	};
 };

server/src/config/envValidation.ts

@@ -1,6 +1,7 @@
 import { z } from "zod";
-import { DbTypes, QueueTypes, QueueModes } from "@/domain/app-settings/app-settings.type.js";
+import { DbTypes, QueueModes } from "@/domain/app-settings/app-settings.type.js";
 import { booleanCoercion } from "@/api/validation/shared.js";
+import { ILogger } from "@/utils/logger.js";
 
 const envSchema = z.object({
 	// Server Configuration
@@ -12,8 +13,8 @@ const envSchema = z.object({
 	DB_CONNECTION_STRING: z.string().min(1, "Database connection string is required"),
 	DB_TYPE: z.enum(DbTypes).default("mongodb"),
 
-	QUEUE_TYPE: z.enum(QueueTypes).default("superSimpleQueue"),
 	QUEUE_MODE: z.enum(QueueModes).default("primary"),
+	QUEUE_PRIMARY_PROCESSES: booleanCoercion.default(true),
 
 	// JWT Authentication
 	JWT_SECRET: z.string().min(1, "JWT_SECRET is required"),
@@ -31,24 +32,39 @@ const envSchema = z.object({
 
 export type ValidatedEnv = z.infer<typeof envSchema>;
 
-export const validateEnv = (): ValidatedEnv => {
+export const validateEnv = (logger: ILogger): ValidatedEnv => {
 	const result = envSchema.safeParse(process.env);
 
 	if (!result.success) {
-		console.error("Environment validation failed:");
-		console.error("");
+		logger.error({
+			message: "Env validation failed",
+			method: "validateEnv",
+			service: "Server",
+		});
 
 		const errors = result.error.format();
 		for (const [key, value] of Object.entries(errors)) {
 			if (key !== "_errors" && value && typeof value === "object" && "_errors" in value) {
-				console.error(`  ${key}: ${value._errors.join(", ")}`);
+				logger.error({
+					message: `${key}: ${value._errors.join(", ")}`,
+					method: "validateEnv",
+					service: "Server",
+				});
 			}
 		}
 
-		console.error("Please check your .env file and ensure all required variables are set.");
+		logger.error({
+			message: "Please check your .env file and ensure all required variables are set.",
+			method: "validateEnv",
+			service: "Server",
+		});
 		process.exit(1);
 	}
 
-	console.log("Environment variables validated successfully");
+	logger.info({
+		message: "Environment variables validated successfully",
+		method: "validateEnv",
+		service: "Server",
+	});
 	return result.data;
 };

server/src/config/routes.ts

@@ -3,7 +3,7 @@ import { createVerifyJWT } from "../api/middleware/verifyJWT.js";
 import { createVerifyStatusPageAccess } from "../api/middleware/verifyStatusPageAccess.js";
 import { authApiLimiter } from "../api/middleware/rateLimiter.js";
 import type { InitializedControllers } from "./controllers.js";
-import type { InitializedServices } from "./services.js";
+import type { ApiServices } from "@/config/services.api.js";
 
 import AuthRoutes from "../api/routes/authRoute.js";
 import InviteRoutes from "../api/routes/inviteRoute.js";
@@ -21,8 +21,8 @@ import TagRoutes from "../api/routes/tagRoutes.js";
 
 import IncidentRoutes from "../api/routes/incidentRoute.js";
 
-export const setupRoutes = (app: Application, controllers: InitializedControllers, services: InitializedServices) => {
-	const verifyJWT = createVerifyJWT(services.settingsService);
+export const setupRoutes = (app: Application, controllers: InitializedControllers, apiServices: ApiServices) => {
+	const verifyJWT = createVerifyJWT(apiServices.settingsService);
 	const authRoutes = new AuthRoutes(controllers.authController, verifyJWT);
 	const monitorRoutes = new MonitorRoutes(controllers.monitorController);
 	const settingsRoutes = new SettingsRoutes(controllers.settingsController);
@@ -32,7 +32,7 @@ export const setupRoutes = (app: Application, controllers: InitializedController
 	const maintenanceWindowRoutes = new MaintenanceWindowRoutes(controllers.maintenanceWindowController);
 	const queueRoutes = new QueueRoutes(controllers.queueController);
 	const logRoutes = new LogRoutes(controllers.logController);
-	const verifyStatusPageAccess = createVerifyStatusPageAccess(services.statusPagesRepository, verifyJWT);
+	const verifyStatusPageAccess = createVerifyStatusPageAccess(apiServices.statusPagesRepository, verifyJWT);
 	const statusPageRoutes = new StatusPageRoutes(controllers.statusPageController, verifyJWT, verifyStatusPageAccess);
 	const notificationRoutes = new NotificationRoutes(controllers.notificationController);
 	const tagRoutes = new TagRoutes(controllers.tagController);

server/src/config/services.api.ts

@@ -0,0 +1,109 @@
+import { DiagnosticService, IDiagnosticService } from "@/domain/diagnostics/diagnostic.service.js";
+import { IInviteService, InviteService } from "@/domain/invites/invite.service.js";
+import { IMaintenanceWindowService, MaintenanceWindowService } from "@/domain/maintenance-windows/maintenance-window.service.js";
+import { IMonitorService, MonitorService } from "@/domain/monitors/monitor.service.js";
+import { IStatusPageService, StatusPageService } from "@/domain/status-pages/status-page.service.js";
+import { ITagsService, TagsService } from "@/domain/tags/tag.service.js";
+import { IUserService, UserService } from "@/domain/users/user.service.js";
+import { IJobScheduler } from "@/worker/worker.interface.js";
+
+// Third-party
+import jwt from "jsonwebtoken";
+import crypto from "crypto";
+import { games } from "gamedig";
+
+import { SharedServices } from "@/config/services.shared.js";
+
+export interface ApiServices extends SharedServices {
+	worker: IJobScheduler; // control-plane handle only (DBQueueWorker in all-in-one, bare JobScheduler in API-only)
+	userService: IUserService;
+	monitorService: IMonitorService;
+	maintenanceWindowService: IMaintenanceWindowService;
+	inviteService: IInviteService;
+	statusPageService: IStatusPageService;
+	tagsService: ITagsService;
+	diagnosticService: IDiagnosticService;
+}
+
+export const buildApi = (shared: SharedServices, jobScheduler: IJobScheduler): ApiServices => {
+	const {
+		logger,
+		db,
+		settingsService,
+		emailService,
+		monitorsRepository,
+		checksRepository,
+		geoChecksRepository,
+		monitorStatsRepository,
+		statusPagesRepository,
+		usersRepository,
+		invitesRepository,
+		recoveryTokensRepository,
+		settingsRepository,
+		tagsRepository,
+		incidentsRepository,
+		teamsRepository,
+		maintenanceWindowsRepository,
+		jobsRepository,
+	} = shared;
+
+	const userService = new UserService({
+		crypto,
+		emailService,
+		settingsService,
+		logger,
+		jwt,
+		scheduler: jobScheduler,
+		monitorsRepository,
+		usersRepository,
+		invitesRepository,
+		recoveryTokensRepository,
+		settingsRepository,
+		teamsRepository,
+	});
+
+	// ***********************
+	//  Business services
+	// ***********************
+
+	const monitorService = new MonitorService({
+		scheduler: jobScheduler,
+		logger,
+		games,
+		monitorsRepository,
+		checksRepository,
+		geoChecksRepository,
+		monitorStatsRepository,
+		statusPagesRepository,
+		incidentsRepository,
+	});
+
+	const maintenanceWindowService = new MaintenanceWindowService({
+		monitorsRepository,
+		maintenanceWindowsRepository,
+		jobsRepository,
+		scheduler: jobScheduler,
+	});
+
+	const inviteService = new InviteService({
+		invitesRepository,
+		settingsService,
+		emailService,
+	});
+
+	const statusPageService = new StatusPageService(statusPagesRepository, settingsService);
+	const tagsService = new TagsService(tagsRepository, monitorsRepository);
+	const diagnosticService = new DiagnosticService(db);
+
+	return {
+		...shared,
+		worker: jobScheduler,
+		userService,
+		monitorService,
+		maintenanceWindowService,
+		inviteService,
+		statusPageService,
+		tagsService,
+		diagnosticService,
+	};
+};

server/src/config/services.shared.ts

@@ -0,0 +1,218 @@
+import { Mongoose } from "mongoose";
+import { hostname } from "node:os";
+import { randomUUID } from "node:crypto";
+import fs from "fs";
+import path from "path";
+import nodemailer from "nodemailer";
+import pkg from "handlebars";
+const { compile } = pkg;
+import mjml2html from "mjml";
+
+import MongoDB from "@/db/db.mongo.js";
+import { IDb } from "@/db/db.interface.js";
+import { ISettingsService, EnvConfig, SettingsService } from "@/domain/app-settings/app-settings.service.js";
+import { ICheckService, CheckService } from "@/domain/checks/check.service.js";
+import { IGeoChecksService, GeoChecksService } from "@/domain/geo-checks/geo-check.service.js";
+import { IIncidentService, IncidentService } from "@/domain/incidents/incident.service.js";
+import { INotificationMessageBuilder, NotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js";
+import { INotificationsService, NotificationsService } from "@/domain/notifications/notification.service.js";
+import { IEmailService, EmailService } from "@/service/emailService.js";
+import { GlobalPingService } from "@/service/globalPingService.js";
+import { ILogger } from "@/utils/logger.js";
+
+// Notification providers
+import { WebhookProvider } from "@/domain/notifications/providers/webhook.js";
+import { SlackProvider } from "@/domain/notifications/providers/slack.js";
+import { EmailProvider } from "@/domain/notifications/providers/email.js";
+import { DiscordProvider } from "@/domain/notifications/providers/discord.js";
+import { PagerDutyProvider } from "@/domain/notifications/providers/pagerduty.js";
+import { MatrixProvider } from "@/domain/notifications/providers/matrix.js";
+import { TeamsProvider } from "@/domain/notifications/providers/teams.js";
+import { TelegramProvider } from "@/domain/notifications/providers/telegram.js";
+import { PushoverProvider } from "@/domain/notifications/providers/pushover.js";
+import { TwilioProvider } from "@/domain/notifications/providers/twilio.js";
+import { NtfyProvider } from "@/domain/notifications/providers/ntfy.js";
+
+// Repository interfaces
+import { ISettingsRepository } from "@/domain/app-settings/app-settings-repository.interface.js";
+import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
+import { IGeoChecksRepository } from "@/domain/geo-checks/geo-check.repository.interface.js";
+import { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js";
+import { IInvitesRepository } from "@/domain/invites/invite.repository.interface.js";
+import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
+import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
+import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js";
+import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
+import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js";
+import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
+import { IRecoveryTokensRepository } from "@/domain/recovery-tokens/recovery-token.repository.interface.js";
+import { IStatusPagesRepository } from "@/domain/status-pages/status-page-repository.interface.js";
+import { ITagsRepository } from "@/domain/tags/tag.repository.interface.js";
+import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
+import { IUsersRepository } from "@/domain/users/user.repository.interface.js";
+
+// Mongo repository implementations
+import MongoSettingsRepository from "@/domain/app-settings/app-settings.repository.mongo.js";
+import MongoChecksRepository from "@/domain/checks/check.repository.mongo.js";
+import MongoGeoChecksRepository from "@/domain/geo-checks/geo-check.repository.mongo.js";
+import MongoIncidentsRepository from "@/domain/incidents/incident.repository.mongo.js";
+import MongoInvitesRepository from "@/domain/invites/invite.repository.mongo.js";
+import MongoJobsRepository from "@/domain/jobs/job.repository.mongo.js";
+import MongoMaintenanceWindowsRepository from "@/domain/maintenance-windows/maintenance-window.repository.mongo.js";
+import MongoMonitorStatsRepository from "@/domain/monitor-stats/monitor-stats.repository.mongo.js";
+import MongoMonitorsRepository from "@/domain/monitors/monitor.repository.mongo.js";
+import MongoNotificationsRepository from "@/domain/notifications/notification.repository.mongo.js";
+import MongoQueueWorkersRepository from "@/domain/queue-workers/queue-worker.repository.mongo.js";
+import MongoRecoveryTokensRepository from "@/domain/recovery-tokens/recovery-token.repository.mongo.js";
+import MongoStatusPagesRepository from "@/domain/status-pages/status-page-repository.mongo.js";
+import MongoTagsRepository from "@/domain/tags/tag.repository.mongo.js";
+import MongoTeamsRepository from "@/domain/teams/team.repository.model.js";
+import MongoUsersRepository from "@/domain/users/user.repository.mongo.js";
+
+// Shared infrastructure + business services that both the API and the worker process construct.
+export interface SharedServices {
+	logger: ILogger;
+	db: IDb<Mongoose>;
+	settingsService: ISettingsService;
+	emailService: IEmailService;
+	notificationMessageBuilder: INotificationMessageBuilder;
+	incidentService: IIncidentService;
+	checkService: ICheckService;
+	geoChecksService: IGeoChecksService;
+	notificationsService: INotificationsService;
+
+	// Queue identity (one per process; the worker reuses these)
+	workerId: string;
+	jobsRepository: IJobsRepository;
+	queueWorkersRepository: IQueueWorkersRepository;
+
+	// Repositories
+	monitorsRepository: IMonitorsRepository;
+	checksRepository: IChecksRepository;
+	geoChecksRepository: IGeoChecksRepository;
+	monitorStatsRepository: IMonitorStatsRepository;
+	statusPagesRepository: IStatusPagesRepository;
+	usersRepository: IUsersRepository;
+	invitesRepository: IInvitesRepository;
+	recoveryTokensRepository: IRecoveryTokensRepository;
+	settingsRepository: ISettingsRepository;
+	notificationsRepository: INotificationsRepository;
+	tagsRepository: ITagsRepository;
+	incidentsRepository: IIncidentsRepository;
+	teamsRepository: ITeamsRepository;
+	maintenanceWindowsRepository: IMaintenanceWindowsRepository;
+}
+
+export const buildShared = async ({
+	logger,
+	envSettings,
+	settingsService,
+}: {
+	logger: ILogger;
+	envSettings: EnvConfig;
+	settingsService: ISettingsService;
+}): Promise<SharedServices> => {
+	// Shared worker ID
+	const workerId = `${hostname()}:${process.pid}:${randomUUID()}`;
+
+	// Create DB
+	let db: IDb<Mongoose> | null = null;
+	db = new MongoDB(logger, envSettings);
+	await db.connect();
+
+	// Repositories
+	const monitorsRepository = new MongoMonitorsRepository();
+	const checksRepository = new MongoChecksRepository(logger);
+	const incidentsRepository = new MongoIncidentsRepository();
+	const usersRepository = new MongoUsersRepository();
+	const geoChecksRepository = new MongoGeoChecksRepository(logger);
+	const notificationsRepository = new MongoNotificationsRepository();
+	const jobsRepository = new MongoJobsRepository(workerId);
+	const queueWorkersRepository = new MongoQueueWorkersRepository();
+	const monitorStatsRepository = new MongoMonitorStatsRepository();
+	const statusPagesRepository = new MongoStatusPagesRepository();
+	const invitesRepository = new MongoInvitesRepository();
+	const recoveryTokensRepository = new MongoRecoveryTokensRepository();
+	const settingsRepository = new MongoSettingsRepository();
+	const tagsRepository = new MongoTagsRepository();
+	const teamsRepository = new MongoTeamsRepository();
+	const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository();
+
+	// Inject settings repository into settings service (now that DB is connected)
+	(settingsService as SettingsService).setRepository(settingsRepository);
+
+	// Services
+	const notificationMessageBuilder = new NotificationMessageBuilder();
+	const emailService = new EmailService(settingsService, fs, path, compile, mjml2html, nodemailer, logger);
+	const incidentService = new IncidentService(logger, incidentsRepository, monitorsRepository, usersRepository, notificationMessageBuilder);
+	const checkService = new CheckService(monitorsRepository, logger, checksRepository);
+
+	const globalPingService = new GlobalPingService(logger);
+	const geoChecksService = new GeoChecksService({
+		logger,
+		geoChecksRepository,
+		globalPingService,
+		monitorsRepository,
+	});
+
+	const webhookProvider = new WebhookProvider(logger);
+	const slackProvider = new SlackProvider(logger);
+	const emailProvider = new EmailProvider(emailService, logger);
+	const discordProvider = new DiscordProvider(logger);
+	const pagerDutyProvider = new PagerDutyProvider(logger);
+	const matrixProvider = new MatrixProvider(logger);
+	const teamsProvider = new TeamsProvider(logger);
+	const telegramProvider = new TelegramProvider(logger);
+	const pushoverProvider = new PushoverProvider(logger);
+	const twilioProvider = new TwilioProvider(logger);
+	const ntfyProvider = new NtfyProvider(logger);
+
+	const notificationsService = new NotificationsService(
+		notificationsRepository,
+		monitorsRepository,
+		webhookProvider,
+		emailProvider,
+		slackProvider,
+		discordProvider,
+		pagerDutyProvider,
+		matrixProvider,
+		teamsProvider,
+		telegramProvider,
+		pushoverProvider,
+		twilioProvider,
+		ntfyProvider,
+		settingsService,
+		logger,
+		notificationMessageBuilder
+	);
+
+	const sharedServices: SharedServices = {
+		logger,
+		db,
+		settingsService,
+		emailService,
+		notificationMessageBuilder,
+		incidentService,
+		checkService,
+		geoChecksService,
+		notificationsService,
+		workerId,
+		jobsRepository,
+		monitorsRepository,
+		queueWorkersRepository,
+		checksRepository,
+		geoChecksRepository,
+		monitorStatsRepository,
+		statusPagesRepository,
+		usersRepository,
+		invitesRepository,
+		recoveryTokensRepository,
+		settingsRepository,
+		notificationsRepository,
+		tagsRepository,
+		incidentsRepository,
+		teamsRepository,
+		maintenanceWindowsRepository,
+	};
+	return sharedServices;
+};

server/src/config/services.ts

@@ -1,450 +0,0 @@
-import { Mongoose } from "mongoose";
-import MongoDB from "../db/db.mongo.js";
-import { IDb } from "@/db/db.interface.js";
-import { EnvConfig, ISettingsService, SettingsService } from "@/domain/app-settings/app-settings.service.js";
-import { CheckService, ICheckService } from "@/domain/checks/check.service.js";
-import { DiagnosticService, IDiagnosticService } from "@/domain/diagnostics/diagnostic.service.js";
-import { GeoChecksService, IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js";
-import { IIncidentService, IncidentService } from "@/domain/incidents/incident.service.js";
-import { IInviteService, InviteService } from "@/domain/invites/invite.service.js";
-import { IMaintenanceWindowService, MaintenanceWindowService } from "@/domain/maintenance-windows/maintenance-window.service.js";
-import { IMonitorService, MonitorService } from "@/domain/monitors/monitor.service.js";
-import { INotificationMessageBuilder, NotificationMessageBuilder } from "@/domain/notifications/notification.message-builder.js";
-import { INotificationsService, NotificationsService } from "@/domain/notifications/notification.service.js";
-import { DiscordProvider } from "@/domain/notifications/providers/discord.js";
-import { EmailProvider } from "@/domain/notifications/providers/email.js";
-import { MatrixProvider } from "@/domain/notifications/providers/matrix.js";
-import { NtfyProvider } from "@/domain/notifications/providers/ntfy.js";
-import { PagerDutyProvider } from "@/domain/notifications/providers/pagerduty.js";
-import { PushoverProvider } from "@/domain/notifications/providers/pushover.js";
-import { SlackProvider } from "@/domain/notifications/providers/slack.js";
-import { TeamsProvider } from "@/domain/notifications/providers/teams.js";
-import { TelegramProvider } from "@/domain/notifications/providers/telegram.js";
-import { TwilioProvider } from "@/domain/notifications/providers/twilio.js";
-import { WebhookProvider } from "@/domain/notifications/providers/webhook.js";
-import { IStatusPageService, StatusPageService } from "@/domain/status-pages/status-page.service.js";
-import { ITagsService, TagsService } from "@/domain/tags/tag.service.js";
-import { IUserService, UserService } from "@/domain/users/user.service.js";
-import { BufferService, IBufferService } from "@/service/bufferService.js";
-import { EmailService, IEmailService } from "@/service/emailService.js";
-import { GlobalPingService } from "@/service/globalPingService.js";
-import { WorkerHelper } from "@/worker/worker.helper.js";
-import { IWorker } from "@/worker/worker.interface.js";
-import { LessSimpleWorker } from "@/worker/worker.less-simple.js";
-import { SuperSimpleQueue } from "@/worker/worker.super-simple.js";
-import { INetworkService, NetworkService } from "@/service/networkService.js";
-import { IStatusService, StatusService } from "@/service/statusService.js";
-import { MonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
-
-// Network providers
-import { PingProvider } from "@/service/network/PingProvider.js";
-import { HttpProvider } from "@/service/network/HttpProvider.js";
-import { AdvancedMatcher } from "@/service/network/AdvancedMatcher.js";
-import { PageSpeedProvider } from "@/service/network/PageSpeedProvider.js";
-import { HardwareProvider } from "@/service/network/HardwareProvider.js";
-import { DockerProvider } from "@/service/network/DockerProvider.js";
-import { PortProvider } from "@/service/network/PortProvider.js";
-import { GameProvider } from "@/service/network/GameProvider.js";
-import { GrpcProvider } from "@/service/network/GrpcProvider.js";
-import { WebSocketProvider } from "@/service/network/WebSocketProvider.js";
-import { DNSProvider } from "@/service/network/DNSProvider.js";
-
-// Third-party
-import { Resolver } from "dns/promises";
-import axios from "axios";
-import got from "got";
-import ping from "ping";
-import Docker from "dockerode";
-import net from "net";
-import fs from "fs";
-import path from "path";
-import nodemailer from "nodemailer";
-import pkg from "handlebars";
-const { compile } = pkg;
-import mjml2html from "mjml";
-import jwt from "jsonwebtoken";
-import crypto from "crypto";
-import { games, GameDig } from "gamedig";
-import jmespath from "jmespath";
-import * as grpc from "@grpc/grpc-js";
-import * as protoLoader from "@grpc/proto-loader";
-import WebSocket from "ws";
-
-// Repositories
-import { ISettingsRepository } from "@/domain/app-settings/app-settings-repository.interface.js";
-import MongoSettingsRepository from "@/domain/app-settings/app-settings.repository.mongo.js";
-import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
-import MongoChecksRepository from "@/domain/checks/check.repository.mongo.js";
-import { IGeoChecksRepository } from "@/domain/geo-checks/geo-check.repository.interface.js";
-import MongoGeoChecksRepository from "@/domain/geo-checks/geo-check.repository.mongo.js";
-import { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js";
-import MongoIncidentsRepository from "@/domain/incidents/incident.repository.mongo.js";
-import { IInvitesRepository } from "@/domain/invites/invite.repository.interface.js";
-import MongoInvitesRepository from "@/domain/invites/invite.repository.mongo.js";
-import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
-import MongoMaintenanceWindowsRepository from "@/domain/maintenance-windows/maintenance-window.repository.mongo.js";
-import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js";
-import MongoMonitorStatsRepository from "@/domain/monitor-stats/monitor-stats.repository.mongo.js";
-import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
-import MongoMonitorsRepository from "@/domain/monitors/monitor.repository.mongo.js";
-import { INotificationsRepository } from "@/domain/notifications/notification.repository.interface.js";
-import MongoNotificationsRepository from "@/domain/notifications/notification.repository.mongo.js";
-import MongoQueueWorkersRepository from "@/domain/queue-workers/queue-worker.repository.mongo.js";
-import { IRecoveryTokensRepository } from "@/domain/recovery-tokens/recovery-token.repository.interface.js";
-import MongoRecoveryTokensRepository from "@/domain/recovery-tokens/recovery-token.repository.mongo.js";
-import { IStatusPagesRepository } from "@/domain/status-pages/status-page-repository.interface.js";
-import MongoStatusPagesRepository from "@/domain/status-pages/status-page-repository.mongo.js";
-import { ITagsRepository } from "@/domain/tags/tag.repository.interface.js";
-import MongoTagsRepository from "@/domain/tags/tag.repository.mongo.js";
-import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
-import MongoTeamsRepository from "@/domain/teams/team.repository.model.js";
-import { IUsersRepository } from "@/domain/users/user.repository.interface.js";
-import MongoUsersRepository from "@/domain/users/user.repository.mongo.js";
-import { ILogger } from "@/utils/logger.js";
-import { AppError } from "@/utils/AppError.js";
-import { type QueueMode } from "@/domain/app-settings/app-settings.type.js";
-import { NotificationReactor } from "@/worker/reactors/reactor.notification.js";
-import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
-import { IncidentReactor } from "@/worker/reactors/reactor.incident.js";
-import { CheckPipeline, GeoChecksPipeline } from "@/worker/worker.check-pipeline.js";
-
-export type InitializedServices = {
-	settingsService: ISettingsService;
-	db: IDb<Mongoose>;
-	networkService: INetworkService;
-	emailService: IEmailService;
-	bufferService: IBufferService;
-	statusService: IStatusService;
-	worker: IWorker;
-	userService: IUserService;
-	checkService: ICheckService;
-	geoChecksService: IGeoChecksService;
-	diagnosticService: IDiagnosticService;
-	inviteService: IInviteService;
-	maintenanceWindowService: IMaintenanceWindowService;
-	monitorService: IMonitorService;
-	incidentService: IIncidentService;
-	logger: ILogger;
-	notificationsService: INotificationsService;
-	tagsService: ITagsService;
-	statusPageService: IStatusPageService;
-	notificationMessageBuilder: INotificationMessageBuilder;
-
-	// Repositories
-	monitorsRepository: IMonitorsRepository;
-	checksRepository: IChecksRepository;
-	geoChecksRepository: IGeoChecksRepository;
-	monitorStatsRepository: IMonitorStatsRepository;
-	statusPagesRepository: IStatusPagesRepository;
-	usersRepository: IUsersRepository;
-	invitesRepository: IInvitesRepository;
-	recoveryTokensRepository: IRecoveryTokensRepository;
-	settingsRepository: ISettingsRepository;
-	notificationsRepository: INotificationsRepository;
-	tagsRepository: ITagsRepository;
-	incidentsRepository: IIncidentsRepository;
-	teamsRepository: ITeamsRepository;
-	maintenanceWindowsRepository: IMaintenanceWindowsRepository;
-};
-
-export const initializeServices = async ({
-	logger,
-	envSettings,
-	settingsService,
-	queueMode = "primary",
-}: {
-	logger: ILogger;
-	envSettings: EnvConfig;
-	settingsService: ISettingsService;
-	queueMode?: QueueMode;
-}): Promise<InitializedServices> => {
-	// Create DB
-
-	let db: IDb<Mongoose> | null = null;
-	db = new MongoDB(logger, envSettings);
-	await db.connect();
-
-	// ** NOTE **
-	// const dbType = envSettings.dbType;
-	// DB type has been fixed to MongoDB for now
-
-	// if (dbType === "mongodb") {
-	// 	db = new MongoDB(logger, envSettings);
-	// }
-
-	// if (!db) {
-	// 	throw new AppError({ message: "Unsupported database type", status: 500 });
-	// }
-
-	// await db.connect();
-
-	// Repositories
-
-	// ** NOTE **
-	// DB Type fixed to MongoDB for now
-	// let monitorsRepository: IMonitorsRepository;
-	// let checksRepository: IChecksRepository;
-	// let geoChecksRepository: IGeoChecksRepository;
-	// let monitorStatsRepository: IMonitorStatsRepository;
-	// let statusPagesRepository: IStatusPagesRepository;
-	// let usersRepository: IUsersRepository;
-	// let invitesRepository: IInvitesRepository;
-	// let recoveryTokensRepository: IRecoveryTokensRepository;
-	// let settingsRepository: ISettingsRepository;
-	// let notificationsRepository: INotificationsRepository;
-	// let tagsRepository: ITagsRepository;
-	// let incidentsRepository: IIncidentsRepository;
-	// let teamsRepository: ITeamsRepository;
-	// let maintenanceWindowsRepository: IMaintenanceWindowsRepository;
-
-	// if (dbType === "mongodb") {
-	// 	monitorsRepository = new MongoMonitorsRepository();
-	// 	checksRepository = new MongoChecksRepository(logger);
-	// 	geoChecksRepository = new MongoGeoChecksRepository(logger);
-	// 	monitorStatsRepository = new MongoMonitorStatsRepository();
-	// 	statusPagesRepository = new MongoStatusPagesRepository();
-	// 	usersRepository = new MongoUsersRepository();
-	// 	invitesRepository = new MongoInvitesRepository();
-	// 	recoveryTokensRepository = new MongoRecoveryTokensRepository();
-	// 	settingsRepository = new MongoSettingsRepository();
-	// 	notificationsRepository = new MongoNotificationsRepository();
-	// 	tagsRepository = new MongoTagsRepository();
-	// 	incidentsRepository = new MongoIncidentsRepository();
-	// 	teamsRepository = new MongoTeamsRepository();
-	// 	maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository();
-	// } else {
-	// 	throw new AppError({ message: "Unsupported database type", status: 500 });
-	// }
-
-	const monitorsRepository = new MongoMonitorsRepository();
-	const checksRepository = new MongoChecksRepository(logger);
-	const geoChecksRepository = new MongoGeoChecksRepository(logger);
-	const monitorStatsRepository = new MongoMonitorStatsRepository();
-	const statusPagesRepository = new MongoStatusPagesRepository();
-	const usersRepository = new MongoUsersRepository();
-	const invitesRepository = new MongoInvitesRepository();
-	const recoveryTokensRepository = new MongoRecoveryTokensRepository();
-	const settingsRepository = new MongoSettingsRepository();
-	const notificationsRepository = new MongoNotificationsRepository();
-	const tagsRepository = new MongoTagsRepository();
-	const incidentsRepository = new MongoIncidentsRepository();
-	const teamsRepository = new MongoTeamsRepository();
-	const maintenanceWindowsRepository = new MongoMaintenanceWindowsRepository();
-	const queueWorkersRepository = new MongoQueueWorkersRepository();
-
-	// Inject settings repository into settings service (now that DB is connected)
-	(settingsService as SettingsService).setRepository(settingsRepository);
-
-	// Network providers
-	const pingProvider = new PingProvider(ping);
-	const httpProvider = new HttpProvider(got, new AdvancedMatcher(jmespath));
-	const pageSpeedProvider = new PageSpeedProvider(httpProvider, settingsService, logger);
-	const hardwareProvider = new HardwareProvider(httpProvider);
-	const dockerProvider = new DockerProvider(logger, Docker);
-	const portProvider = new PortProvider(net);
-	const gameProvider = new GameProvider(logger, GameDig);
-	const grpcProvider = new GrpcProvider(grpc, protoLoader);
-	const webSocketProvider = new WebSocketProvider(WebSocket);
-	const dnsProvider = new DNSProvider(() => new Resolver());
-
-	const networkService = new NetworkService(axios, logger, [
-		pingProvider,
-		httpProvider,
-		pageSpeedProvider,
-		hardwareProvider,
-		dockerProvider,
-		portProvider,
-		gameProvider,
-		grpcProvider,
-		webSocketProvider,
-		dnsProvider,
-	]);
-	const emailService = new EmailService(settingsService, fs, path, compile, mjml2html, nodemailer, logger);
-
-	const notificationMessageBuilder = new NotificationMessageBuilder();
-
-	const incidentService = new IncidentService(logger, incidentsRepository, monitorsRepository, usersRepository, notificationMessageBuilder);
-
-	const checkService = new CheckService(monitorsRepository, logger, checksRepository);
-
-	const globalPingService = new GlobalPingService(logger);
-
-	const geoChecksService = new GeoChecksService({
-		logger,
-		geoChecksRepository,
-		globalPingService,
-		monitorsRepository,
-	});
-
-	const bufferService = new BufferService(logger, checkService, geoChecksService, settingsService);
-
-	const statusService = new StatusService(logger, bufferService, monitorsRepository, monitorStatsRepository, checksRepository);
-
-	// Notification providers
-	const webhookProvider = new WebhookProvider(logger);
-	const slackProvider = new SlackProvider(logger);
-	const emailProvider = new EmailProvider(emailService, logger);
-	const discordProvider = new DiscordProvider(logger);
-	const pagerDutyProvider = new PagerDutyProvider(logger);
-	const matrixProvider = new MatrixProvider(logger);
-	const teamsProvider = new TeamsProvider(logger);
-	const telegramProvider = new TelegramProvider(logger);
-	const pushoverProvider = new PushoverProvider(logger);
-	const twilioProvider = new TwilioProvider(logger);
-	const ntfyProvider = new NtfyProvider(logger);
-
-	const notificationsService = new NotificationsService(
-		notificationsRepository,
-		monitorsRepository,
-		webhookProvider,
-		emailProvider,
-		slackProvider,
-		discordProvider,
-		pagerDutyProvider,
-		matrixProvider,
-		teamsProvider,
-		telegramProvider,
-		pushoverProvider,
-		twilioProvider,
-		ntfyProvider,
-		settingsService,
-		logger,
-		notificationMessageBuilder
-	);
-
-	const tagsService = new TagsService(tagsRepository, monitorsRepository);
-
-	const monitorStatusPolicy = new MonitorStatusPolicy();
-	// reactors and dispatcher
-	const notificationReactor = new NotificationReactor(notificationsService);
-	const incidentReactor = new IncidentReactor(incidentService);
-	const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]);
-
-	// pipelines
-	const checkPipeline = new CheckPipeline(
-		monitorsRepository,
-		maintenanceWindowsRepository,
-		checkService,
-		networkService,
-		bufferService,
-		monitorStatusPolicy,
-		statusService,
-		logger
-	);
-
-	const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger);
-
-	const workerHelper = new WorkerHelper(
-		logger,
-		checkService,
-		settingsService,
-		monitorsRepository,
-		teamsRepository,
-		monitorStatsRepository,
-		checksRepository,
-		incidentsRepository,
-		geoChecksRepository,
-		reactorDispatcher,
-		checkPipeline,
-		geoCheckPipeline
-	);
-
-	if (queueMode === "worker" && envSettings.queueType !== "lessSimpleQueue") {
-		throw new AppError({
-			message: `Worker mode requires QUEUE_TYPE="lessSimpleQueue" got ${envSettings.queueType}`,
-			status: 500,
-			service: "config services",
-			method: "initializeServices",
-		});
-	}
-
-	let worker: IWorker;
-	switch (envSettings.queueType) {
-		case "lessSimpleQueue":
-			worker = await LessSimpleWorker.create(logger, workerHelper, monitorsRepository, queueWorkersRepository, envSettings, queueMode);
-			break;
-		default:
-			worker = await SuperSimpleQueue.create(logger, workerHelper, monitorsRepository);
-	}
-
-	// Business services
-	const userService = new UserService({
-		crypto,
-		emailService,
-		settingsService,
-		logger,
-		jwt,
-		worker,
-		monitorsRepository,
-		usersRepository,
-		invitesRepository,
-		recoveryTokensRepository,
-		settingsRepository,
-		teamsRepository,
-	});
-
-	const diagnosticService = new DiagnosticService(db);
-	const inviteService = new InviteService({
-		invitesRepository,
-		settingsService,
-		emailService,
-	});
-	const maintenanceWindowService = new MaintenanceWindowService({
-		monitorsRepository,
-		maintenanceWindowsRepository,
-	});
-	const monitorService = new MonitorService({
-		worker,
-		logger,
-		games,
-		monitorsRepository,
-		checksRepository,
-		geoChecksRepository,
-		monitorStatsRepository,
-		statusPagesRepository,
-		incidentsRepository,
-	});
-
-	const statusPageService = new StatusPageService(statusPagesRepository, settingsService);
-
-	const services = {
-		settingsService,
-		db,
-		networkService,
-		emailService,
-		bufferService,
-		statusService,
-		worker,
-		userService,
-		checkService,
-		geoChecksService,
-		diagnosticService,
-		inviteService,
-		maintenanceWindowService,
-		monitorService,
-		incidentService,
-		logger,
-		notificationsService,
-		tagsService,
-		statusPageService,
-		notificationMessageBuilder,
-
-		// Repositories
-		monitorsRepository,
-		checksRepository,
-		geoChecksRepository,
-		monitorStatsRepository,
-		statusPagesRepository,
-		usersRepository,
-		invitesRepository,
-		recoveryTokensRepository,
-		settingsRepository,
-		notificationsRepository,
-		tagsRepository,
-		incidentsRepository,
-		teamsRepository,
-		maintenanceWindowsRepository,
-	};
-
-	return services;
-};

server/src/config/services.worker.ts

@@ -0,0 +1,151 @@
+import { Resolver } from "dns/promises";
+import axios from "axios";
+import got from "got";
+import ping from "ping";
+import Docker from "dockerode";
+import net from "net";
+import { GameDig } from "gamedig";
+import jmespath from "jmespath";
+import * as grpc from "@grpc/grpc-js";
+import * as protoLoader from "@grpc/proto-loader";
+import WebSocket from "ws";
+
+import { EnvConfig } from "@/domain/app-settings/app-settings.service.js";
+import { SharedServices } from "@/config/services.shared.js";
+import { INetworkService, NetworkService } from "@/service/networkService.js";
+import { IBufferService, BufferService } from "@/service/bufferService.js";
+import { IStatusService, StatusService } from "@/service/statusService.js";
+import { IQueueWorker } from "@/worker/worker.interface.js";
+import { MonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
+import { WorkerHelper } from "@/worker/worker.helper.js";
+import { CheckProducer } from "@/worker/worker.check-producer.js";
+import { CheckEvaluator } from "@/worker/worker.check-evaluator.js";
+import { GeoChecksPipeline } from "@/worker/worker.check-pipeline.js";
+import { NotificationReactor } from "@/worker/reactors/reactor.notification.js";
+import { IncidentReactor } from "@/worker/reactors/reactor.incident.js";
+import { ReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
+import { DBQueueWorker } from "@/worker/worker.db-queue.js";
+
+// Network providers
+import { PingProvider } from "@/service/network/PingProvider.js";
+import { HttpProvider } from "@/service/network/HttpProvider.js";
+import { AdvancedMatcher } from "@/service/network/AdvancedMatcher.js";
+import { PageSpeedProvider } from "@/service/network/PageSpeedProvider.js";
+import { HardwareProvider } from "@/service/network/HardwareProvider.js";
+import { DockerProvider } from "@/service/network/DockerProvider.js";
+import { PortProvider } from "@/service/network/PortProvider.js";
+import { GameProvider } from "@/service/network/GameProvider.js";
+import { GrpcProvider } from "@/service/network/GrpcProvider.js";
+import { WebSocketProvider } from "@/service/network/WebSocketProvider.js";
+import { DNSProvider } from "@/service/network/DNSProvider.js";
+export interface WorkerServices {
+	worker: IQueueWorker;
+	networkService: INetworkService;
+	bufferService: IBufferService;
+	statusService: IStatusService;
+}
+
+export const buildWorker = async (shared: SharedServices, envSettings: EnvConfig): Promise<WorkerServices> => {
+	const {
+		logger,
+		settingsService,
+		checkService,
+		geoChecksService,
+		notificationsService,
+		incidentService,
+		workerId,
+		jobsRepository,
+		queueWorkersRepository,
+		monitorsRepository,
+		checksRepository,
+		geoChecksRepository,
+		monitorStatsRepository,
+		incidentsRepository,
+		teamsRepository,
+		maintenanceWindowsRepository,
+	} = shared;
+
+	// ***********************
+	// Network providers
+	// ***********************
+	const pingProvider = new PingProvider(ping);
+	const httpProvider = new HttpProvider(got, new AdvancedMatcher(jmespath));
+	const pageSpeedProvider = new PageSpeedProvider(httpProvider, settingsService, logger);
+	const hardwareProvider = new HardwareProvider(httpProvider);
+	const dockerProvider = new DockerProvider(logger, Docker);
+	const portProvider = new PortProvider(net);
+	const gameProvider = new GameProvider(logger, GameDig);
+	const grpcProvider = new GrpcProvider(grpc, protoLoader);
+	const webSocketProvider = new WebSocketProvider(WebSocket);
+	const dnsProvider = new DNSProvider(() => new Resolver());
+
+	const networkService = new NetworkService(axios, logger, [
+		pingProvider,
+		httpProvider,
+		pageSpeedProvider,
+		hardwareProvider,
+		dockerProvider,
+		portProvider,
+		gameProvider,
+		grpcProvider,
+		webSocketProvider,
+		dnsProvider,
+	]);
+
+	const bufferService = new BufferService(logger, checkService, geoChecksService, settingsService, jobsRepository);
+	const statusService = new StatusService(logger, monitorsRepository, monitorStatsRepository);
+	const monitorStatusPolicy = new MonitorStatusPolicy();
+
+	// ***********************
+	// Reactors and dispatcher
+	// Handles notifications and incidents
+	// ***********************
+
+	const notificationReactor = new NotificationReactor(notificationsService);
+	const incidentReactor = new IncidentReactor(incidentService);
+	const reactorDispatcher = new ReactorDispatcher(logger, [notificationReactor, incidentReactor]);
+
+	// ***********************
+	// Check producer/evaluator
+	// Handles creating and evaluatiog checks
+	// ***********************
+	const checkProducer = new CheckProducer(monitorsRepository, maintenanceWindowsRepository, checkService, networkService, bufferService, logger);
+	const checkEvaluator = new CheckEvaluator(statusService, monitorStatusPolicy);
+	const geoCheckPipeline = new GeoChecksPipeline(maintenanceWindowsRepository, geoChecksService, bufferService, logger);
+
+	// ***********************
+	// Worker
+	// ***********************
+
+	const workerHelper = new WorkerHelper(
+		logger,
+		checkService,
+		settingsService,
+		monitorsRepository,
+		jobsRepository,
+		teamsRepository,
+		monitorStatsRepository,
+		checksRepository,
+		incidentsRepository,
+		geoChecksRepository
+	);
+
+	const worker = await DBQueueWorker.create(
+		logger,
+		jobsRepository,
+		monitorsRepository,
+		checksRepository,
+		checkService,
+		checkProducer,
+		checkEvaluator,
+		geoCheckPipeline,
+		reactorDispatcher,
+		workerHelper,
+		queueWorkersRepository,
+		envSettings.queueMode,
+		envSettings.queuePrimaryProcesses,
+		workerId
+	);
+
+	return { worker, networkService, bufferService, statusService };
+};

server/src/db/db.mongo.ts

@@ -42,7 +42,7 @@ class MongoDB implements IDb<Mongoose> {
 			// the unique index on MonitorStats.monitorId is enforced, otherwise databases
 			// with stray duplicates (caused by the pre-fix race in updateRunningStats)
 			// will crash at startup when the unique index fails to build.
-			await runMigrations();
+			await runMigrations(this.logger);
 
 			// Sync indexes
 			const models = mongoose.modelNames();

server/src/db/migration/0003_cleanupDuplicateMonitorStats.ts

@@ -1,12 +1,12 @@
 import { MonitorStatsModel } from "../../domain/monitor-stats/monitor-stats.model.js";
-import { logger } from "@/utils/logger.js";
+import type { ILogger } from "@/utils/logger.js";
 
 /**
  * Cleanup duplicate MonitorStats documents
  * Keeps the most recent document (by updatedAt) for each monitorId
  * and deletes all older duplicates
  */
-export async function cleanupDuplicateMonitorStats(): Promise<void> {
+export async function cleanupDuplicateMonitorStats(logger: ILogger): Promise<void> {
 	const SERVICE_NAME = "Migration:CleanupDuplicateMonitorStats";
 
 	try {

server/src/db/migration/0004_fixInfrastructureThresholds.ts

@@ -1,11 +1,11 @@
 import { MonitorModel } from "../../domain/monitors/monitor.model.js";
-import { logger } from "@/utils/logger.js";
+import type { ILogger } from "@/utils/logger.js";
 
 /**
  * Fix infrastructure monitors that had thresholds incorrectly set to 0 or 5
  * due to the old default value. Updates them to 100 (disabled).
  */
-export async function fixInfrastructureThresholds(): Promise<void> {
+export async function fixInfrastructureThresholds(logger: ILogger): Promise<void> {
 	const SERVICE_NAME = "Migration:FixInfrastructureThresholds";
 
 	try {

server/src/db/migration/0005_migrateStatusPageTypeToArray.ts

@@ -1,8 +1,8 @@
 import mongoose from "mongoose";
-import { logger } from "@/utils/logger.js";
+import type { ILogger } from "@/utils/logger.js";
 import StatusPageModel from "../../domain/status-pages/status-page.model.js";
 
-export async function migrateStatusPageTypeToArray(): Promise<void> {
+export async function migrateStatusPageTypeToArray(logger: ILogger): Promise<void> {
 	const SERVICE_NAME = "Migration:MigrateStatusPageTypeToArray";
 
 	try {

server/src/db/migration/0006_cleanupDuplicateMonitorStatsForUniqueIndex.ts

@@ -1,5 +1,5 @@
 import { MonitorStatsModel } from "../../domain/monitor-stats/monitor-stats.model.js";
-import { logger } from "@/utils/logger.js";
+import type { ILogger } from "@/utils/logger.js";
 
 /**
  * Second-pass dedupe of MonitorStats before the unique index on monitorId is enforced.
@@ -11,7 +11,7 @@ import { logger } from "@/utils/logger.js";
  * Keeps the document with the highest totalChecks for each monitorId (most data) and
  * deletes the rest. Falls back to most recent updatedAt when totalChecks tie.
  */
-export async function cleanupDuplicateMonitorStatsForUniqueIndex(): Promise<void> {
+export async function cleanupDuplicateMonitorStatsForUniqueIndex(logger: ILogger): Promise<void> {
 	const SERVICE_NAME = "Migration:CleanupDuplicateMonitorStatsForUniqueIndex";
 
 	try {

server/src/db/migration/0007_migrateMaintenanceWindowMonitorIdToArray.ts

@@ -1,6 +1,6 @@
 import mongoose from "mongoose";
 import type { AnyBulkWriteOperation, Document } from "mongodb";
-import { logger } from "@/utils/logger.js";
+import type { ILogger } from "@/utils/logger.js";
 import MaintenanceWindowModel from "../../domain/maintenance-windows/maintenance-window.model.js";
 
 type GroupResult = {
@@ -18,7 +18,7 @@ type GroupResult = {
 	monitorIds: mongoose.Types.ObjectId[];
 };
 
-export async function migrateMaintenanceWindowMonitorIdToArray(): Promise<void> {
+export async function migrateMaintenanceWindowMonitorIdToArray(logger: ILogger): Promise<void> {
 	const SERVICE_NAME = "Migration:MigrateMaintenanceWindowMonitorIdToArray";
 
 	try {

server/src/db/migration/0008_backfillMonitorLastEvaluatedAt.ts

@@ -0,0 +1,41 @@
+import mongoose from "mongoose";
+import { MonitorModel } from "../../domain/monitors/monitor.model.js";
+import type { ILogger } from "@/utils/logger.js";
+
+const JOBS_COLLECTION = "jobs";
+
+/**
+ * Backfill lastEvaluatedAt on monitors created before the lastEvaluatedAt existed.
+ *
+ * Stamp the upgrade moment, not 0. If set to 0, all checks will be replayed
+ */
+export async function backfillMonitorLastEvaluatedAt(logger: ILogger): Promise<void> {
+	const SERVICE_NAME = "Migration:BackfillMonitorLastEvaluatedAt";
+
+	try {
+		// Drop the legacy jobs collection first. The old super-simple-scheduler persisted jobs to the
+		// same "jobs" collection the new db-queue uses, so any leftover rows carry an incompatible schema.
+		// Migrations run before syncIndexes(), so the new Job indexes are rebuilt right after this drop.
+		const db = mongoose.connection.db;
+		if (db) {
+			const existing = await db.listCollections({ name: JOBS_COLLECTION }).toArray();
+			if (existing.length > 0) {
+				await db.collection(JOBS_COLLECTION).drop();
+				logger.info({ service: SERVICE_NAME, message: `Dropped legacy "${JOBS_COLLECTION}" collection` });
+			}
+		}
+
+		logger.info({ service: SERVICE_NAME, message: "Starting lastEvaluatedAt backfill" });
+
+		const result = await MonitorModel.updateMany({ lastEvaluatedAt: { $exists: false } }, { $set: { lastEvaluatedAt: Date.now() } });
+
+		logger.info({
+			service: SERVICE_NAME,
+			message: `Backfilled lastEvaluatedAt on ${result.modifiedCount} monitors`,
+		});
+	} catch (error) {
+		const errorMessage = error instanceof Error ? error.message : String(error);
+		logger.error({ service: SERVICE_NAME, message: `Error backfilling lastEvaluatedAt: ${errorMessage}` });
+		throw error;
+	}
+}

server/src/db/migration/index.ts

@@ -6,11 +6,12 @@ import MigrationModel from "../../domain/migrations/migration.model.js";
 import { migrateStatusPageTypeToArray } from "./0005_migrateStatusPageTypeToArray.js";
 import { cleanupDuplicateMonitorStatsForUniqueIndex } from "./0006_cleanupDuplicateMonitorStatsForUniqueIndex.js";
 import { migrateMaintenanceWindowMonitorIdToArray } from "./0007_migrateMaintenanceWindowMonitorIdToArray.js";
+import { backfillMonitorLastEvaluatedAt } from "./0008_backfillMonitorLastEvaluatedAt.js";
 import type { ILogger } from "@/utils/logger.js";
 
 type MigrationEntry = {
 	name: string;
-	execute: () => Promise<void>;
+	execute: (logger: ILogger) => Promise<void>;
 };
 
 const migrations: MigrationEntry[] = [
@@ -21,36 +22,37 @@ const migrations: MigrationEntry[] = [
 	{ name: "0005_migrateStatusPageTypeToArray", execute: migrateStatusPageTypeToArray },
 	{ name: "0006_cleanupDuplicateMonitorStatsForUniqueIndex", execute: cleanupDuplicateMonitorStatsForUniqueIndex },
 	{ name: "0007_migrateMaintenanceWindowMonitorIdToArray", execute: migrateMaintenanceWindowMonitorIdToArray },
+	{ name: "0008_backfillMonitorLastEvaluatedAt", execute: backfillMonitorLastEvaluatedAt },
 ];
 
-const runMigrations = async (logger?: ILogger) => {
+const runMigrations = async (logger: ILogger) => {
 	try {
-		logger?.info({ message: "Running migrations", service: "Migrations" });
+		logger.info({ message: "Running migrations", service: "Migrations" });
 		for (const migration of migrations) {
 			const exists = await MigrationModel.findOne({ name: migration.name, status: "completed" });
 			if (exists) {
-				logger?.info({ message: `Skipping ${migration.name}`, service: "Migrations" });
+				logger.info({ message: `Skipping ${migration.name}`, service: "Migrations" });
 				continue;
 			}
 
 			try {
-				await migration.execute();
+				await migration.execute(logger);
 				await MigrationModel.findOneAndUpdate(
 					{ name: migration.name },
 					{ status: "completed", completedAt: new Date(), error: undefined },
 					{ upsert: true }
 				);
-				logger?.info({ message: `Completed ${migration.name}`, service: "Migrations" });
+				logger.info({ message: `Completed ${migration.name}`, service: "Migrations" });
 			} catch (error) {
 				const err = error as Error;
 				await MigrationModel.findOneAndUpdate({ name: migration.name }, { status: "failed", error: err?.message }, { upsert: true });
 				throw error;
 			}
 		}
-		logger?.info({ message: "Migrations completed", service: "Migrations" });
+		logger.info({ message: "Migrations completed", service: "Migrations" });
 	} catch (error) {
 		const err = error as Error;
-		logger?.error({ message: "Migration failed", service: "Migrations", details: { error: err?.message }, stack: err?.stack });
+		logger.error({ message: "Migration failed", service: "Migrations", details: { error: err?.message }, stack: err?.stack });
 		throw error;
 	}
 };

server/src/domain/app-settings/app-settings.service.ts

@@ -1,5 +1,5 @@
 import { ISettingsRepository } from "@/domain/app-settings/app-settings-repository.interface.js";
-import { Settings, SettingsUpdate, type DbType, type QueueMode, type QueueType } from "@/domain/app-settings/app-settings.type.js";
+import { Settings, SettingsUpdate, type DbType, type QueueMode } from "@/domain/app-settings/app-settings.type.js";
 import { AppError } from "@/utils/AppError.js";
 import { ValidatedEnv } from "@/config/envValidation.js";
 import type { StringValue } from "ms";
@@ -13,8 +13,8 @@ export type EnvConfig = {
 	clientHost: string;
 	dbConnectionString: string;
 	dbType: DbType;
-	queueType: QueueType;
 	queueMode: QueueMode;
+	queuePrimaryProcesses: boolean;
 	statusPageThemesEnabled: boolean;
 };
 
@@ -41,8 +41,8 @@ export class SettingsService implements ISettingsService {
 			clientHost: env.CLIENT_HOST,
 			dbConnectionString: env.DB_CONNECTION_STRING,
 			dbType: env.DB_TYPE,
-			queueType: env.QUEUE_TYPE,
 			queueMode: env.QUEUE_MODE,
+			queuePrimaryProcesses: env.QUEUE_PRIMARY_PROCESSES,
 			statusPageThemesEnabled: env.STATUS_PAGE_THEMES_ENABLED,
 		};
 	}

server/src/domain/app-settings/app-settings.type.ts

@@ -1,9 +1,6 @@
 export const DbTypes = ["mongodb"] as const;
 export type DbType = (typeof DbTypes)[number];
 
-export const QueueTypes = ["lessSimpleQueue", "superSimpleQueue"] as const;
-export type QueueType = (typeof QueueTypes)[number];
-
 export const QueueModes = ["primary", "worker"] as const;
 export type QueueMode = (typeof QueueModes)[number];
 

server/src/domain/checks/check.model.ts

@@ -241,6 +241,7 @@ const CheckSchema = new Schema<CheckDocument>(
 		},
 		errors: {
 			type: [errorSchema],
+			suppressReservedKeysWarning: true,
 			default: () => [],
 		},
 		capture: {

server/src/domain/checks/check.repository.interface.ts

@@ -42,6 +42,7 @@ export interface IChecksRepository {
 		options?: { type?: MonitorType }
 	): Promise<UptimeChecksResult | HardwareChecksResult | PageSpeedChecksResult>;
 	findSummaryByTeamId(teamId: string, dateRange: string): Promise<ChecksSummary>;
+	findUnevaluatedByMonitorId(monitorId: string, since: number): Promise<Check[]>;
 	// update
 	//delete
 	deleteByMonitorId(monitorId: string): Promise<number>;

server/src/domain/checks/check.repository.mongo.ts

@@ -386,6 +386,16 @@ class MongoChecksRepository implements IChecksRepository {
 		};
 	};
 
+	findUnevaluatedByMonitorId = async (monitorId: string, since: number) => {
+		const docs = await CheckModel.find({
+			"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
+			createdAt: { $gt: new Date(since) },
+		})
+			.sort({ createdAt: 1 })
+			.lean<CheckDocument[]>();
+		return docs.map(this.toEntity);
+	};
+
 	deleteByMonitorId = async (monitorId: string): Promise<number> => {
 		const result = await CheckModel.deleteMany({ "metadata.monitorId": new mongoose.Types.ObjectId(monitorId) });
 		return result.deletedCount;

server/src/domain/checks/check.service.ts

@@ -11,7 +11,9 @@ const SERVICE_NAME = "checkService";
 
 export interface ICheckService {
 	createChecks(checks: Check[]): Promise<Check[]>;
-	buildCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
+	toLastEvaluatedAt(check: Check): number;
+	toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
+	toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
 	getChecksByMonitor(params: {
 		monitorId: string;
 		teamId: string;
@@ -56,7 +58,11 @@ export class CheckService implements ICheckService {
 		return this.checksRepository.createChecks(checks);
 	};
 
-	buildCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
+	toLastEvaluatedAt = (check: Check) => {
+		return new Date(check.createdAt).getTime();
+	};
+
+	toCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
 		const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
 
 		const now = new Date().toISOString();
@@ -82,7 +88,7 @@ export class CheckService implements ICheckService {
 				this.logger.warn({
 					message: "Failed to build check",
 					service: SERVICE_NAME,
-					method: "buildCheck",
+					method: "toCheck",
 				});
 				return undefined;
 			}
@@ -131,6 +137,24 @@ export class CheckService implements ICheckService {
 		return check;
 	};
 
+	toStatusResponse = (check: Check) => {
+		const statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]> = {
+			monitorId: check.metadata.monitorId,
+			teamId: check.metadata.teamId,
+			type: check.metadata.type,
+			status: check.status,
+			code: check.statusCode,
+			message: check.message,
+			responseTime: check.responseTime,
+			// re-nest hardware metrics so updateMonitorStatus / extractThresholdBreaches see payload.data
+			payload:
+				check.metadata.type === "hardware"
+					? ({ data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload)
+					: undefined,
+		};
+		return statusResponse;
+	};
+
 	getChecksByMonitor = async ({
 		monitorId,
 		teamId,

server/src/domain/diagnostics/diagnostic.service.ts

@@ -6,6 +6,22 @@ import { AppError } from "@/utils/AppError.js";
 
 const SERVICE_NAME = "diagnosticService";
 
+export interface MongoStats {
+	reads: number;
+	inserts: number;
+	updates: number;
+	deletes: number;
+	timestamp: number;
+}
+
+export interface MongoOpsPerSecond {
+	readsPerSecond: number;
+	insertsPerSecond: number;
+	updatesPerSecond: number;
+	deletesPerSecond: number;
+	writesPerSecond: number;
+}
+
 export interface CollectionDiagnostics {
 	name: string;
 	documentCount: number;
@@ -17,6 +33,11 @@ export interface CollectionDiagnostics {
 
 export interface MongoDiagnostics {
 	readyState: number;
+	readsPerSecond: number;
+	insertsPerSecond: number;
+	updatesPerSecond: number;
+	deletesPerSecond: number;
+	writesPerSecond: number;
 	host: string;
 	port: number;
 	dbName: string;
@@ -44,6 +65,8 @@ export interface IDiagnosticService {
 export class DiagnosticService implements IDiagnosticService {
 	static SERVICE_NAME = SERVICE_NAME;
 
+	private lastMongoOpsPerSecond: MongoStats | null = null;
+
 	constructor(private db: IDb<Mongoose>) {
 		/**
 		 * Performance Observer for monitoring system performance metrics.
@@ -68,6 +91,48 @@ export class DiagnosticService implements IDiagnosticService {
 		return ((endUsage.user + endUsage.system) / 1000 / timingPeriod) * 100;
 	};
 
+	private getMongoOpsPerSecond = async (): Promise<MongoOpsPerSecond | null> => {
+		try {
+			const mongo = await this.db.getConnection();
+			const db = mongo.connection.db;
+			if (!db) {
+				throw new AppError({ message: "Database connection is not available", service: SERVICE_NAME, method: "getMongoDBStats" });
+			}
+			const sample = async (): Promise<MongoStats> => {
+				const res = await db.command({ serverStatus: 1 });
+				return {
+					reads: (res.opcounters?.query ?? 0) + (res.opcounters?.getmore ?? 0),
+					inserts: res.opcounters?.insert ?? 0,
+					updates: res.opcounters?.update ?? 0,
+					deletes: res.opcounters?.delete ?? 0,
+					timestamp: Date.now(),
+				};
+			};
+
+			// First call has no baseline, seed one so we always have a rate
+			if (this.lastMongoOpsPerSecond === null) {
+				this.lastMongoOpsPerSecond = await sample();
+				await new Promise((resolve) => setTimeout(resolve, 1000));
+			}
+
+			const previous = this.lastMongoOpsPerSecond;
+			const current = await sample();
+			this.lastMongoOpsPerSecond = current;
+
+			const elapsedSeconds = (current.timestamp - previous.timestamp) / 1000;
+			return {
+				readsPerSecond: (current.reads - previous.reads) / elapsedSeconds,
+				insertsPerSecond: (current.inserts - previous.inserts) / elapsedSeconds,
+				updatesPerSecond: (current.updates - previous.updates) / elapsedSeconds,
+				deletesPerSecond: (current.deletes - previous.deletes) / elapsedSeconds,
+				writesPerSecond:
+					(current.inserts - previous.inserts + current.updates - previous.updates + current.deletes - previous.deletes) / elapsedSeconds,
+			};
+		} catch {
+			return null;
+		}
+	};
+
 	private getMongoDBStats = async (): Promise<MongoDiagnostics> => {
 		const mongo = await this.db.getConnection();
 		const db = mongo.connection.db;
@@ -75,6 +140,8 @@ export class DiagnosticService implements IDiagnosticService {
 			throw new AppError({ message: "Database connection is not available", service: SERVICE_NAME, method: "getMongoDBStats" });
 		}
 
+		const opsPerSecond = await this.getMongoOpsPerSecond();
+
 		const dbStats = await db.stats();
 		const rawCollections = await db.collections();
 
@@ -100,6 +167,11 @@ export class DiagnosticService implements IDiagnosticService {
 
 		return {
 			readyState: mongo.connection.readyState,
+			readsPerSecond: opsPerSecond?.readsPerSecond ?? 0,
+			insertsPerSecond: opsPerSecond?.insertsPerSecond ?? 0,
+			updatesPerSecond: opsPerSecond?.updatesPerSecond ?? 0,
+			deletesPerSecond: opsPerSecond?.deletesPerSecond ?? 0,
+			writesPerSecond: opsPerSecond?.writesPerSecond ?? 0,
 			host: mongo.connection.host,
 			port: mongo.connection.port,
 			dbName: mongo.connection.name,

server/src/domain/jobs/job.model.ts

@@ -0,0 +1,79 @@
+import { Schema, model } from "mongoose";
+import { JOB_TYPES, type Job } from "@/domain/jobs/job.type.js";
+
+export interface JobDocument extends Omit<Job, "id"> {
+	_id: string;
+}
+
+const JobSchema = new Schema<JobDocument>(
+	{
+		_id: {
+			type: String,
+			required: true,
+		},
+		type: {
+			type: String,
+			enum: JOB_TYPES,
+			required: true,
+		},
+		// monitorId for monitor-bound jobs; null for global cleanup jobs. Kept as a String pointer (no populate).
+		refId: {
+			type: String,
+			default: null,
+		},
+		// paused monitor => false; the claim scan skips inactive rows (replaces pauseJob/resumeJob)
+		isActive: {
+			type: Boolean,
+			default: true,
+		},
+
+		// Scheduling — epoch ms, so the claim/$min queries operate on plain numbers
+		nextScheduledAt: {
+			type: Number,
+			required: true,
+		},
+		intervalMs: {
+			type: Number,
+			default: null,
+		},
+
+		// Lock / lease
+		lockedBy: {
+			type: String,
+			default: null,
+		},
+		lockedUntil: {
+			type: Number,
+			default: null,
+		},
+
+		// Observability
+		runCount: {
+			type: Number,
+			default: 0,
+		},
+		failCount: {
+			type: Number,
+			default: 0,
+		},
+		lastFinishedAt: {
+			type: Number,
+			default: null,
+		},
+		lastFailReason: {
+			type: String,
+			default: null,
+		},
+	},
+	{ timestamps: true }
+);
+
+// Used for scanning for claims
+JobSchema.index({ type: 1, isActive: 1, nextScheduledAt: 1 });
+// for findById, upsertEvaluate, deleteById
+JobSchema.index({ refId: 1, type: 1 });
+
+const JobModel = model<JobDocument>("Job", JobSchema);
+
+export { JobModel };
+export default JobModel;

server/src/domain/jobs/job.repository.interface.ts

@@ -0,0 +1,75 @@
+import { Job, JobSeed, JobType } from "@/domain/jobs/job.type.js";
+
+export type JobPageQuery = {
+	page?: number;
+	rowsPerPage?: number;
+};
+export type JobPage = {
+	jobs: Job[];
+	count: number;
+};
+
+export interface IJobsRepository {
+	// ********************
+	// Claims
+	// ********************
+
+	// Atomically claim a job
+	claimDue(type: JobType, now: number): Promise<Job | null>;
+
+	// locks up to `limit` due rows in one batch
+	claimDueBatch(type: JobType, limit: number, now: number): Promise<Job[]>;
+
+	// Extends the locks we still hold. Returns how many were renewed
+	renewLocks(ids: string[], now: number): Promise<number>;
+
+	// Completes a repeating job
+	recordSuccess(id: string, nextScheduledAt: number, intervalMs: number, now: number): Promise<boolean>;
+
+	// Record failure, release lease, bump fail count, record reason, reset nextScheduledAt for retry
+	recordFailure(id: string, error: unknown, now: number): Promise<boolean>;
+
+	recordOneShot(id: string, now: number): Promise<boolean>;
+
+	// ********************
+	// Hand off to evaluator // This is accomplished by creating an "evaluation" type job
+	// ********************
+	upsertEvaluate(monitorId: string, now: number): Promise<boolean>;
+
+	// ********************
+	// Job CRUD
+	// ********************
+
+	// Create jobs. Takes a JobSeed
+	upsertJob(job: JobSeed): Promise<boolean>;
+
+	// Creates cleanup jobs, these are executed immediately so need their own method
+	upsertCleanupJob(job: JobSeed): Promise<boolean>;
+
+	// Pause/Resume
+	setActiveById(refId: string, isActive: boolean): Promise<boolean>;
+
+	// Edit job schedule
+	updateScheduleById(refId: string, type: JobType, intervalMs: number | null): Promise<boolean>;
+
+	// Re-arm the check/geo-check rows.  Add jitter to prevent herd
+	markMonitorsDue(monitorIds: string[], now: number): Promise<number>;
+
+	// Delete, drop all rows
+	deleteById(refId: string): Promise<boolean>;
+
+	// Delete a single job of type
+	deleteByIdAndType(refId: string, type: JobType): Promise<boolean>;
+
+	// Delete jobs that reference monitors that no longer exist
+	deleteByMonitorIdsNotIn(monitorIds: string[]): Promise<number>;
+
+	// Get job
+	findById(refId: string): Promise<Job[]>;
+
+	// ********************
+	// Observability
+	// ********************
+	findPage(pagination: JobPageQuery): Promise<JobPage>;
+	findAll(): Promise<Job[]>;
+}

server/src/domain/jobs/job.repository.mongo.ts

@@ -0,0 +1,306 @@
+import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js";
+import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
+import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js";
+
+// Cap for rearm interval, never want to wait more than 15s.
+const REARM_JITTER_MAX_MS = 15_000;
+
+class MongoJobsRepository implements IJobsRepository {
+	constructor(private readonly workerId: string) {}
+
+	private toEntity(doc: JobDocument): Job {
+		return {
+			id: doc._id,
+			type: doc.type,
+			refId: doc.refId,
+			isActive: doc.isActive,
+			nextScheduledAt: doc.nextScheduledAt,
+			intervalMs: doc.intervalMs,
+			lockedBy: doc.lockedBy,
+			lockedUntil: doc.lockedUntil,
+			runCount: doc.runCount,
+			failCount: doc.failCount,
+			lastFinishedAt: doc.lastFinishedAt,
+			lastFailReason: doc.lastFailReason,
+		};
+	}
+
+	claimDue = async (type: JobType, now: number) => {
+		const doc = await JobModel.findOneAndUpdate(
+			{
+				type,
+				isActive: true,
+				nextScheduledAt: { $lte: now }, // Find due jobs
+				$or: [
+					{ lockedUntil: null }, // Not locked
+					{ lockedUntil: { $lt: now } }, // Free or expired lock
+				],
+			},
+			{
+				$set: {
+					lockedBy: this.workerId,
+					lockedUntil: now + LOCK_MS,
+				},
+			},
+			{
+				new: true,
+				sort: { nextScheduledAt: 1 }, // Get the oldest due job
+			}
+		).lean<JobDocument>();
+		return doc ? this.toEntity(doc) : null;
+	};
+
+	claimDueBatch = async (type: JobType, limit: number, now: number) => {
+		if (limit <= 0) return [];
+		const dueAndFree = {
+			type,
+			isActive: true,
+			nextScheduledAt: { $lte: now }, // due
+			$or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }], // free or expired lock
+		};
+
+		// 1. Pick the oldest-due candidates
+		const candidates = await JobModel.find(dueAndFree).sort({ nextScheduledAt: 1 }).limit(limit).select({ _id: 1 }).lean<{ _id: string }[]>();
+		if (candidates.length === 0) return [];
+		const ids = candidates.map((d) => d._id);
+
+		// 2. lock them. Re-check free/expired so we never steal another worker's live lock
+		//    only the rows this updateMany actually flips end up with claimed
+		const lockedUntil = now + LOCK_MS;
+		await JobModel.updateMany(
+			{ _id: { $in: ids }, $or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }] },
+			{ $set: { lockedBy: this.workerId, lockedUntil } }
+		);
+
+		// 3. Read back exactly the rows we now hold from this batch, keyed by the lockedUntil time
+		const docs = await JobModel.find({ _id: { $in: ids }, lockedBy: this.workerId, lockedUntil }).lean<JobDocument[]>();
+		return docs.map(this.toEntity);
+	};
+
+	renewLocks = async (ids: string[], now: number) => {
+		if (ids.length === 0) return 0;
+		const res = await JobModel.updateMany(
+			{ _id: { $in: ids }, lockedBy: this.workerId }, // only locks held by this worker can be renewed
+			{ $set: { lockedUntil: now + LOCK_MS } }
+		);
+		return res.modifiedCount;
+	};
+
+	recordSuccess = async (id: string, scheduledRunAt: number, intervalMs: number, now: number) => {
+		// Fixed rate job scheduling
+
+		let nextScheduledAt = scheduledRunAt + intervalMs;
+
+		// If the job fell behind, skip the missed runs. Spread the next run with jitter to avoid a herd
+		if (nextScheduledAt <= now) {
+			nextScheduledAt = now + intervalMs + Math.floor(Math.random() * intervalMs);
+		}
+
+		const res = await JobModel.updateOne(
+			{
+				_id: id,
+				lockedBy: this.workerId, // Only the worker that claimed can record success
+			},
+			{
+				$set: {
+					nextScheduledAt,
+					lockedBy: null, // Remove lock
+					lockedUntil: null,
+					lastFinishedAt: now,
+				},
+				$inc: {
+					runCount: 1, // Increment run count
+				},
+			}
+		);
+		return res.modifiedCount === 1;
+	};
+
+	recordFailure = async (id: string, error: unknown, now: number) => {
+		const res = await JobModel.updateOne(
+			{
+				_id: id,
+				lockedBy: this.workerId, // Only the worker that claimed can record success
+			},
+			{
+				$set: {
+					nextScheduledAt: now + BACKOFF_MS,
+					lockedBy: null, // Remove lock
+					lockedUntil: null,
+					lastFinishedAt: now, // a failed run still finished used in getMetrics for failedAt timestamp
+					lastFailReason: error instanceof Error ? error.message : String(error),
+				},
+				$inc: {
+					failCount: 1, // Increment fail count
+				},
+			}
+		);
+		return res.modifiedCount === 1;
+	};
+
+	recordOneShot = async (id: string, now: number) => {
+		const res = await JobModel.updateOne(
+			{
+				_id: id,
+				lockedBy: this.workerId, // Only the worker that claimed can record success
+			},
+			{
+				$set: {
+					nextScheduledAt: PARKED,
+					lockedBy: null, // Remove lock
+					lockedUntil: null,
+					lastFinishedAt: now,
+				},
+				$inc: {
+					runCount: 1, // Increment run count
+				},
+			}
+		);
+		return res.modifiedCount === 1;
+	};
+
+	upsertEvaluate = async (monitorId: string, now: number) => {
+		const filter = { type: "evaluate", refId: monitorId };
+		const update = {
+			$setOnInsert: {
+				_id: jobId("evaluate", monitorId),
+				// refId and Type inferred from filter
+				isActive: true,
+				intervalMs: null,
+				lockedBy: null,
+				lockedUntil: null,
+				runCount: 0,
+				failCount: 0,
+				lastFinishedAt: null,
+				lastFailReason: null,
+			},
+			$min: {
+				nextScheduledAt: now, // no op if there is already a pending evaluation that is older
+			},
+		};
+		try {
+			const res = await JobModel.updateOne(filter, update, { upsert: true });
+			return res.acknowledged;
+		} catch (error) {
+			// Two callers can race the first insert for the same monitor, so the
+			// loser hits a duplicate _id. Retry once, the row now exists, so this becomes the $min.
+			if (error && typeof error === "object" && (error as { code?: number }).code === 11000) {
+				const res = await JobModel.updateOne(filter, update, { upsert: true });
+				return res.acknowledged;
+			}
+			throw error;
+		}
+	};
+	upsertJob = async (job: JobSeed) => {
+		const res = await JobModel.updateOne(
+			{ _id: job.id },
+			{
+				$set: {
+					// These fields are owned by the monitor, always safe to set
+					type: job.type,
+					refId: job.refId,
+					isActive: job.isActive,
+					intervalMs: job.intervalMs,
+				},
+				$setOnInsert: {
+					// These fields are scheduling related, only set on insert so they aren't clobbered
+					nextScheduledAt: job.nextScheduledAt,
+					lockedBy: null,
+					lockedUntil: null,
+					runCount: 0,
+					failCount: 0,
+					lastFinishedAt: null,
+					lastFailReason: null,
+				},
+			},
+			{ upsert: true }
+		);
+		return res.acknowledged;
+	};
+
+	upsertCleanupJob = async (job: JobSeed) => {
+		const res = await JobModel.updateOne(
+			{ _id: job.id },
+			{
+				$set: {
+					type: job.type,
+					refId: job.refId,
+					isActive: job.isActive,
+					intervalMs: job.intervalMs,
+					nextScheduledAt: job.nextScheduledAt,
+				},
+				$setOnInsert: {
+					lockedBy: null,
+					lockedUntil: null,
+					runCount: 0,
+					failCount: 0,
+					lastFinishedAt: null,
+					lastFailReason: null,
+				},
+			},
+			{ upsert: true }
+		);
+		return res.acknowledged;
+	};
+
+	setActiveById = async (refId: string, isActive: boolean) => {
+		const res = await JobModel.updateMany({ refId }, { $set: { isActive } }); // inverts check and geo check rows together
+		return res.modifiedCount > 0;
+	};
+
+	updateScheduleById = async (refId: string, type: JobType, intervalMs: number | null) => {
+		const res = await JobModel.updateOne({ refId, type }, { $set: { intervalMs } });
+		return res.modifiedCount === 1;
+	};
+
+	markMonitorsDue = async (monitorIds: string[], now: number) => {
+		if (monitorIds.length === 0) return 0;
+		const res = await JobModel.updateMany({ refId: { $in: monitorIds }, type: { $in: ["check", "geo-check"] } }, [
+			{
+				// Spread rearmed jobs, but never for more than rearm jitter max
+				$set: {
+					nextScheduledAt: {
+						$add: [now, { $floor: { $multiply: [{ $rand: {} }, { $min: ["$intervalMs", REARM_JITTER_MAX_MS] }] } }],
+					},
+				},
+			},
+		]);
+		return res.modifiedCount;
+	};
+
+	deleteById = async (refId: string) => {
+		const res = await JobModel.deleteMany({ refId }); // Delete all jobs for monitor
+		return res.deletedCount > 0;
+	};
+
+	deleteByIdAndType = async (refId: string, type: JobType) => {
+		const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row
+		return res.deletedCount > 0;
+	};
+
+	deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise<number> => {
+		// remove jobs who'se refId is not null (cleanup jobs) and not in the list of monitor IDs (orphaned monitor jobs)
+		const res = await JobModel.deleteMany({ refId: { $ne: null, $nin: monitorIds } });
+		return res.deletedCount ?? 0;
+	};
+
+	findById = async (refId: string) => {
+		const docs = await JobModel.find({ refId }).lean<JobDocument[]>();
+		return docs.map(this.toEntity);
+	};
+
+	findPage = async ({ page = 0, rowsPerPage = 0 }: JobPageQuery) => {
+		const count = await JobModel.countDocuments();
+		const query = JobModel.find().sort({ nextScheduledAt: 1 });
+		if (rowsPerPage > 0) query.skip(Math.max(page, 0) * rowsPerPage).limit(rowsPerPage);
+		const docs = await query.lean<JobDocument[]>();
+		return { jobs: docs.map(this.toEntity), count };
+	};
+
+	findAll = async () => {
+		const docs = await JobModel.find().lean<JobDocument[]>();
+		return docs.map(this.toEntity);
+	};
+}
+
+export default MongoJobsRepository;

server/src/domain/jobs/job.type.ts

@@ -0,0 +1,32 @@
+export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"] as const;
+export type JobType = (typeof JOB_TYPES)[number];
+export const LOCK_MS = 60_000;
+export const BACKOFF_MS = 5_000;
+export const PARKED = Number.MAX_SAFE_INTEGER;
+
+// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null) are just the type.
+export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`);
+
+export type Job = {
+	id: string;
+	type: JobType;
+	refId: string | null;
+	isActive: boolean;
+
+	// Scheduling fields
+	nextScheduledAt: number; // epoch ms; due when nextScheduledAt <= now
+	intervalMs: number | null; // repeat period in ms; null = one time job
+
+	// Lock fields
+	lockedBy: string | null; // workerId holding the lease
+	lockedUntil: number | null; // epoch ms; lease expiry => reclaimable after a crash
+
+	// observability (replaces what getMetrics/getJobs)
+	runCount: number;
+	failCount: number;
+	lastFinishedAt: number | null;
+	lastFailReason: string | null;
+};
+
+// These are the only values to set when creating a job.  Repo owns the rest
+export type JobSeed = Pick<Job, "id" | "type" | "refId" | "isActive" | "intervalMs" | "nextScheduledAt">;

server/src/domain/maintenance-windows/maintenance-window.service.ts

@@ -1,8 +1,10 @@
 import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
 import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
+import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
 import type { DurationUnit, MaintenanceWindow } from "@/domain/maintenance-windows/maintenance-window.type.js";
 import { AppError } from "@/utils/AppError.js";
 import { isWindowActive } from "@/utils/maintenanceWindow.js";
+import { IJobScheduler } from "@/worker/worker.interface.js";
 
 const SERVICE_NAME = "maintenanceWindowService";
 
@@ -40,16 +42,24 @@ export class MaintenanceWindowService implements IMaintenanceWindowService {
 	static SERVICE_NAME = SERVICE_NAME;
 	private monitorsRepository: IMonitorsRepository;
 	private maintenanceWindowsRepository: IMaintenanceWindowsRepository;
+	private jobsRepository: IJobsRepository;
+	private scheduler: IJobScheduler;
 
 	constructor({
 		monitorsRepository,
 		maintenanceWindowsRepository,
+		jobsRepository,
+		scheduler,
 	}: {
 		monitorsRepository: IMonitorsRepository;
 		maintenanceWindowsRepository: IMaintenanceWindowsRepository;
+		jobsRepository: IJobsRepository;
+		scheduler: IJobScheduler;
 	}) {
 		this.monitorsRepository = monitorsRepository;
 		this.maintenanceWindowsRepository = maintenanceWindowsRepository;
+		this.jobsRepository = jobsRepository;
+		this.scheduler = scheduler;
 	}
 
 	get serviceName() {
@@ -71,6 +81,11 @@ export class MaintenanceWindowService implements IMaintenanceWindowService {
 		const toInitializing = monitorIds.filter((monitorId) => !stillCovered.has(monitorId));
 		if (toInitializing.length > 0) {
 			await this.monitorsRepository.updateByIds(toInitializing, teamId, { status: "initializing" }, ["paused"]);
+			// Rearm jobs to run soon (now + jitter to avoid herding)
+			await this.jobsRepository.markMonitorsDue(toInitializing, now.getTime());
+			// Wake the (possibly idle, backed-off) loops so the rearmed jobs run promptly instead of waiting up to POLL_MAX_MS
+			this.scheduler.wake("check");
+			this.scheduler.wake("geo-check");
 		}
 	};
 

server/src/domain/monitors/monitor.model.ts

@@ -1,6 +1,6 @@
 import { Schema, model, Types } from "mongoose";
 import type { Monitor, MonitorMatchMethod, CheckSnapshot } from "@/domain/monitors/monitor.types.js";
-import { DnsRecordTypes, MonitorTypes, MonitorStatuses, PageSpeedStrategies } from "@/domain/monitors/monitor.types.js";
+import { DnsRecordTypes, MonitorTypes, MonitorStatuses, PageSpeedStrategies, HttpMethods } from "@/domain/monitors/monitor.types.js";
 import type {
 	CheckAudits,
 	CheckCaptureInfo,
@@ -34,6 +34,7 @@ interface MonitorDocument extends MonitorDocumentBase {
 	teamId: Types.ObjectId;
 	createdAt: Date;
 	updatedAt: Date;
+	lastEvaluatedAt: number; // epoch ms
 }
 
 const snapshotTimingPhasesSchema = new Schema<GotTimings["phases"]>(
@@ -196,7 +197,7 @@ const checkSnapshotSchema = new Schema<CheckSnapshotDocument>(
 		audits: { type: snapshotAuditsSchema },
 		createdAt: { type: Date, required: true },
 	},
-	{ _id: false }
+	{ _id: false, suppressReservedKeysWarning: true }
 );
 
 const MonitorSchema = new Schema<MonitorDocument>(
@@ -220,6 +221,12 @@ const MonitorSchema = new Schema<MonitorDocument>(
 		description: {
 			type: String,
 		},
+		method: {
+			type: String,
+			enum: HttpMethods,
+			default: "GET",
+			required: true,
+		},
 		status: {
 			type: String,
 			enum: MonitorStatuses,
@@ -377,6 +384,10 @@ const MonitorSchema = new Schema<MonitorDocument>(
 			type: [checkSnapshotSchema],
 			default: [],
 		},
+		lastEvaluatedAt: {
+			type: Number,
+			default: 0,
+		},
 	},
 	{
 		timestamps: true,

server/src/domain/monitors/monitor.repository.interface.ts

@@ -22,6 +22,8 @@ export interface IMonitorsRepository {
 	createMonitors(monitors: Monitor[]): Promise<Monitor[]>;
 	// single fetch
 	findById(monitorId: string, teamId: string): Promise<Monitor>;
+	// for workers, don't need all details
+	findByIdLean(monitorId: string): Promise<Monitor | null>;
 
 	// collection fetch
 	findAll(): Promise<Monitor[] | null>;

server/src/domain/monitors/monitor.repository.mongo.ts

@@ -38,6 +38,11 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 		return this.toEntity(monitor);
 	};
 
+	findByIdLean = async (monitorId: string): Promise<Monitor | null> => {
+		const monitor = await MonitorModel.findOne({ _id: monitorId });
+		return monitor ? this.toEntity(monitor) : null;
+	};
+
 	findAll = async (): Promise<Monitor[]> => {
 		const monitors = await MonitorModel.find();
 		return this.mapDocuments(monitors);
@@ -531,6 +536,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 			teamId: toStringId(doc.teamId),
 			name: doc.name,
 			description: doc.description ?? undefined,
+			method: doc.method ?? "GET",
 			status: doc.status ?? "initializing",
 			statusWindow: doc.statusWindow ?? [],
 			statusWindowSize: doc.statusWindowSize,
@@ -571,6 +577,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 			dnsRecordType: doc.dnsRecordType ?? undefined,
 			createdAt: toDateString(doc.createdAt),
 			updatedAt: toDateString(doc.updatedAt),
+			lastEvaluatedAt: doc.lastEvaluatedAt,
 		};
 	};
 
@@ -596,6 +603,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 			teamId: toStringId(doc.teamId),
 			name: doc.name,
 			description: doc.description ?? undefined,
+			method: doc.method ?? "GET",
 			status: doc.status ?? "initializing",
 			statusWindow: doc.statusWindow ?? [],
 			statusWindowSize: doc.statusWindowSize,
@@ -636,6 +644,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
 			dnsRecordType: doc.dnsRecordType ?? undefined,
 			createdAt: toDateString(doc.createdAt),
 			updatedAt: toDateString(doc.updatedAt),
+			lastEvaluatedAt: doc.lastEvaluatedAt,
 		};
 	};
 

server/src/domain/monitors/monitor.service.ts

@@ -21,8 +21,8 @@ import type { IStatusPagesRepository } from "@/domain/status-pages/status-page-r
 import demoMonitorsData from "@/utils/demoMonitors.json" with { type: "json" };
 import { AppError } from "@/utils/AppError.js";
 import type { ImportedMonitor } from "@/api/validation/monitorValidation.js";
-import { IWorker } from "@/worker/worker.interface.js";
 import { ILogger } from "@/utils/logger.js";
+import { IJobScheduler } from "@/worker/worker.interface.js";
 
 const SERVICE_NAME = "MonitorService";
 type DateRangeKey = "recent" | "day" | "week" | "month" | "all";
@@ -94,7 +94,7 @@ export interface IMonitorService {
 export class MonitorService implements IMonitorService {
 	static SERVICE_NAME = SERVICE_NAME;
 
-	private worker: IWorker;
+	private scheduler: IJobScheduler;
 	private logger: ILogger;
 	private games: GamesMap;
 	private monitorsRepository: IMonitorsRepository;
@@ -105,7 +105,7 @@ export class MonitorService implements IMonitorService {
 	private incidentsRepository: IIncidentsRepository;
 
 	constructor({
-		worker,
+		scheduler,
 		logger,
 		games,
 		monitorsRepository,
@@ -115,7 +115,7 @@ export class MonitorService implements IMonitorService {
 		statusPagesRepository,
 		incidentsRepository,
 	}: {
-		worker: IWorker;
+		scheduler: IJobScheduler;
 		logger: ILogger;
 		games: GamesMap;
 		monitorsRepository: IMonitorsRepository;
@@ -125,7 +125,7 @@ export class MonitorService implements IMonitorService {
 		statusPagesRepository: IStatusPagesRepository;
 		incidentsRepository: IIncidentsRepository;
 	}) {
-		this.worker = worker;
+		this.scheduler = scheduler;
 		this.logger = logger;
 		this.games = games;
 		this.monitorsRepository = monitorsRepository;
@@ -171,7 +171,7 @@ export class MonitorService implements IMonitorService {
 			throw new AppError({ message: "Failed to create monitor", status: 500, service: SERVICE_NAME, method: "createMonitor" });
 		}
 
-		this.worker.addJob(monitor.id, monitor);
+		this.scheduler.addJob(monitor.id, monitor);
 	};
 
 	createMonitors = async (monitors: Array<Monitor>): Promise<Monitor[] | null> => {
@@ -180,7 +180,7 @@ export class MonitorService implements IMonitorService {
 			throw new AppError({ message: "Failed to create monitors", status: 500, service: SERVICE_NAME, method: "createMonitors" });
 		}
 
-		await Promise.all(createdMonitors.map((monitor) => this.worker.addJob(monitor.id, monitor)));
+		await Promise.all(createdMonitors.map((monitor) => this.scheduler.addJob(monitor.id, monitor)));
 		return createdMonitors;
 	};
 
@@ -196,7 +196,7 @@ export class MonitorService implements IMonitorService {
 		}));
 		const demoMonitors = await this.monitorsRepository.createMonitors(monitors as unknown as Monitor[]);
 
-		await Promise.all(demoMonitors.map((monitor) => this.worker.addJob(monitor.id, monitor)));
+		await Promise.all(demoMonitors.map((monitor) => this.scheduler.addJob(monitor.id, monitor)));
 		return demoMonitors;
 	};
 
@@ -428,7 +428,7 @@ export class MonitorService implements IMonitorService {
 
 	editMonitor = async ({ teamId, monitorId, body }: { teamId: string; monitorId: string; body: Partial<Monitor> }) => {
 		const editedMonitor = await this.monitorsRepository.updateById(monitorId, teamId, body);
-		await this.worker.updateJob(editedMonitor);
+		await this.scheduler.updateJob(editedMonitor);
 		return editedMonitor;
 	};
 
@@ -448,7 +448,7 @@ export class MonitorService implements IMonitorService {
 		// If notifications were updated, we should update the jobs in the queue
 		if (modifiedCount > 0) {
 			const monitors = await this.monitorsRepository.findByIds(monitorIds);
-			await Promise.all(monitors.map((monitor) => this.worker.updateJob(monitor)));
+			await Promise.all(monitors.map((monitor) => this.scheduler.updateJob(monitor)));
 		}
 
 		return modifiedCount;
@@ -457,9 +457,9 @@ export class MonitorService implements IMonitorService {
 	pauseMonitor = async ({ teamId, monitorId }: { teamId: string; monitorId: string }): Promise<Monitor> => {
 		const monitor = await this.monitorsRepository.togglePauseById(monitorId, teamId);
 		if (monitor.isActive) {
-			await this.worker.resumeJob(monitor);
+			await this.scheduler.resumeJob(monitor);
 		} else {
-			await this.worker.pauseJob(monitor);
+			await this.scheduler.pauseJob(monitor);
 		}
 		return monitor;
 	};
@@ -478,9 +478,9 @@ export class MonitorService implements IMonitorService {
 		const results = await Promise.allSettled(
 			monitors.map(async (monitor) => {
 				if (monitor.isActive) {
-					await this.worker.resumeJob(monitor);
+					await this.scheduler.resumeJob(monitor);
 				} else {
-					await this.worker.pauseJob(monitor);
+					await this.scheduler.pauseJob(monitor);
 				}
 			})
 		);
@@ -541,7 +541,7 @@ export class MonitorService implements IMonitorService {
 			});
 		});
 
-		await this.worker.deleteJob(monitor);
+		await this.scheduler.deleteJob(monitor);
 		return monitor;
 	};
 
@@ -550,7 +550,7 @@ export class MonitorService implements IMonitorService {
 		await Promise.all(
 			monitors.map(async (monitor) => {
 				try {
-					await this.worker.deleteJob(monitor);
+					await this.scheduler.deleteJob(monitor);
 					await this.checksRepository.deleteByMonitorId(monitor.id);
 					await this.geoChecksRepository.deleteByMonitorId(monitor.id);
 					await this.statusPagesRepository.removeMonitorFromStatusPages(monitor.id);
@@ -597,6 +597,7 @@ export class MonitorService implements IMonitorService {
 			recentChecks: [],
 			createdAt: "",
 			updatedAt: "",
+			lastEvaluatedAt: 0,
 		}));
 
 		const createdMonitors = await this.createMonitors(cleanedMonitors);

server/src/domain/monitors/monitor.types.ts

@@ -4,7 +4,29 @@ import type { GeoContinent, GroupedGeoCheck } from "@/domain/geo-checks/geo-chec
 export type { GeoContinent } from "@/domain/geo-checks/geo-check.type.js";
 import http from "node:http";
 
-export const HttpStatusCodes = Object.keys(http.STATUS_CODES).map(Number);
+export const HttpStatusCodes = [
+	...Object.keys(http.STATUS_CODES).map(Number),
+	419, // Page Expired (Laravel)
+	420, // Enhance Your Calm (Twitter)
+	440, // Login Time-out (IIS)
+	449, // Retry With (IIS)
+	460, // Client Closed Connection (AWS ELB)
+	463, // X-Forwarded-For Too Large (AWS ELB)
+	497, // HTTP Request Sent to HTTPS Port (NGINX)
+	499, // Client Closed Request (NGINX)
+	509, // Bandwidth Limit Exceeded (Apache)
+	520, // Web Server Returned an Unknown Error (Cloudflare)
+	521, // Web Server Is Down (Cloudflare)
+	522, // Connection Timed Out (Cloudflare)
+	523, // Origin Is Unreachable (Cloudflare)
+	524, // A Timeout Occurred (Cloudflare)
+	525, // SSL Handshake Failed (Cloudflare)
+	526, // Invalid SSL Certificate (Cloudflare)
+	527, // Railgun Error (Cloudflare)
+	529, // Site is overloaded
+	530, // Site is frozen (Cloudflare)
+	561, // Unauthorized (AWS ELB)
+];
 export const HttpStatusCodeSet = new Set(HttpStatusCodes);
 export type HttpStatusCode = number;
 
@@ -40,6 +62,9 @@ export type MonitorMatchMethod = (typeof MonitorMatchMethods)[number] | "";
 export const DnsRecordTypes = ["A", "AAAA", "CNAME", "MX", "TXT", "NS"] as const;
 export type DnsRecordType = (typeof DnsRecordTypes)[number];
 
+export const HttpMethods = ["GET", "HEAD"] as const;
+export type HttpMethod = (typeof HttpMethods)[number];
+
 export const MAX_RECENT_CHECKS = 50;
 
 export interface Monitor {
@@ -48,6 +73,7 @@ export interface Monitor {
 	teamId: string;
 	name: string;
 	description?: string;
+	method: HttpMethod;
 	status: MonitorStatus;
 	statusWindow: boolean[];
 	statusWindowSize: number;
@@ -88,6 +114,7 @@ export interface Monitor {
 	recentChecks: CheckSnapshot[];
 	createdAt: string;
 	updatedAt: string;
+	lastEvaluatedAt: number; // epoch ms
 }
 
 export interface MonitorsSummary {

server/src/domain/queue-workers/queue-worker.model.ts

@@ -2,11 +2,15 @@ import { Schema, model } from "mongoose";
 import { QueueModes } from "@/domain/app-settings/app-settings.type.js";
 import { QueueWorkerDocument } from "@/domain/queue-workers/queue-worker.type.js";
 
+// Single source of truth for worker liveness: the TTL that GCs stale records and
+// the window readers use to count a worker as alive. Heartbeat cadence must stay well below it.
+export const WORKER_TTL_SECONDS = 180;
+
 const QueueWorkerSchema = new Schema<QueueWorkerDocument>(
 	{
 		_id: { type: String, required: true },
 		mode: { type: String, enum: QueueModes, required: true },
-		lastSeenAt: { type: Date, default: Date.now, expires: 30 },
+		lastSeenAt: { type: Date, default: Date.now, expires: WORKER_TTL_SECONDS },
 	},
 	{ timestamps: true }
 );

server/src/domain/users/user.service.ts

@@ -11,9 +11,9 @@ import { AppError } from "@/utils/AppError.js";
 import { IEmailService } from "@/service/emailService.js";
 import { EnvConfig, ISettingsService } from "@/domain/app-settings/app-settings.service.js";
 import { ILogger } from "@/utils/logger.js";
+import { IJobScheduler } from "@/worker/worker.interface.js";
 import jwt from "jsonwebtoken";
 import crypto from "crypto";
-import { IWorker } from "@/worker/worker.interface.js";
 type CryptoType = typeof crypto;
 type JwtType = typeof jwt;
 const SERVICE_NAME = "userService";
@@ -53,7 +53,7 @@ export class UserService implements IUserService {
 	private settingsService: ISettingsService;
 	private logger: ILogger;
 	private jwt: JwtType;
-	private worker: IWorker;
+	private scheduler: IJobScheduler;
 	private crypto: CryptoType;
 	private monitorsRepository: IMonitorsRepository;
 	private usersRepository: IUsersRepository;
@@ -68,7 +68,7 @@ export class UserService implements IUserService {
 		settingsService,
 		logger,
 		jwt,
-		worker,
+		scheduler,
 		monitorsRepository,
 		usersRepository,
 		invitesRepository,
@@ -81,7 +81,7 @@ export class UserService implements IUserService {
 		settingsService: ISettingsService;
 		logger: ILogger;
 		jwt: JwtType;
-		worker: IWorker;
+		scheduler: IJobScheduler;
 		monitorsRepository: IMonitorsRepository;
 		usersRepository: IUsersRepository;
 		invitesRepository: IInvitesRepository;
@@ -93,7 +93,7 @@ export class UserService implements IUserService {
 		this.settingsService = settingsService;
 		this.logger = logger;
 		this.jwt = jwt;
-		this.worker = worker;
+		this.scheduler = scheduler;
 		this.crypto = crypto;
 		this.monitorsRepository = monitorsRepository;
 		this.usersRepository = usersRepository;
@@ -330,7 +330,7 @@ export class UserService implements IUserService {
 		if (roles.includes("superadmin")) {
 			const monitors = await this.monitorsRepository.findByTeamId(teamId, {});
 			if (monitors) {
-				await Promise.all(monitors.map((monitor) => this.worker.deleteJob(monitor)));
+				await Promise.all(monitors.map((monitor) => this.scheduler.deleteJob(monitor)));
 			}
 		}
 		// 6. Delete the user by id

server/src/index.ts

@@ -1,4 +1,7 @@
-import { initializeServices } from "./config/services.js";
+import { buildShared } from "@/config/services.shared.js";
+import { buildApi } from "@/config/services.api.js";
+import { buildWorker } from "@/config/services.worker.js";
+
 import { initializeControllers } from "./config/controllers.js";
 import { createApp } from "./app.js";
 import { initShutdownListener } from "@/shutdown.js";
@@ -9,44 +12,84 @@ import fs from "fs";
 
 import Logger, { ILogger } from "@/utils/logger.js";
 import { SettingsService } from "@/domain/app-settings/app-settings.service.js";
-
+import { JobScheduler } from "@/worker/worker.job-scheduler.js";
+import { IJobScheduler } from "@/worker/worker.interface.js";
 const SERVICE_NAME = "Server";
 let logger: ILogger;
 
 const startApp = async () => {
-	// Validate environment variables first
-	const env = validateEnv();
+	// ***********************
+	// Basic setup
+	// ***********************
 
-	// Create settings service (env only — DB repository injected after connect)
+	logger = new Logger();
+	const env = validateEnv(logger);
+	logger.setLogLevel(env.LOG_LEVEL);
 	const settingsService = new SettingsService(env);
 	const envSettings = settingsService.loadSettings();
-	// Create logger
-	logger = new Logger({ envSettings });
+	const { queueMode, queuePrimaryProcesses } = envSettings;
 
-	// Initialize services (connects DB, creates repositories, injects settingsRepository)
-	const services = await initializeServices({ logger, envSettings, settingsService, queueMode: envSettings.queueMode });
+	logger.info({
+		message: `Queue mode: ${queueMode}`,
+		service: SERVICE_NAME,
+		method: "startApp",
+	});
 
-	// If this is a worker instance, we're done.  No need for express
-	if (envSettings.queueMode === "worker") {
-		logger.info({
-			message: "Worker instance started. API will not be started",
-			service: SERVICE_NAME,
-		});
-		initShutdownListener(null, services);
+	logger.info({
+		message: `Process: ${queuePrimaryProcesses}`,
+		service: SERVICE_NAME,
+		method: "startApp",
+	});
+
+	// ***********************
+	// Build shared services
+	// ***********************
+	const shared = await buildShared({ logger, envSettings, settingsService });
+
+	// ***********************
+	// Worker node path, don't need API
+	// ***********************
+	if (queueMode === "worker") {
+		const { worker } = await buildWorker(shared, envSettings);
+		logger.info({ message: "Worker instance started. API will not be started", service: SERVICE_NAME });
+		initShutdownListener(null, { worker, db: shared.db, logger });
 		return;
 	}
 
-	// FE path
+	// ***********************
+	// Primary node path
+	// ***********************
+	let scheduler: IJobScheduler;
+
+	// ***********************
+	//Primary node processes jobs, need full worker
+	// ***********************
+	if (queuePrimaryProcesses === true) {
+		const { worker } = await buildWorker(shared, envSettings);
+		scheduler = worker;
+	}
+	// ***********************
+	// Primary node does not process jobs, only need scheduler
+	// ***********************
+	else {
+		scheduler = new JobScheduler(shared.jobsRepository, shared.queueWorkersRepository, shared.workerId);
+	}
+
+	const services = buildApi(shared, scheduler);
+
+	// ***********************
+	// FE assets always needed for primary node
+	// ***********************
+
 	const __filename = fileURLToPath(import.meta.url);
 	const __dirname = path.dirname(__filename);
 	const openApiSpec = JSON.parse(fs.readFileSync(path.join(__dirname, "../openapi.json"), "utf8"));
 	const frontendPath = path.join(__dirname, "..", "public");
 
-	// Initialize controllers
 	const controllers = initializeControllers(services);
 
 	const app = createApp({
-		services,
+		apiServices: services,
 		controllers,
 		envSettings,
 		frontendPath,

server/src/service/bufferService.ts

@@ -3,6 +3,7 @@ import type { GeoCheck } from "@/domain/geo-checks/geo-check.type.js";
 import type { IGeoChecksService } from "../domain/geo-checks/geo-check.service.js";
 import type { ILogger } from "@/utils/logger.js";
 import type { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
+import type { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
 import { ICheckService } from "../domain/checks/check.service.js";
 const SERVICE_NAME = "BufferService";
 
@@ -24,12 +25,20 @@ export class BufferService implements IBufferService {
 	private bufferTimer: NodeJS.Timeout | null = null;
 	private checksService: ICheckService;
 	private geoChecksService: IGeoChecksService;
+	private jobsRepository: IJobsRepository;
 
-	constructor(logger: ILogger, checkService: ICheckService, geoChecksService: IGeoChecksService, settingsService: ISettingsService) {
+	constructor(
+		logger: ILogger,
+		checkService: ICheckService,
+		geoChecksService: IGeoChecksService,
+		settingsService: ISettingsService,
+		jobsRepository: IJobsRepository
+	) {
 		this.BUFFER_TIMEOUT = settingsService.getSettings().nodeEnv === "development" ? 1000 : 1000 * 60 * 1; // 1 minute
 		this.logger = logger;
 		this.checksService = checkService;
 		this.geoChecksService = geoChecksService;
+		this.jobsRepository = jobsRepository;
 		this.SERVICE_NAME = SERVICE_NAME;
 		this.buffer = [];
 		this.geoBuffer = [];
@@ -93,16 +102,23 @@ export class BufferService implements IBufferService {
 		}, this.BUFFER_TIMEOUT);
 	}
 	async flushBuffer() {
+		if (this.buffer.length === 0) {
+			return;
+		}
+		// Take the batch first so a write that drains it can't be appended to mid-flush
+		const batch = this.buffer;
+		this.buffer = [];
 		try {
-			if (this.buffer.length > 0) {
-				this.logger.debug({
-					message: `Flushing ${this.buffer.length} checks to database`,
-					service: this.SERVICE_NAME,
-					method: "flushBuffer",
-				});
-				await this.checksService.createChecks(this.buffer);
-				this.buffer = [];
-			}
+			this.logger.debug({
+				message: `Flushing ${batch.length} checks to database`,
+				service: this.SERVICE_NAME,
+				method: "flushBuffer",
+			});
+			await this.checksService.createChecks(batch);
+			// Need to evaluate checks when they are flushed
+			const monitorIds = [...new Set(batch.map((check) => check.metadata.monitorId))];
+			const now = Date.now();
+			await Promise.all(monitorIds.map((monitorId) => this.jobsRepository.upsertEvaluate(monitorId, now)));
 		} catch (error: unknown) {
 			this.logger.error({
 				message: error instanceof Error ? error.message : "Unknown error",
@@ -110,8 +126,7 @@ export class BufferService implements IBufferService {
 				method: "flushBuffer",
 				stack: error instanceof Error ? error.stack : undefined,
 			});
-			// Clear buffer even on error to prevent infinite retry loops
-			this.buffer = [];
+			// Batch is already cleared from the buffer; dropping it prevents infinite retry loops
 		}
 	}
 

server/src/service/network/HttpProvider.ts

@@ -53,6 +53,22 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
 		}
 	): MonitorStatusResponse<T> {
 		const { body, contentType, statusCode, statusUp, message, responseTime, timings } = opts;
+
+		// Return early for HEAD requests, no body to parse
+		if (monitor.method === "HEAD") {
+			return {
+				monitorId: monitor.id,
+				teamId: monitor.teamId,
+				type: monitor.type,
+				status: statusUp,
+				code: statusCode,
+				message,
+				responseTime,
+				timings,
+				payload: body as T | string,
+			};
+		}
+
 		const isJson = contentType.includes("application/json");
 
 		if (monitor.jsonPath && !isJson) {
@@ -155,6 +171,8 @@ export class HttpProvider implements IStatusProvider<HttpStatusPayload> {
 			https: ignoreTlsErrors ? this.httpsAgentInsecure : this.httpsAgent,
 		};
 
+		options.method = monitor.method;
+
 		try {
 			const response = await this.got<string>(url, options);
 			const statusUp = isStatusUp(response.statusCode, monitor.customUpCodes);

server/src/service/statusService.ts

@@ -1,4 +1,3 @@
-import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
 import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js";
 import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
 import type { Check, CheckDiskInfo, CheckSnapshot } from "@/domain/checks/check.type.js";
@@ -17,7 +16,6 @@ import type {
 } from "@/types/network.js";
 import { AppError } from "@/utils/AppError.js";
 import { ILogger } from "@/utils/logger.js";
-import { IBufferService } from "./bufferService.js";
 import type { HardwareStatusMetrics } from "@/types/network.js";
 import { MAX_RECENT_CHECKS } from "@/domain/monitors/monitor.types.js";
 
@@ -42,30 +40,21 @@ export interface IStatusService {
 			| GrpcStatusPayload
 			| undefined
 		>,
-		check: Check
+		check: Check,
+		monitor: Monitor
 	): Promise<StatusChangeResult>;
 }
 
 export class StatusService implements IStatusService {
 	static SERVICE_NAME = SERVICE_NAME;
 	private logger: ILogger;
-	private buffer: IBufferService;
 	private monitorsRepository: IMonitorsRepository;
 	private monitorStatsRepository: IMonitorStatsRepository;
-	private checksRepository: IChecksRepository;
 
-	constructor(
-		logger: ILogger,
-		buffer: IBufferService,
-		monitorsRepository: IMonitorsRepository,
-		monitorStatsRepository: IMonitorStatsRepository,
-		checksRepository: IChecksRepository
-	) {
+	constructor(logger: ILogger, monitorsRepository: IMonitorsRepository, monitorStatsRepository: IMonitorStatsRepository) {
 		this.logger = logger;
-		this.buffer = buffer;
 		this.monitorsRepository = monitorsRepository;
 		this.monitorStatsRepository = monitorStatsRepository;
-		this.checksRepository = checksRepository;
 	}
 
 	get serviceName() {
@@ -212,11 +201,11 @@ export class StatusService implements IStatusService {
 			| GrpcStatusPayload
 			| undefined
 		>,
-		check: Check
+		check: Check,
+		monitor: Monitor
 	): Promise<StatusChangeResult> => {
 		try {
-			const { monitorId, teamId, status, code } = statusResponse;
-			const monitor = await this.monitorsRepository.findById(monitorId, teamId);
+			const { status, code } = statusResponse;
 
 			// Update running stats
 			await this.tryUpdateRunningStats(monitor, statusResponse);
@@ -232,10 +221,10 @@ export class StatusService implements IStatusService {
 			// Build the status patch — computed against the projected window
 			const patch: Partial<Monitor> = {};
 
-			// Resolve "initializing" up-front, incidents should be created on initialization && down
+			// Resolve "initializing" and "maintenance" up front, incidents should be created on initialization && down
 			let newStatus: MonitorStatus = monitor.status;
 			let statusChanged = false;
-			if (monitor.status === "initializing") {
+			if (monitor.status === "initializing" || monitor.status === "maintenance") {
 				newStatus = status === true ? "up" : "down";
 				patch.status = newStatus;
 				statusChanged = newStatus === "down";
@@ -262,7 +251,7 @@ export class StatusService implements IStatusService {
 				};
 			}
 
-			// First evaluate reachability-based status changes, which apply to all monitor types
+			// First evaluate reachability status changes, which apply to all monitor types
 			// and take precedence over hardware breaches.
 			const reachabilityResult = this.computeReachability(newStatus, projectedWindow, monitor.statusWindowThreshold);
 			if (reachabilityResult.transitioned) {

server/src/shutdown.ts

@@ -1,9 +1,17 @@
-import { InitializedServices } from "./config/services.js";
-import { logger } from "./utils/logger.js";
+import { IDb } from "@/db/db.interface.js";
+import type { ILogger } from "@/utils/logger.js";
 import type { Server } from "http";
+import { Mongoose } from "mongoose";
+import { IJobScheduler } from "@/worker/worker.interface.js";
 
-export const initShutdownListener = (server: Server | null, services: InitializedServices) => {
+type ShutdownTargets = {
+	worker: IJobScheduler;
+	db: IDb<Mongoose>;
+	logger: ILogger;
+};
+export const initShutdownListener = (server: Server | null, services: ShutdownTargets) => {
 	const SERVICE_NAME = "Server";
+	const logger = services.logger;
 
 	let isShuttingDown = false;
 

server/src/utils/logger.ts

@@ -1,7 +1,6 @@
 import { createLogger, format, transports, Logger as WinstonLogger } from "winston";
 import type { Logform } from "winston";
 import dotenv from "dotenv";
-import { EnvConfig } from "@/domain/app-settings/app-settings.service.js";
 dotenv.config();
 
 const SERVICE_NAME = "Logger";
@@ -28,18 +27,17 @@ export interface ILogger {
 	cacheLog(entry: LogEntry): void;
 	getLogs(): LogEntry[];
 	buildLogEntry(level: string, config: LogConfig): LogEntry;
+	setLogLevel(level: string): void;
 }
 
 class Logger implements ILogger {
 	static SERVICE_NAME = SERVICE_NAME;
 
 	private logger: WinstonLogger;
-	private envSettings: Partial<EnvConfig>;
 	private logCache: LogEntry[];
 	private maxCacheSize: number;
 
-	constructor({ envSettings }: { envSettings: Partial<EnvConfig> }) {
-		this.envSettings = envSettings;
+	constructor() {
 		this.logCache = [];
 		this.maxCacheSize = 1000;
 		const consoleFormat = format.printf((info: Logform.TransformableInfo) => {
@@ -84,10 +82,8 @@ class Logger implements ILogger {
 			return msg;
 		});
 
-		const logLevel = this.envSettings.logLevel || "info";
-
 		this.logger = createLogger({
-			level: logLevel,
+			level: "info",
 			format: format.combine(format.timestamp()),
 			transports: [
 				new transports.Console({
@@ -151,11 +147,10 @@ class Logger implements ILogger {
 			timestamp: new Date().toISOString(),
 		};
 	}
+
+	setLogLevel(level: string): void {
+		this.logger.level = level || "info";
+	}
 }
 
 export default Logger;
-
-// Legacy logger
-const logger = new Logger({ envSettings: { logLevel: "debug" } });
-export { logger };
-export type { LogConfig, LogEntry };

server/src/worker/worker.check-evaluator.ts

@@ -0,0 +1,34 @@
+import { MonitorStatusResponse } from "@/types/network.js";
+import { MonitorEvaluation } from "@/worker/worker.interface.js";
+import { Check } from "@/domain/checks/check.type.js";
+import { Monitor } from "@/domain/monitors/monitor.types.js";
+import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
+import { IStatusService } from "@/service/statusService.js";
+
+export interface ICheckEvaluator {
+	evaluate(status: MonitorStatusResponse, check: Check, monitor: Monitor): Promise<MonitorEvaluation>;
+}
+
+export class CheckEvaluator implements ICheckEvaluator {
+	constructor(
+		private statusService: IStatusService,
+		private monitorStatusPolicy: IMonitorStatusPolicy
+	) {}
+	evaluate = async (status: MonitorStatusResponse, check: Check, monitor: Monitor) => {
+		// ****************************
+		// Step 3:  Evaluate and return result to reactors
+		// ****************************
+		const statusChangeResult = await this.statusService.updateMonitorStatus(status, check, monitor);
+
+		// Step 5.  Get decisions and create an evaluation obj
+		const decision = this.monitorStatusPolicy.evaluate(statusChangeResult);
+		const evaluation: MonitorEvaluation = {
+			monitor: statusChangeResult.monitor,
+			status,
+			check,
+			statusChange: statusChangeResult,
+			decision,
+		};
+		return evaluation;
+	};
+}

server/src/worker/worker.check-pipeline.ts

@@ -1,14 +1,9 @@
 import { Monitor } from "@/domain/monitors/monitor.types.js";
 import { MonitorEvaluation } from "@/worker/worker.interface.js";
 import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
-import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
 import { isWindowActive } from "@/utils/maintenanceWindow.js";
 import { ILogger } from "@/utils/logger.js";
-import { INetworkService } from "@/service/networkService.js";
-import { ICheckService } from "@/domain/checks/check.service.js";
 import { IBufferService } from "@/service/bufferService.js";
-import { IMonitorStatusPolicy } from "@/worker/worker.monitor-status-policy.js";
-import { IStatusService } from "@/service/statusService.js";
 import { AppError } from "@/utils/AppError.js";
 import { supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
 import { IGeoChecksService } from "@/domain/geo-checks/geo-check.service.js";
@@ -19,89 +14,6 @@ export interface ICheckPipeline {
 	run(monitor: Monitor): Promise<MonitorEvaluation | null>; // null = skipped
 }
 
-export class CheckPipeline implements ICheckPipeline {
-	constructor(
-		private monitorsRepository: IMonitorsRepository,
-		private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
-		private checkService: ICheckService,
-		private networkService: INetworkService,
-		private bufferService: IBufferService,
-		private monitorStatusPolicy: IMonitorStatusPolicy,
-		private statusService: IStatusService,
-		private logger: ILogger
-	) {}
-
-	private async isInMaintenanceWindow(monitorId: string, teamId: string) {
-		const maintenanceWindows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
-		const now = new Date();
-		return maintenanceWindows.some((window) => isWindowActive(window, now));
-	}
-
-	run = async (monitor: Monitor): Promise<MonitorEvaluation | null> => {
-		const monitorId = monitor.id;
-		if (!monitorId) {
-			throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "getMonitorJob" });
-		}
-		// ****************************
-		// Step 1:  Acquire
-		// ****************************
-
-		// Step 1a:  Maintenance window gate - skip if in maintenance
-		const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId);
-		if (maintenanceWindowActive) {
-			this.logger.debug({
-				message: `Monitor ${monitor.id} is in maintenance window`,
-				service: SERVICE_NAME,
-				method: "runCheckPipeline",
-			});
-			if (monitor.status !== "maintenance") {
-				await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance" });
-			}
-			return null;
-		}
-
-		// Step 1b: Acquire status
-		const status = await this.networkService.requestStatus(monitor);
-		if (!status) {
-			throw new Error("No network response");
-		}
-
-		// ****************************
-		// Step 2: Record
-		// ****************************
-
-		// Step 2a:  Create & record a check, return null if fail
-		const check = this.checkService.buildCheck(status);
-		if (!check) {
-			this.logger.warn({
-				message: `No check could be built for monitor ${monitor.id}`,
-				service: SERVICE_NAME,
-				method: "runCHeckPipeline",
-				details: { code: status.code, message: status.message },
-			});
-			return null;
-		}
-		// Step 2b: Add to buffer
-		this.bufferService.addToBuffer(check);
-
-		// ****************************
-		// Step 3:  Evaluate and return result to reactors
-		// ****************************
-		const statusChangeResult = await this.statusService.updateMonitorStatus(status, check);
-
-		// Step 5.  Get decisions and create an evaluation obj
-		const decision = this.monitorStatusPolicy.evaluate(statusChangeResult);
-		const evaluation: MonitorEvaluation = {
-			monitor: statusChangeResult.monitor,
-			status,
-			check,
-			statusChange: statusChangeResult,
-			decision,
-		};
-		return evaluation;
-	};
-}
-
 export class GeoChecksPipeline implements ICheckPipeline {
 	constructor(
 		private maintenanceWindowsRepository: IMaintenanceWindowsRepository,

server/src/worker/worker.check-producer.ts

@@ -0,0 +1,83 @@
+import { Monitor } from "@/domain/monitors/monitor.types.js";
+import { Check } from "@/domain/checks/check.type.js";
+import { MonitorStatusResponse } from "@/types/network.js";
+import { AppError } from "@/utils/AppError.js";
+import { ILogger } from "@/utils/logger.js";
+import { IBufferService } from "@/service/bufferService.js";
+import { INetworkService } from "@/service/networkService.js";
+import { ICheckService } from "@/domain/checks/check.service.js";
+import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
+import { IMaintenanceWindowsRepository } from "@/domain/maintenance-windows/maintenance-window.repository.interface.js";
+import { isWindowActive } from "@/utils/maintenanceWindow.js";
+
+export interface ICheckProducer {
+	produce(monitor: Monitor): Promise<{ status: MonitorStatusResponse; check: Check } | null>;
+}
+
+const SERVICE_NAME = "CheckProducer";
+
+export class CheckProducer implements ICheckProducer {
+	constructor(
+		private monitorsRepository: IMonitorsRepository,
+		private maintenanceWindowsRepository: IMaintenanceWindowsRepository,
+		private checkService: ICheckService,
+		private networkService: INetworkService,
+		private bufferService: IBufferService,
+		private logger: ILogger
+	) {}
+
+	private async isInMaintenanceWindow(monitorId: string, teamId: string) {
+		const windows = await this.maintenanceWindowsRepository.findByMonitorId(monitorId, teamId);
+		const now = new Date();
+		return windows.some((w) => isWindowActive(w, now));
+	}
+
+	produce = async (monitor: Monitor) => {
+		if (!monitor.id) {
+			throw new AppError({ message: "No monitor id", service: SERVICE_NAME, method: "produce" });
+		}
+		// ****************************
+		// Step 1:  Acquire
+		// ****************************
+
+		// Step 1a:  Maintenance window gate - skip if in maintenance
+		const maintenanceWindowActive = await this.isInMaintenanceWindow(monitor.id, monitor.teamId);
+		if (maintenanceWindowActive) {
+			this.logger.debug({
+				message: `Monitor ${monitor.id} is in maintenance window`,
+				service: SERVICE_NAME,
+				method: "produce",
+			});
+			if (monitor.status !== "maintenance") {
+				// Clear the status window to avoid incidents being created on next check
+				await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance", statusWindow: [] });
+			}
+			return null;
+		}
+
+		// Step 1b: Acquire status
+		const status = await this.networkService.requestStatus(monitor);
+		if (!status) {
+			throw new Error("No network response");
+		}
+
+		// ****************************
+		// Step 2: Record
+		// ****************************
+
+		// Step 2a:  Create & record a check, return null if fail
+		const check = this.checkService.toCheck(status);
+		if (!check) {
+			this.logger.warn({
+				message: `No check could be built for monitor ${monitor.id}`,
+				service: SERVICE_NAME,
+				method: "produce",
+				details: { code: status.code, message: status.message },
+			});
+			return null;
+		}
+		// Step 2b: Add to buffer
+		this.bufferService.addToBuffer(check);
+		return { status, check };
+	};
+}

server/src/worker/worker.db-queue.ts

@@ -0,0 +1,290 @@
+import { supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
+import { IQueueWorker } from "@/worker/worker.interface.js";
+import { type Job, type JobSeed, type JobType, jobId, LOCK_MS } from "@/domain/jobs/job.type.js";
+import { ILogger } from "@/utils/logger.js";
+import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
+import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
+import { IChecksRepository } from "@/domain/checks/check.repository.interface.js";
+import { ICheckService } from "@/domain/checks/check.service.js";
+import { ICheckProducer } from "@/worker/worker.check-producer.js";
+import { ICheckEvaluator } from "@/worker/worker.check-evaluator.js";
+import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
+import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
+import { IWorkerHelper } from "@/worker/worker.helper.js";
+import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
+import { WORKER_TTL_SECONDS } from "@/domain/queue-workers/queue-worker.model.js";
+import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
+import { JobScheduler } from "@/worker/worker.job-scheduler.js";
+const SERVICE_NAME = "JobQueue";
+const POLL_MS = 250; // base poll interval while a loop is actively claiming work
+const POLL_MAX_MS = 5000; // back off poll interval to 5s if no jobs
+const WORKER_STALE_MS = WORKER_TTL_SECONDS * 1000; // a worker counts as alive if seen within this window
+const HEARTBEAT_MS = WORKER_STALE_MS / 3; // Worker can miss two beats without being considered stale
+const LOCK_RENEW_MS = LOCK_MS / 3; // renew an in-flight job's lock at 1/3 the lease, so it survives two missed renewals
+const CLEANUP_INTERVAL_MS = 24 * 60 * 60 * 1000;
+const CONCURRENCY: Record<JobType, number> = {
+	check: 500,
+	"geo-check": 20,
+	evaluate: 20,
+	"cleanup-orphaned": 1,
+	"cleanup-retention": 1,
+};
+
+export class DBQueueWorker extends JobScheduler implements IQueueWorker {
+	static SERVICE_NAME = SERVICE_NAME;
+	private inFlight: Record<JobType, number> = {
+		check: 0,
+		"geo-check": 0,
+		evaluate: 0,
+		"cleanup-orphaned": 0,
+		"cleanup-retention": 0,
+	};
+
+	constructor(
+		private logger: ILogger,
+		jobsRepository: IJobsRepository, // ← no modifier: forwarded to super
+		private monitorsRepository: IMonitorsRepository,
+		private checksRepository: IChecksRepository,
+		private checkService: ICheckService,
+		private checkProducer: ICheckProducer,
+		private checkEvaluator: ICheckEvaluator,
+		private geoCheckPipeline: ICheckPipeline,
+		private dispatcher: IReactorDispatcher,
+		private helper: IWorkerHelper,
+		queueWorkersRepository: IQueueWorkersRepository, // ← no modifier: forwarded
+		private queueMode: QueueMode,
+		private queuePrimaryProcesses: boolean,
+		workerId: string
+	) {
+		super(jobsRepository, queueWorkersRepository, workerId);
+	}
+
+	get serviceName() {
+		return DBQueueWorker.SERVICE_NAME;
+	}
+
+	static async create(
+		logger: ILogger,
+		jobsRepository: IJobsRepository,
+		monitorsRepository: IMonitorsRepository,
+		checksRepository: IChecksRepository,
+		checkService: ICheckService,
+		checkProducer: ICheckProducer,
+		checkEvaluator: ICheckEvaluator,
+		geoCheckPipeline: ICheckPipeline,
+		dispatcher: IReactorDispatcher,
+		helper: IWorkerHelper,
+		queueWorkersRepository: IQueueWorkersRepository,
+		queueMode: QueueMode,
+		queuePrimaryProcesses: boolean,
+		workerId: string
+	): Promise<DBQueueWorker> {
+		const instance = new DBQueueWorker(
+			logger,
+			jobsRepository,
+			monitorsRepository,
+			checksRepository,
+			checkService,
+			checkProducer,
+			checkEvaluator,
+			geoCheckPipeline,
+			dispatcher,
+			helper,
+			queueWorkersRepository,
+			queueMode,
+			queuePrimaryProcesses,
+			workerId
+		);
+		await instance.init();
+		return instance;
+	}
+
+	private toCleanupJob = (type: "cleanup-orphaned" | "cleanup-retention", now: number): JobSeed => ({
+		id: jobId(type, null),
+		type,
+		refId: null,
+		isActive: true,
+		nextScheduledAt: now,
+		intervalMs: CLEANUP_INTERVAL_MS,
+	});
+
+	// ********************
+	// Stage 1:  This can eventually be offloaded to a separate service
+	// ********************
+
+	private runCheck = async (job: Job) => {
+		if (!job.refId) return;
+		const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
+		if (!monitor) return;
+		await this.checkProducer.produce(monitor);
+	};
+
+	// ********************
+	// Stage 2:  Evaluator loop stays here
+	// ********************
+
+	private runEvaluate = async (job: Job) => {
+		if (!job.refId) return;
+		const monitor = await this.monitorsRepository.findByIdLean(job.refId); // job row has no teamId
+		if (!monitor) return;
+		const checks = await this.checksRepository.findUnevaluatedByMonitorId(job.refId, monitor.lastEvaluatedAt);
+
+		let current = monitor;
+		for (const check of checks) {
+			const status = this.checkService.toStatusResponse(check);
+			const evaluation = await this.checkEvaluator.evaluate(status, check, current);
+			await this.dispatcher.dispatch(evaluation); // Handle incidents and notifications
+			await this.monitorsRepository.updateById(job.refId, current.teamId, { lastEvaluatedAt: this.checkService.toLastEvaluatedAt(check) });
+			current = evaluation.statusChange.monitor; // fresh statusWindow/status/counters for the next check
+		}
+	};
+
+	private runGeoCheck = async (job: Job) => {
+		const monitor = await this.monitorsRepository.findByIdLean(job.refId!);
+		if (monitor) await this.geoCheckPipeline.run(monitor); // returns null; no evaluate handoff
+	};
+
+	// Extend the lock while a job runs so a slow-but-alive job is never reclaimed.
+	private renewLock = async (job: Job): Promise<boolean> => {
+		try {
+			const held = await this.jobsRepository.renewLocks([job.id], Date.now());
+			if (held === 0) {
+				this.logger.warn({ message: `Lost lock on job ${job.id}, stopping renewal`, service: SERVICE_NAME, method: "renewLock" });
+				return false;
+			}
+			return true;
+		} catch (error: unknown) {
+			this.logger.warn({ message: error instanceof Error ? error.message : String(error), service: SERVICE_NAME, method: "renewLock" });
+			return true;
+		}
+	};
+
+	private runJob = async (job: Job) => {
+		const renewTimer = setInterval(async () => {
+			const renewLock = await this.renewLock(job);
+			if (!renewLock) clearInterval(renewTimer);
+		}, LOCK_RENEW_MS);
+		try {
+			switch (job.type) {
+				case "check":
+					await this.runCheck(job);
+					break;
+				case "evaluate":
+					await this.runEvaluate(job);
+					break;
+				case "geo-check":
+					await this.runGeoCheck(job);
+					break;
+				case "cleanup-orphaned":
+					await this.helper.getCleanupOrphanedJob()(); // Get job and execute
+					break;
+				case "cleanup-retention":
+					await this.helper.getCleanupRetentionJob()(); // Get job and execute
+					break;
+			}
+
+			if (job.intervalMs === null) {
+				// One-shot job
+				await this.jobsRepository.recordOneShot(job.id, Date.now());
+			} else {
+				await this.jobsRepository.recordSuccess(job.id, job.nextScheduledAt, job.intervalMs, Date.now());
+			}
+		} catch (error: unknown) {
+			await this.jobsRepository.recordFailure(job.id, error, Date.now()); // Record failure and release lock
+			this.logger.warn({
+				message: error instanceof Error ? error.message : String(error),
+				service: SERVICE_NAME,
+				method: `runJob:${job.type}`,
+			});
+		} finally {
+			clearInterval(renewTimer);
+		}
+	};
+
+	private startLoop = (type: JobType) => {
+		this.pollMs[type] = POLL_MS;
+		const tick = async () => {
+			if (this.stopped) return;
+			this.ticking[type] = true;
+			try {
+				// Claim a batch sized to the free capacity
+				const capacity = CONCURRENCY[type] - this.inFlight[type];
+				const jobs = await this.jobsRepository.claimDueBatch(type, capacity, Date.now());
+				for (const job of jobs) {
+					this.inFlight[type]++; // stay within concurrency limits
+					this.runJob(job).finally(() => {
+						this.inFlight[type]--; // job done, free the slot
+					});
+				}
+				// Back off if no jobs found, reset to base if jobs found
+				this.pollMs[type] = capacity > 0 && jobs.length === 0 ? Math.min(this.pollMs[type] * 2, POLL_MAX_MS) : POLL_MS;
+			} catch (error: unknown) {
+				this.logger.error({
+					message: error instanceof Error ? error.message : String(error),
+					service: SERVICE_NAME,
+					method: `loop:${type}`,
+				});
+			}
+			// Clearing the flag and re-arming run synchronously (no await between), so wake() can never interleave here.
+			this.ticking[type] = false;
+			if (!this.stopped) this.timers.set(type, setTimeout(tick, this.pollMs[type])); // re-arm unless shutting down
+		};
+		this.tickFns.set(type, tick);
+		tick();
+	};
+
+	// Wake a worker loop after adding a job that's due immediately.
+	// Otherwise the loop could be delayed by POLL_MAX_MS if it is already fully backed off
+
+	// Seed queue
+	private reconcile = async () => {
+		const now = Date.now();
+		const monitors = (await this.monitorsRepository.findAll()) ?? [];
+		for (const monitor of monitors) {
+			await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now));
+			if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
+				await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now));
+			}
+		}
+		// Both cleanup jobs run immediately on every startup, then once a day
+		await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-orphaned", now));
+		await this.jobsRepository.upsertCleanupJob(this.toCleanupJob("cleanup-retention", now));
+	};
+
+	// Register worker
+	private heartbeat = async () => {
+		try {
+			await this.queueWorkersRepository.upsert(this.workerId, this.queueMode);
+		} catch (error: unknown) {
+			this.logger.warn({
+				message: error instanceof Error ? error.message : String(error),
+				service: SERVICE_NAME,
+				method: "heartbeat",
+			});
+		}
+	};
+
+	init = async () => {
+		this.stopped = false;
+
+		await this.heartbeat();
+		this.timers.set("heartbeat", setInterval(this.heartbeat, HEARTBEAT_MS));
+
+		if (this.queueMode === "primary") {
+			await this.reconcile();
+		}
+
+		if (this.queuePrimaryProcesses === true || this.queueMode === "worker") {
+			for (const type of Object.keys(this.inFlight) as JobType[]) {
+				this.startLoop(type);
+			}
+		}
+		return true;
+	};
+
+	override flushQueues = async () => {
+		await this.shutdown();
+		const ok = await this.init();
+		return { success: ok };
+	};
+}

server/src/worker/worker.helper.ts

@@ -1,5 +1,4 @@
 const SERVICE_NAME = "JobQueueHelper";
-import type { Monitor } from "@/domain/monitors/monitor.types.js";
 import { ISettingsService } from "@/domain/app-settings/app-settings.service.js";
 import { ICheckService } from "@/domain/checks/check.service.js";
 import { CHECK_TTL_SENTINEL } from "@/domain/checks/check.type.js";
@@ -8,15 +7,12 @@ import { IGeoChecksRepository } from "@/domain/geo-checks/geo-check.repository.i
 import { IIncidentsRepository } from "@/domain/incidents/incident.repository.interface.js";
 import { IMonitorStatsRepository } from "@/domain/monitor-stats/monitor-stats.repository.interface.js";
 import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
+import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
 import { ITeamsRepository } from "@/domain/teams/team.repository.interface.js";
 import { ILogger } from "@/utils/logger.js";
-import { IReactorDispatcher } from "@/worker/reactors/reactor.dispatcher.js";
-import { ICheckPipeline } from "@/worker/worker.check-pipeline.js";
 
 export interface IWorkerHelper {
 	readonly serviceName: string;
-	getHeartbeatJob(): (monitor: Monitor) => Promise<void>;
-	getHeartbeatGeoJob(): (monitor: Monitor) => Promise<void>;
 	getCleanupOrphanedJob(): () => Promise<void>;
 	getCleanupRetentionJob(): () => Promise<void>;
 }
@@ -42,80 +38,41 @@ export class WorkerHelper implements IWorkerHelper {
 	private checkService: ICheckService;
 	private settingsService: ISettingsService;
 	private monitorsRepository: IMonitorsRepository;
+	private jobsRepository: IJobsRepository;
 	private teamsRepository: ITeamsRepository;
 	private monitorStatsRepository: IMonitorStatsRepository;
 	private checksRepository: IChecksRepository;
 	private incidentsRepository: IIncidentsRepository;
 	private geoChecksRepository: IGeoChecksRepository;
-	private reactorDispatcher: IReactorDispatcher;
-	private checkPipeline: ICheckPipeline;
-	private geoCheckPipeline: ICheckPipeline;
 
 	constructor(
 		logger: ILogger,
 		checkService: ICheckService,
 		settingsService: ISettingsService,
 		monitorsRepository: IMonitorsRepository,
+		jobsRepository: IJobsRepository,
 		teamsRepository: ITeamsRepository,
 		monitorStatsRepository: IMonitorStatsRepository,
 		checksRepository: IChecksRepository,
 		incidentsRepository: IIncidentsRepository,
-		geoChecksRepository: IGeoChecksRepository,
-		reactorDispatcher: IReactorDispatcher,
-		checkPipeline: ICheckPipeline,
-		geoCheckPipeline: ICheckPipeline
+		geoChecksRepository: IGeoChecksRepository
 	) {
 		this.logger = logger;
 		this.checkService = checkService;
 		this.settingsService = settingsService;
 		this.monitorsRepository = monitorsRepository;
+		this.jobsRepository = jobsRepository;
 		this.teamsRepository = teamsRepository;
 		this.monitorStatsRepository = monitorStatsRepository;
 		this.checksRepository = checksRepository;
 		this.incidentsRepository = incidentsRepository;
 		this.geoChecksRepository = geoChecksRepository;
-		this.reactorDispatcher = reactorDispatcher;
-		this.checkPipeline = checkPipeline;
-		this.geoCheckPipeline = geoCheckPipeline;
 	}
 
 	get serviceName() {
 		return WorkerHelper.SERVICE_NAME;
 	}
 
-	getHeartbeatJob = () => {
-		return async (monitor: Monitor) => {
-			try {
-				const evaluation = await this.checkPipeline.run(monitor);
-				if (evaluation) await this.reactorDispatcher.dispatch(evaluation);
-			} catch (error: unknown) {
-				this.logger.warn({
-					message: error instanceof Error ? error.message : "Unknown error",
-					service: SERVICE_NAME,
-					method: "getMonitorJob",
-					stack: error instanceof Error ? error.stack : undefined,
-				});
-				throw error;
-			}
-		};
-	};
-
-	getHeartbeatGeoJob = () => {
-		return async (monitor: Monitor) => {
-			try {
-				await this.geoCheckPipeline.run(monitor);
-			} catch (error: unknown) {
-				this.logger.error({
-					message: error instanceof Error ? error.message : "Unknown error",
-					service: SERVICE_NAME,
-					method: "getHeartbeatGeoJob",
-					stack: error instanceof Error ? error.stack : undefined,
-				});
-				// Don't throw - geo check failures shouldn't crash the job scheduler
-			}
-		};
-	};
-
 	getCleanupOrphanedJob = () => {
 		return async () => {
 			try {
@@ -190,6 +147,16 @@ export class WorkerHelper implements IWorkerHelper {
 					});
 				}
 
+				// Remove orphaned jobs
+				const deletedJobsCount = await this.jobsRepository.deleteByMonitorIdsNotIn(allMonitorIds);
+				if (deletedJobsCount > 0) {
+					this.logger.info({
+						message: `Deleted ${deletedJobsCount} orphaned jobs`,
+						service: SERVICE_NAME,
+						method: "getCleanupOrphanedJob",
+					});
+				}
+
 				this.logger.info({
 					message: "Cleanup of orphaned data completed",
 					service: SERVICE_NAME,

server/src/worker/worker.interface.ts

@@ -1,8 +1,9 @@
 import { Monitor } from "@/domain/monitors/monitor.types.js";
-import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
 import { Check } from "@/domain/checks/check.type.js";
+import { JobType } from "@/domain/jobs/job.type.js";
 import { MonitorPayloadMap, MonitorStatusResponse, StatusChangeResult } from "@/types/network.js";
 import { MonitorActionDecision } from "@/worker/worker.helper.js";
+import { QueueWorker } from "@/domain/queue-workers/queue-worker.type.js";
 
 export type MonitorEvaluation = {
 	monitor: Monitor;
@@ -33,23 +34,16 @@ export type WorkerMetrics = {
 
 export type WorkerJobSummary = {
 	monitorId: string | number;
-	monitorUrl: string | null;
 	monitorType: string | null;
 	monitorInterval: number | null;
-	monitorGeoInterval: number | null;
 	monitorActive: boolean | null;
-	active: boolean;
 	lockedBy: string | null;
 	lockedUntil: number | null;
-	lockedAt: number | null;
+	nextScheduledAt: number;
 	runCount: number;
 	failCount: number;
 	failReason: string | null;
-	lastRunAt: number | null;
 	lastFinishedAt: number | null;
-	lastRunTook: number | null;
-	lastFailedAt: number | null;
-	repeat: number | null;
 };
 
 export type WorkerJobsPagination = {
@@ -62,16 +56,21 @@ export type WorkerJobsPage = {
 	count: number;
 };
 
-export interface IWorker {
+export interface IJobScheduler {
 	readonly serviceName: string;
-	init(): Promise<boolean>;
 	addJob(monitorId: string, monitor: Monitor): Promise<void>;
 	deleteJob(monitor: Monitor): Promise<void>;
 	pauseJob(monitor: Monitor): Promise<void>;
 	resumeJob(monitor: Monitor): Promise<void>;
 	updateJob(monitor: Monitor): Promise<void>;
-	shutdown(): Promise<void>;
+	wake(type: JobType): void;
 	getMetrics(): Promise<WorkerMetrics>;
 	getJobs(pagination: WorkerJobsPagination): Promise<WorkerJobsPage>;
 	flushQueues(): Promise<{ success: boolean }>;
+	shutdown(): Promise<void>;
+}
+
+export interface IQueueWorker extends IJobScheduler {
+	readonly serviceName: string;
+	init(): Promise<boolean>;
 }

server/src/worker/worker.job-scheduler.ts

@@ -0,0 +1,167 @@
+import { IJobScheduler } from "@/worker/worker.interface.js";
+import { IJobsRepository } from "@/domain/jobs/job.repository.interface.js";
+import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
+import { type Job, type JobSeed, type JobType, jobId } from "@/domain/jobs/job.type.js";
+import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
+import { WorkerJobsPagination, WorkerJobSummary, WorkerMetrics } from "@/worker/worker.interface.js";
+import { WORKER_TTL_SECONDS } from "@/domain/queue-workers/queue-worker.model.js";
+
+const POLL_MS = 250; // base poll interval while a loop is actively claiming work
+const SERVICE_NAME = "JobScheduler";
+const WORKER_STALE_MS = WORKER_TTL_SECONDS * 1000; // a worker counts as alive if seen within this window
+
+export class JobScheduler implements IJobScheduler {
+	static SERVICE_NAME = SERVICE_NAME;
+	constructor(
+		protected jobsRepository: IJobsRepository,
+		protected queueWorkersRepository: IQueueWorkersRepository,
+		protected readonly workerId: string // same id MongoJobsRepository stamps into lockedBy
+	) {}
+
+	get serviceName() {
+		return JobScheduler.SERVICE_NAME;
+	}
+
+	protected stopped = false;
+	protected ticking: Partial<Record<JobType, boolean>> = {}; // Whether a loop is mid tick
+	protected tickFns = new Map<JobType, () => void>(); // Store tick fns so we can run them immediately on wake
+	protected timers = new Map<JobType | "heartbeat", NodeJS.Timeout>();
+
+	// Per job type poll intervals.  These back off on idle and reset on work or wake()
+	protected pollMs: Record<JobType, number> = {
+		check: POLL_MS,
+		"geo-check": POLL_MS,
+		evaluate: POLL_MS,
+		"cleanup-orphaned": POLL_MS,
+		"cleanup-retention": POLL_MS,
+	};
+	// Helpers, builds job seeds. immediate=true runs on the next tick; otherwise jitter spreads the herd.
+	protected toCheckJob = (monitor: Monitor, now: number, immediate = false): JobSeed => ({
+		id: jobId("check", monitor.id),
+		type: "check",
+		refId: monitor.id,
+		isActive: monitor.isActive,
+		nextScheduledAt: immediate ? now : now + Math.floor(Math.random() * monitor.interval), // Avoid herd
+		intervalMs: monitor.interval,
+	});
+
+	protected toGeoCheckJob = (monitor: Monitor, now: number, immediate = false): JobSeed => ({
+		...this.toCheckJob(monitor, now, immediate),
+		id: jobId("geo-check", monitor.id),
+		type: "geo-check",
+		intervalMs: monitor.geoCheckInterval!,
+	});
+
+	private toSummary = (job: Job): WorkerJobSummary => ({
+		monitorId: job.refId ?? job.id,
+		monitorType: job.type,
+		monitorInterval: job.intervalMs,
+		monitorActive: job.isActive,
+		lockedBy: job.lockedBy,
+		lockedUntil: job.lockedUntil,
+		nextScheduledAt: job.nextScheduledAt,
+		runCount: job.runCount,
+		failCount: job.failCount,
+		failReason: job.lastFailReason,
+		lastFinishedAt: job.lastFinishedAt,
+	});
+	wake = (type: JobType) => {
+		this.pollMs[type] = POLL_MS;
+		if (this.stopped || this.ticking[type]) return; // If it's not running or mid tick, do nothing
+		const tick = this.tickFns.get(type);
+		if (!tick) return; // Try to get a handle on the tick fn, if we can't we're done
+		const pending = this.timers.get(type);
+		if (pending) clearTimeout(pending); // Clear out scheduled tick if any
+		this.timers.set(type, setTimeout(tick, 0)); // Set immediate tick
+	};
+
+	addJob = async (_monitorId: string, monitor: Monitor) => {
+		const now = Date.now();
+		await this.jobsRepository.upsertJob(this.toCheckJob(monitor, now, true));
+		this.wake("check");
+		if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
+			await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, now, true));
+			this.wake("geo-check");
+		}
+	};
+
+	deleteJob = async (monitor: Monitor) => {
+		await this.jobsRepository.deleteById(monitor.id);
+	};
+
+	pauseJob = async (monitor: Monitor) => {
+		await this.jobsRepository.setActiveById(monitor.id, false);
+	};
+
+	resumeJob = async (monitor: Monitor) => {
+		await this.jobsRepository.setActiveById(monitor.id, true); // a paused job's nextScheduledAt is now in the past, so it's due immediately
+		this.wake("check");
+		if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) this.wake("geo-check");
+	};
+
+	updateJob = async (monitor: Monitor) => {
+		await this.jobsRepository.updateScheduleById(monitor.id, "check", monitor.interval);
+		if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
+			await this.jobsRepository.upsertJob(this.toGeoCheckJob(monitor, Date.now(), true)); // a newly enabled geo check should run right away
+			this.wake("geo-check");
+		} else {
+			await this.jobsRepository.deleteByIdAndType(monitor.id, "geo-check");
+		}
+	};
+
+	getMetrics = async (): Promise<WorkerMetrics> => {
+		const rows = await this.jobsRepository.findAll();
+		const workers = await this.queueWorkersRepository.findRecent(WORKER_STALE_MS);
+		const now = Date.now();
+		return rows.reduce<WorkerMetrics>(
+			(acc, job) => {
+				acc.jobs++;
+				acc.totalRuns += job.runCount;
+				acc.totalFailures += job.failCount;
+				if (job.lockedBy !== null && job.lockedUntil !== null && job.lockedUntil > now) acc.activeJobs++;
+				if (job.failCount > 0) {
+					acc.failingJobs++;
+					acc.jobsWithFailures.push({
+						monitorId: job.refId ?? job.id,
+						monitorUrl: null,
+						monitorType: job.type,
+						failedAt: job.lastFinishedAt,
+						failCount: job.failCount,
+						failReason: job.lastFailReason,
+					});
+				}
+				return acc;
+			},
+			{
+				jobs: 0,
+				activeJobs: 0,
+				failingJobs: 0,
+				jobsWithFailures: [],
+				totalRuns: 0,
+				totalFailures: 0,
+				workers,
+			}
+		);
+	};
+
+	getJobs = async (pagination: WorkerJobsPagination) => {
+		const { jobs, count } = await this.jobsRepository.findPage(pagination);
+		return { jobs: jobs.map(this.toSummary), count };
+	};
+
+	flushQueues = async () => {
+		await this.shutdown();
+		return { success: true };
+	};
+
+	shutdown = async () => {
+		this.stopped = true;
+		// Stop all timers
+		for (const timer of this.timers.values()) {
+			clearTimeout(timer);
+		}
+		this.timers.clear(); // Clear the map out
+		//  Remove workers from registry
+		await this.queueWorkersRepository.deleteById(this.workerId);
+	};
+}

server/src/worker/worker.less-simple.ts

@@ -1,427 +0,0 @@
-import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
-import { IQueueWorkersRepository } from "@/domain/queue-workers/queue-worker.repository.interface.js";
-import { ILogger } from "@/utils/logger.js";
-import { MongoStore, Scheduler, IJob, AddJobInput } from "less-simple-scheduler";
-import { Monitor, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
-import { IWorkerHelper } from "@/worker/worker.helper.js";
-import { type WorkerMetrics, type WorkerJobsPagination, type WorkerJobsPage, IWorker } from "@/worker/worker.interface.js";
-import { type QueueMode } from "@/domain/app-settings/app-settings.type.js";
-import { EnvConfig } from "@/domain/app-settings/app-settings.service.js";
-const SERVICE_NAME = "JobQueue";
-
-// How long after its last heartbeat a worker is still considered alive.  Must be longer than the scheduler's heartbeat
-const WORKER_TTL_MS = 30_000;
-
-// For discriminating job types
-type MonitorJob = Omit<IJob, "data"> & { data: Monitor };
-const MONITOR_TEMPLATES = new Set<string>(["monitor-job", "geo-check-job"]);
-const isMonitorJob = (job: IJob): job is MonitorJob => MONITOR_TEMPLATES.has(job.template);
-
-export class LessSimpleWorker implements IWorker {
-	static SERVICE_NAME = SERVICE_NAME;
-
-	private logger: ILogger;
-	private helper: IWorkerHelper;
-	private monitorsRepository: IMonitorsRepository;
-	private workersRepository: IQueueWorkersRepository;
-	private readonly scheduler: Scheduler;
-	private queueMode: QueueMode;
-	private heartbeatListener: ((workerId: string) => void) | null = null;
-
-	constructor(
-		logger: ILogger,
-		helper: IWorkerHelper,
-		monitorsRepository: IMonitorsRepository,
-		workersRepository: IQueueWorkersRepository,
-		scheduler: Scheduler,
-		queueMode: QueueMode
-	) {
-		this.logger = logger;
-		this.helper = helper;
-		this.monitorsRepository = monitorsRepository;
-		this.workersRepository = workersRepository;
-		this.scheduler = scheduler;
-		this.queueMode = queueMode;
-		this.registerListeners();
-	}
-
-	get serviceName() {
-		return LessSimpleWorker.SERVICE_NAME;
-	}
-
-	private registerListeners = () => {
-		this.scheduler.on("scheduler:start", () => {
-			this.logger.info({
-				message: "Scheduler started",
-				service: SERVICE_NAME,
-			});
-		});
-
-		this.scheduler.on("scheduler:stop", () => {
-			this.logger.info({
-				message: "Scheduler stopped",
-				service: SERVICE_NAME,
-			});
-		});
-
-		this.scheduler.on("scheduler:error", (workerId, error) => {
-			this.logger.error({
-				message: `${workerId} error: ${error instanceof Error ? error.message : String(error)}`,
-				service: SERVICE_NAME,
-				stack: error instanceof Error ? error.stack : undefined,
-			});
-		});
-
-		this.scheduler.on("job:locked", (workerId, job) => {
-			this.logger.debug({
-				message: `${workerId} ${job.id} locked`,
-				service: SERVICE_NAME,
-			});
-		});
-
-		this.scheduler.on("job:abort", (workerId, job, reason) => {
-			this.logger.warn({
-				message: `${workerId} ${job.id} aborted: ${reason}`,
-				service: SERVICE_NAME,
-			});
-		});
-
-		this.scheduler.on("job:attempt", (workerId, job, attempt) => {
-			this.logger.debug({
-				message: `${workerId} ${job.id} attempt ${attempt}`,
-				service: SERVICE_NAME,
-			});
-		});
-
-		this.scheduler.on("job:complete", (workerId, job) => {
-			this.logger.debug({
-				message: `${workerId} ${job.id} completed successfully`,
-				service: SERVICE_NAME,
-			});
-		});
-
-		this.scheduler.on("job:exhausted", (workerId, job, error) => {
-			this.logger.error({
-				message: `${workerId} ${job.id} exhausted all retries: ${error instanceof Error ? error.message : String(error)}`,
-				service: SERVICE_NAME,
-				stack: error instanceof Error ? error.stack : undefined,
-			});
-		});
-
-		this.scheduler.on("job:fail", (workerId, job, error, attempt) => {
-			this.logger.warn({
-				message: `${workerId} ${job.id} failed on attempt ${attempt}: ${error instanceof Error ? error.message : String(error)}`,
-				service: SERVICE_NAME,
-				stack: error instanceof Error ? error.stack : undefined,
-			});
-		});
-
-		this.scheduler.on("job:start", (workerId, job) => {
-			this.logger.debug({
-				message: `${workerId} ${job.id} started`,
-				service: SERVICE_NAME,
-			});
-		});
-	};
-
-	static async create(
-		logger: ILogger,
-		helper: IWorkerHelper,
-		monitorsRepository: IMonitorsRepository,
-		workersRepository: IQueueWorkersRepository,
-		envSettings: EnvConfig,
-		queueMode: QueueMode
-	) {
-		const store = new MongoStore({
-			url: envSettings.dbConnectionString ?? "mongodb://localhost:27017/uptime_db",
-		});
-		const scheduler = new Scheduler(store, {
-			concurrency: envSettings.queueMode === "primary" ? 0 : 50,
-			processEvery: 100,
-		});
-		const instance = new LessSimpleWorker(logger, helper, monitorsRepository, workersRepository, scheduler, queueMode);
-		await instance.init(queueMode);
-
-		return instance;
-	}
-
-	private startHeartbeat = (queueMode: QueueMode) => {
-		const beat = (
-			workerId: string // This is the heartbeat function that registers workers
-		) =>
-			this.workersRepository.upsert(workerId, queueMode).catch((error: unknown) => {
-				this.logger.warn({
-					message: `Worker heartbeat failed: ${error instanceof Error ? error.message : String(error)}`,
-					service: SERVICE_NAME,
-					method: "startHeartbeat",
-				});
-			});
-		beat(this.scheduler.workerId); // Beat immediately to register
-		if (this.heartbeatListener) {
-			this.scheduler.off("scheduler:heartbeat", this.heartbeatListener); // remove old listeners
-		}
-		this.heartbeatListener = beat; /// Keep a reference to the listener
-		this.scheduler.on("scheduler:heartbeat", beat); // beat on every heartbeat
-	};
-
-	init = async (queueMode: QueueMode = "primary") => {
-		try {
-			await this.scheduler.start();
-			this.scheduler.addTemplate("monitor-job", this.helper.getHeartbeatJob());
-			this.scheduler.addTemplate("geo-check-job", this.helper.getHeartbeatGeoJob());
-			this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob());
-			this.scheduler.addTemplate("cleanup-retention-job", this.helper.getCleanupRetentionJob());
-
-			// Register in the worker registry
-			this.startHeartbeat(queueMode);
-
-			// If this is a worker, return early - primary will populate queue
-			if (queueMode === "worker") {
-				this.logger.info({
-					message: `Worker queue: ${this.scheduler.workerId} initialized`,
-					service: SERVICE_NAME,
-					method: "init",
-				});
-				return true;
-			}
-
-			const monitors = await this.monitorsRepository.findAll();
-			if (!monitors) {
-				return true;
-			}
-			const jobInputs = monitors.flatMap((monitor) => this.buildJobInputs(monitor));
-			if (jobInputs.length > 0) {
-				await this.scheduler.addJobs(jobInputs);
-			}
-
-			this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true, upsert: true });
-			this.scheduler.addJob({ id: "cleanup-retention", template: "cleanup-retention-job", active: true, repeat: 24 * 60 * 60 * 1000, upsert: true });
-
-			return true;
-		} catch (error: unknown) {
-			this.logger.error({
-				message: `Failed to initialize LessSimpleQueue: ${error instanceof Error ? error.message : String(error)}`,
-				service: SERVICE_NAME,
-				method: "init",
-			});
-			return false;
-		}
-	};
-
-	// Builds the scheduler inputs for a monitor: always a monitor-job, plus a
-	// geo-check job when the monitor supports and enables geo checks.
-	private buildJobInputs = (monitor: Monitor): AddJobInput[] => {
-		const inputs: AddJobInput[] = [
-			{
-				id: monitor.id,
-				template: "monitor-job",
-				repeat: monitor.interval,
-				active: monitor.isActive,
-				data: monitor,
-				upsert: true,
-				jitter: true,
-			},
-		];
-
-		if (supportsGeoCheck(monitor.type) && monitor.geoCheckEnabled) {
-			inputs.push({
-				id: `${monitor.id}-geo`,
-				template: "geo-check-job",
-				repeat: monitor.geoCheckInterval,
-				active: monitor.isActive,
-				data: monitor,
-				upsert: true,
-				jitter: true,
-			});
-		}
-
-		return inputs;
-	};
-
-	addJob = async (_monitorId: string, monitor: Monitor) => {
-		await this.scheduler.addJobs(this.buildJobInputs(monitor));
-	};
-
-	deleteJob = async (monitor: Monitor) => {
-		this.scheduler.removeJob(monitor.id);
-		const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
-		if (geoJob) await this.scheduler.removeJob(`${monitor.id}-geo`);
-	};
-
-	pauseJob = async (monitor: Monitor) => {
-		const result = await this.scheduler.pauseJob(monitor.id);
-		if (result === false) {
-			throw new Error("Failed to pause monitor");
-		}
-		this.scheduler.updateJob(monitor.id, { data: monitor });
-
-		const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
-		if (geoJob) {
-			const geoRes = await this.scheduler.pauseJob(`${monitor.id}-geo`);
-			if (geoRes === false) {
-				this.logger.error({
-					message: `Failed to pause geo check job for monitor ${monitor.id}`,
-					service: SERVICE_NAME,
-					method: "pauseJob",
-				});
-			}
-			this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
-		}
-
-		this.logger.debug({
-			message: `Paused monitor ${monitor.id}`,
-			service: SERVICE_NAME,
-			method: "pauseJob",
-		});
-	};
-
-	resumeJob = async (monitor: Monitor) => {
-		const result = await this.scheduler.resumeJob(monitor.id);
-		if (result === false) {
-			throw new Error("Failed to resume monitor");
-		}
-		this.scheduler.updateJob(monitor.id, { data: monitor });
-
-		const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
-		if (geoJob) {
-			const geoRes = await this.scheduler.resumeJob(`${monitor.id}-geo`);
-			if (geoRes === false) {
-				this.logger.error({
-					message: `Failed to resume geo check job for monitor ${monitor.id}`,
-					service: SERVICE_NAME,
-					method: "resumeJob",
-				});
-			}
-			this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
-		}
-		this.logger.debug({
-			message: `Resumed monitor ${monitor.id}`,
-			service: SERVICE_NAME,
-			method: "resumeJob",
-		});
-	};
-
-	private syncGeoJob = async (monitor: Monitor) => {
-		const geoJobId = `${monitor.id}-geo`;
-		const existingGeoJob = await this.scheduler.getJob(geoJobId);
-
-		// If geoChecks have been disabled, or the monitor type doesn't support them, remove
-		if (!monitor.geoCheckEnabled || !supportsGeoCheck(monitor.type)) {
-			if (existingGeoJob) this.scheduler.removeJob(geoJobId);
-			return;
-		}
-
-		// If the job exists, update it
-		if (existingGeoJob) {
-			this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor });
-			return;
-		}
-
-		// Otherwise, create it
-		this.scheduler.addJob({
-			id: geoJobId,
-			template: "geo-check-job",
-			repeat: monitor.geoCheckInterval,
-			active: monitor.isActive,
-			data: monitor,
-			upsert: true,
-		});
-	};
-
-	updateJob = async (monitor: Monitor) => {
-		this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor });
-		await this.syncGeoJob(monitor);
-	};
-
-	shutdown = async () => {
-		// Stop beating before we remove this worker, so a heartbeat firing between
-		// the remove and the scheduler stopping can't re-insert it.
-		if (this.heartbeatListener) {
-			this.scheduler.off("scheduler:heartbeat", this.heartbeatListener);
-			this.heartbeatListener = null;
-		}
-		await this.workersRepository.deleteById(this.scheduler.workerId).catch((error: unknown) => {
-			this.logger.warn({
-				message: `Failed to deregister worker on shutdown: ${error instanceof Error ? error.message : String(error)}`,
-				service: SERVICE_NAME,
-				method: "shutdown",
-			});
-		});
-		this.scheduler.stop();
-	};
-
-	getMetrics = async (): Promise<WorkerMetrics> => {
-		const stats = await this.scheduler.getStats();
-
-		const jobsWithFailures = stats.jobsWithFailures.map((failure) => {
-			// `data` is untyped at the store level: for monitor jobs it's the Monitor,
-			// for system jobs (e.g. cleanup) it has no url/type => null
-			const monitor = failure.data as Monitor | undefined;
-			return {
-				monitorId: failure.id,
-				monitorUrl: monitor?.url ?? null,
-				monitorType: monitor?.type ?? null,
-				failedAt: failure.failedAt,
-				failCount: failure.failCount,
-				failReason: failure.failReason,
-			};
-		});
-
-		const workers = await this.workersRepository.findRecent(WORKER_TTL_MS);
-		return {
-			jobs: stats.jobs,
-			activeJobs: stats.activeJobs,
-			failingJobs: stats.failingJobs,
-			totalRuns: stats.totalRuns,
-			totalFailures: stats.totalFailures,
-			jobsWithFailures,
-			workers,
-		};
-	};
-
-	getJobs = async ({ page = 0, rowsPerPage = 0 }: WorkerJobsPagination): Promise<WorkerJobsPage> => {
-		const listOptions = rowsPerPage > 0 ? { skip: Math.max(page, 0) * rowsPerPage, limit: rowsPerPage } : undefined;
-		const [jobs, count] = await Promise.all([this.scheduler.getJobs(listOptions), this.scheduler.countJobs()]);
-		const mapped = jobs.map((job) => {
-			const monitor = isMonitorJob(job) ? job.data : undefined;
-
-			return {
-				monitorId: job.id,
-				monitorUrl: monitor?.url || null,
-				monitorType: monitor?.type || null,
-				monitorInterval: monitor?.interval || null,
-				monitorGeoInterval: monitor?.geoCheckInterval || null,
-				monitorActive: monitor?.isActive ?? null,
-				active: job.active,
-				repeat: job.repeat ?? null,
-				lockedBy: job.lockedBy ?? null,
-				lockedUntil: job.lockedUntil ?? null,
-				lockedAt: job.lockedAt ?? null,
-				runCount: job.runCount ?? 0,
-				failCount: job.failCount ?? 0,
-				failReason: job.lastError ?? null,
-				lastRunAt: job.lastStartedAt ?? null,
-				lastFinishedAt: job.lastFinishedAt ?? null,
-				lastRunTook: job.lockedAt || !job.lastFinishedAt || !job.lastStartedAt ? null : job.lastFinishedAt - job.lastStartedAt,
-				lastFailedAt: job.lastFailedAt ?? null,
-			};
-		});
-
-		return { jobs: mapped, count };
-	};
-
-	flushQueues = async () => {
-		if (this.queueMode !== "primary") {
-			// Don't allow workers to flush, though this isn't currently reachable since workers have no API endpoints
-			this.logger.warn({ message: "Ignoring flush on non-primary worker", service: SERVICE_NAME, method: "flushQueues" });
-			return { success: false };
-		}
-
-		const flushRes = await this.scheduler.flushJobs(); // Less Simple Queue must flush before stopping as it needs DB
-		const stopRes = await this.scheduler.stop(); // Destructive, other workers will abort their jobs
-		const initRes = await this.init();
-		return {
-			success: Boolean(stopRes && flushRes && initRes),
-		};
-	};
-}

server/src/worker/worker.super-simple.ts

@@ -1,384 +0,0 @@
-import { IMonitorsRepository } from "@/domain/monitors/monitor.repository.interface.js";
-import { ILogger } from "@/utils/logger.js";
-import Scheduler from "super-simple-scheduler";
-import { Monitor, MonitorType, supportsGeoCheck } from "@/domain/monitors/monitor.types.js";
-import { hostname } from "node:os";
-import { randomUUID } from "node:crypto";
-import { QueueMode } from "@/domain/app-settings/app-settings.type.js";
-import { IWorker, WorkerJobSummary } from "@/worker/worker.interface.js";
-import { IWorkerHelper } from "@/worker/worker.helper.js";
-
-const SERVICE_NAME = "JobQueue";
-
-export class SuperSimpleQueue implements IWorker {
-	static SERVICE_NAME = SERVICE_NAME;
-
-	private logger: ILogger;
-	private helper: IWorkerHelper;
-	private workerId: string;
-	private monitorsRepository: IMonitorsRepository;
-	private readonly scheduler: Scheduler;
-
-	constructor(logger: ILogger, helper: IWorkerHelper, monitorsRepository: IMonitorsRepository, scheduler: Scheduler) {
-		this.logger = logger;
-		this.helper = helper;
-		this.monitorsRepository = monitorsRepository;
-		this.scheduler = scheduler;
-		this.registerListeners();
-		this.workerId = `${hostname()}:${process.pid}:${randomUUID()}`;
-	}
-
-	get serviceName() {
-		return SuperSimpleQueue.SERVICE_NAME;
-	}
-
-	private registerListeners = () => {
-		this.scheduler.on("scheduler:start", () => {
-			this.logger.info({
-				message: "Scheduler started",
-				service: SERVICE_NAME,
-			});
-		});
-
-		this.scheduler.on("scheduler:stop", () => {
-			this.logger.info({
-				message: "Scheduler stopped",
-				service: SERVICE_NAME,
-			});
-		});
-
-		this.scheduler.on("scheduler:error", (error) => {
-			this.logger.error({
-				message: `Scheduler error: ${error instanceof Error ? error.message : String(error)}`,
-				service: SERVICE_NAME,
-				stack: error instanceof Error ? error.stack : undefined,
-			});
-		});
-
-		this.scheduler.on("job:abort", (job, reason) => {
-			this.logger.warn({
-				message: `${job.id} aborted: ${reason}`,
-				service: SERVICE_NAME,
-			});
-		});
-
-		this.scheduler.on("job:attempt", (job, attempt) => {
-			this.logger.debug({
-				message: `${job.id} attempt ${attempt}`,
-				service: SERVICE_NAME,
-			});
-		});
-
-		this.scheduler.on("job:complete", (job) => {
-			this.logger.debug({
-				message: `${job.id} completed successfully`,
-				service: SERVICE_NAME,
-			});
-		});
-
-		this.scheduler.on("job:exhausted", (job, error) => {
-			this.logger.error({
-				message: `${job.id} exhausted all retries: ${error instanceof Error ? error.message : String(error)}`,
-				service: SERVICE_NAME,
-				stack: error instanceof Error ? error.stack : undefined,
-			});
-		});
-
-		this.scheduler.on("job:fail", (job, error, attempt) => {
-			this.logger.warn({
-				message: `${job.id} failed on attempt ${attempt}: ${error instanceof Error ? error.message : String(error)}`,
-				service: SERVICE_NAME,
-				stack: error instanceof Error ? error.stack : undefined,
-			});
-		});
-
-		this.scheduler.on("job:start", (job) => {
-			this.logger.debug({
-				message: `${job.id} started`,
-				service: SERVICE_NAME,
-			});
-		});
-	};
-
-	static async create(logger: ILogger, helper: IWorkerHelper, monitorsRepository: IMonitorsRepository) {
-		const scheduler = new Scheduler({
-			concurrency: 50,
-			// storeType: "mongo",
-			// storeType: "redis",
-			// dbUri: envSettings.dbConnectionString,
-		});
-		const instance = new SuperSimpleQueue(logger, helper, monitorsRepository, scheduler);
-		await instance.init();
-
-		return instance;
-	}
-
-	init = async () => {
-		try {
-			this.scheduler.start();
-			this.scheduler.addTemplate("monitor-job", this.helper.getHeartbeatJob());
-			this.scheduler.addTemplate("geo-check-job", this.helper.getHeartbeatGeoJob());
-			this.scheduler.addTemplate("cleanup-orphaned", this.helper.getCleanupOrphanedJob());
-			this.scheduler.addTemplate("cleanup-retention-job", this.helper.getCleanupRetentionJob());
-			const monitors = await this.monitorsRepository.findAll();
-			if (!monitors) {
-				return true;
-			}
-			for (const monitor of monitors) {
-				const randomOffset = Math.floor(Math.random() * 100);
-				setTimeout(() => {
-					this.addJob(monitor.id, monitor);
-				}, randomOffset);
-			}
-
-			this.scheduler.addJob({ id: "cleanup-orphaned", template: "cleanup-orphaned", active: true });
-			this.scheduler.addJob({ id: "cleanup-retention", template: "cleanup-retention-job", active: true, repeat: 24 * 60 * 60 * 1000 });
-
-			return true;
-		} catch (error: unknown) {
-			this.logger.error({
-				message: `Failed to initialize SuperSimpleQueue: ${error instanceof Error ? error.message : String(error)}`,
-				service: SERVICE_NAME,
-				method: "init",
-			});
-			return false;
-		}
-	};
-
-	addJob = async (monitorId: string, monitor: Monitor) => {
-		this.scheduler.addJob({
-			id: monitorId,
-			template: "monitor-job",
-			repeat: monitor.interval,
-			active: monitor.isActive,
-			data: monitor,
-		});
-
-		// Return early if we don't need geo checks
-		if (!supportsGeoCheck(monitor.type)) {
-			return;
-		}
-
-		// Add geo check job if enabled for HTTP monitors
-		if (monitor.geoCheckEnabled) {
-			this.scheduler.addJob({
-				id: `${monitorId}-geo`,
-				template: "geo-check-job",
-				repeat: monitor.geoCheckInterval,
-				active: monitor.isActive,
-				data: monitor,
-			});
-		}
-	};
-
-	deleteJob = async (monitor: Monitor) => {
-		this.scheduler.removeJob(monitor.id);
-		const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
-		if (geoJob) await this.scheduler.removeJob(`${monitor.id}-geo`);
-	};
-
-	pauseJob = async (monitor: Monitor) => {
-		const result = await this.scheduler.pauseJob(monitor.id);
-		if (result === false) {
-			throw new Error("Failed to pause monitor");
-		}
-		this.scheduler.updateJob(monitor.id, { data: monitor });
-
-		const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
-		if (geoJob) {
-			const geoRes = await this.scheduler.pauseJob(`${monitor.id}-geo`);
-			if (geoRes === false) {
-				this.logger.error({
-					message: `Failed to pause geo check job for monitor ${monitor.id}`,
-					service: SERVICE_NAME,
-					method: "pauseJob",
-				});
-			}
-			this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
-		}
-
-		this.logger.debug({
-			message: `Paused monitor ${monitor.id}`,
-			service: SERVICE_NAME,
-			method: "pauseJob",
-		});
-	};
-
-	resumeJob = async (monitor: Monitor) => {
-		const result = await this.scheduler.resumeJob(monitor.id);
-		if (result === false) {
-			throw new Error("Failed to resume monitor");
-		}
-		this.scheduler.updateJob(monitor.id, { data: monitor });
-
-		const geoJob = await this.scheduler.getJob(`${monitor.id}-geo`);
-		if (geoJob) {
-			const geoRes = await this.scheduler.resumeJob(`${monitor.id}-geo`);
-			if (geoRes === false) {
-				this.logger.error({
-					message: `Failed to resume geo check job for monitor ${monitor.id}`,
-					service: SERVICE_NAME,
-					method: "resumeJob",
-				});
-			}
-			this.scheduler.updateJob(`${monitor.id}-geo`, { data: monitor });
-		}
-		this.logger.debug({
-			message: `Resumed monitor ${monitor.id}`,
-			service: SERVICE_NAME,
-			method: "resumeJob",
-		});
-	};
-
-	private syncGeoJob = async (monitor: Monitor) => {
-		const geoJobId = `${monitor.id}-geo`;
-		const existingGeoJob = await this.scheduler.getJob(geoJobId);
-
-		// If geoChecks have been disabled, or the monitor type doesn't support them, remove
-		if (!monitor.geoCheckEnabled || !supportsGeoCheck(monitor.type)) {
-			if (existingGeoJob) this.scheduler.removeJob(geoJobId);
-			return;
-		}
-
-		// If the job exists, update it
-		if (existingGeoJob) {
-			this.scheduler.updateJob(geoJobId, { repeat: monitor.geoCheckInterval, active: monitor.isActive, data: monitor });
-			return;
-		}
-
-		// Otherwise, create it
-		this.scheduler.addJob({
-			id: geoJobId,
-			template: "geo-check-job",
-			repeat: monitor.geoCheckInterval,
-			active: monitor.isActive,
-			data: monitor,
-		});
-	};
-
-	updateJob = async (monitor: Monitor) => {
-		this.scheduler.updateJob(monitor.id, { repeat: monitor.interval, data: monitor });
-		await this.syncGeoJob(monitor);
-	};
-
-	shutdown = async () => {
-		this.scheduler.stop();
-	};
-
-	getMetrics = async () => {
-		const jobs = await this.scheduler.getJobs();
-		const metrics: {
-			jobs: number;
-			activeJobs: number;
-			failingJobs: number;
-			jobsWithFailures: Array<{
-				monitorId: string | number;
-				monitorUrl: string;
-				monitorType: MonitorType;
-				failedAt: number | null;
-				failCount: number;
-				failReason: string | null;
-			}>;
-			totalRuns: number;
-			totalFailures: number;
-			workers: Array<{
-				workerId: string;
-				mode: QueueMode;
-				lastSeenAt: number;
-			}>;
-		} = jobs.reduce(
-			(acc, job) => {
-				const runCount = job.runCount ?? 0;
-				const failCount = job.failCount ?? 0;
-				const lastFailedAt = job.lastFailedAt ?? 0;
-				const lastRunAt = job.lastRunAt ?? 0;
-				acc.totalRuns += runCount;
-				acc.totalFailures += failCount;
-				acc.jobs++;
-				if (failCount > 0 && lastFailedAt >= lastRunAt) {
-					acc.failingJobs++;
-				}
-
-				if (job.lockedAt) {
-					acc.activeJobs++;
-				}
-
-				if (failCount > 0) {
-					acc.jobsWithFailures.push({
-						monitorId: job.id,
-						monitorUrl: job.data?.url || null,
-						monitorType: job.data?.type || null,
-						failedAt: job.lastFailedAt ?? null,
-						failCount,
-						failReason: job.lastFailReason ?? null,
-					});
-				}
-				return acc;
-			},
-			{
-				jobs: 0,
-				activeJobs: 0,
-				failingJobs: 0,
-				jobsWithFailures: [] as Array<{
-					monitorId: string | number;
-					monitorUrl: string;
-					monitorType: MonitorType;
-					failedAt: number | null;
-					failCount: number;
-					failReason: string | null;
-				}>,
-				totalRuns: 0,
-				totalFailures: 0,
-				workers: [
-					{
-						workerId: this.workerId,
-						mode: "primary",
-						lastSeenAt: Date.now(),
-					},
-				],
-			}
-		);
-		return metrics;
-	};
-
-	getJobs = async ({ page = 0, rowsPerPage = 0 }: { page?: number; rowsPerPage?: number }): Promise<{ jobs: WorkerJobSummary[]; count: number }> => {
-		const jobs = await this.scheduler.getJobs();
-		const mapped = jobs.map((job) => {
-			return {
-				monitorId: job.id,
-				monitorUrl: job.data?.url || null,
-				monitorType: job.data?.type || null,
-				monitorInterval: job.data?.interval || null,
-				monitorGeoInterval: job.data?.geoCheckInterval || null,
-				monitorActive: job.data?.isActive ?? null,
-				active: job.active,
-				repeat: job.repeat ?? null,
-				lockedBy: job.lockedAt ? this.workerId : null,
-				lockedUntil: null,
-				lockedAt: job.lockedAt ?? null,
-				runCount: job.runCount ?? 0,
-				failCount: job.failCount ?? 0,
-				failReason: job.lastFailReason ?? null,
-				lastRunAt: job.lastRunAt ?? null,
-				lastFinishedAt: job.lastFinishedAt ?? null,
-				lastRunTook: job.lockedAt ? null : (job.lastFinishedAt ?? 0) - (job.lastRunAt ?? 0),
-				lastFailedAt: job.lastFailedAt ?? null,
-			};
-		});
-
-		const start = Math.max(page, 0) * rowsPerPage;
-		return {
-			jobs: rowsPerPage > 0 ? mapped.slice(start, start + rowsPerPage) : mapped,
-			count: mapped.length,
-		};
-	};
-
-	flushQueues = async () => {
-		const stopRes = await this.scheduler.stop();
-		const flushRes = await this.scheduler.flushJobs();
-		const initRes = await this.init();
-		return {
-			success: Boolean(stopRes && flushRes && initRes),
-		};
-	};
-}