Merge pull request #3728 from bluewave-labs/feat/db-queue

feat: db queue
This commit is contained in:
Alexander Holliday authored and GitHub committed 2026-06-17 09:48:40 -07:00
commit 8277c4ed13
42 files changed
+3499 -3514

No files matched your search

@@ -1,5 +1,5 @@
import { ISettingsRepository } from "@/domain/app-settings/app-settings-repository.interface.js";
import { Settings, SettingsUpdate, type DbType, type QueueMode, type QueueType } from "@/domain/app-settings/app-settings.type.js";
import { Settings, SettingsUpdate, type DbType, type QueueMode } from "@/domain/app-settings/app-settings.type.js";
import { AppError } from "@/utils/AppError.js";
import { ValidatedEnv } from "@/config/envValidation.js";
import type { StringValue } from "ms";
@@ -13,8 +13,8 @@ export type EnvConfig = {
clientHost: string;
dbConnectionString: string;
dbType: DbType;
queueType: QueueType;
queueMode: QueueMode;
queuePrimaryProcesses: boolean;
statusPageThemesEnabled: boolean;
};
@@ -41,8 +41,8 @@ export class SettingsService implements ISettingsService {
clientHost: env.CLIENT_HOST,
dbConnectionString: env.DB_CONNECTION_STRING,
dbType: env.DB_TYPE,
queueType: env.QUEUE_TYPE,
queueMode: env.QUEUE_MODE,
queuePrimaryProcesses: env.QUEUE_PRIMARY_PROCESSES,
statusPageThemesEnabled: env.STATUS_PAGE_THEMES_ENABLED,
};
}
@@ -1,9 +1,6 @@
export const DbTypes = ["mongodb"] as const;
export type DbType = (typeof DbTypes)[number];
export const QueueTypes = ["lessSimpleQueue", "superSimpleQueue"] as const;
export type QueueType = (typeof QueueTypes)[number];
export const QueueModes = ["primary", "worker"] as const;
export type QueueMode = (typeof QueueModes)[number];
@@ -42,6 +42,7 @@ export interface IChecksRepository {
options?: { type?: MonitorType }
): Promise<UptimeChecksResult | HardwareChecksResult | PageSpeedChecksResult>;
findSummaryByTeamId(teamId: string, dateRange: string): Promise<ChecksSummary>;
findUnevaluatedByMonitorId(monitorId: string, since: number): Promise<Check[]>;
// update
//delete
deleteByMonitorId(monitorId: string): Promise<number>;
@@ -386,6 +386,16 @@ class MongoChecksRepository implements IChecksRepository {
};
};
findUnevaluatedByMonitorId = async (monitorId: string, since: number) => {
const docs = await CheckModel.find({
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
createdAt: { $gt: new Date(since) },
})
.sort({ createdAt: 1 })
.lean<CheckDocument[]>();
return docs.map(this.toEntity);
};
deleteByMonitorId = async (monitorId: string): Promise<number> => {
const result = await CheckModel.deleteMany({ "metadata.monitorId": new mongoose.Types.ObjectId(monitorId) });
return result.deletedCount;
+27 -3
View File
@@ -11,7 +11,9 @@ const SERVICE_NAME = "checkService";
export interface ICheckService {
createChecks(checks: Check[]): Promise<Check[]>;
buildCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
toLastEvaluatedAt(check: Check): number;
toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
getChecksByMonitor(params: {
monitorId: string;
teamId: string;
@@ -56,7 +58,11 @@ export class CheckService implements ICheckService {
return this.checksRepository.createChecks(checks);
};
buildCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
toLastEvaluatedAt = (check: Check) => {
return new Date(check.createdAt).getTime();
};
toCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
const now = new Date().toISOString();
@@ -82,7 +88,7 @@ export class CheckService implements ICheckService {
this.logger.warn({
message: "Failed to build check",
service: SERVICE_NAME,
method: "buildCheck",
method: "toCheck",
});
return undefined;
}
@@ -131,6 +137,24 @@ export class CheckService implements ICheckService {
return check;
};
toStatusResponse = (check: Check) => {
const statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]> = {
monitorId: check.metadata.monitorId,
teamId: check.metadata.teamId,
type: check.metadata.type,
status: check.status,
code: check.statusCode,
message: check.message,
responseTime: check.responseTime,
// re-nest hardware metrics so updateMonitorStatus / extractThresholdBreaches see payload.data
payload:
check.metadata.type === "hardware"
? ({ data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload)
: undefined,
};
return statusResponse;
};
getChecksByMonitor = async ({
monitorId,
teamId,
+79
View File
@@ -0,0 +1,79 @@
import { Schema, model } from "mongoose";
import { JOB_TYPES, type Job } from "@/domain/jobs/job.type.js";
export interface JobDocument extends Omit<Job, "id"> {
_id: string;
}
const JobSchema = new Schema<JobDocument>(
{
_id: {
type: String,
required: true,
},
type: {
type: String,
enum: JOB_TYPES,
required: true,
},
// monitorId for monitor-bound jobs; null for global cleanup jobs. Kept as a String pointer (no populate).
refId: {
type: String,
default: null,
},
// paused monitor => false; the claim scan skips inactive rows (replaces pauseJob/resumeJob)
isActive: {
type: Boolean,
default: true,
},
// Scheduling — epoch ms, so the claim/$min queries operate on plain numbers
nextScheduledAt: {
type: Number,
required: true,
},
intervalMs: {
type: Number,
default: null,
},
// Lock / lease
lockedBy: {
type: String,
default: null,
},
lockedUntil: {
type: Number,
default: null,
},
// Observability
runCount: {
type: Number,
default: 0,
},
failCount: {
type: Number,
default: 0,
},
lastFinishedAt: {
type: Number,
default: null,
},
lastFailReason: {
type: String,
default: null,
},
},
{ timestamps: true }
);
// Used for scanning for claims
JobSchema.index({ type: 1, isActive: 1, nextScheduledAt: 1 });
// for findById, upsertEvaluate, deleteById
JobSchema.index({ refId: 1, type: 1 });
const JobModel = model<JobDocument>("Job", JobSchema);
export { JobModel };
export default JobModel;
@@ -0,0 +1,72 @@
import { Job, JobSeed, JobType } from "@/domain/jobs/job.type.js";
export type JobPageQuery = {
page?: number;
rowsPerPage?: number;
};
export type JobPage = {
jobs: Job[];
count: number;
};
export interface IJobsRepository {
// ********************
// Claims
// ********************
// Atomically claim a job
claimDue(type: JobType, now: number): Promise<Job | null>;
// locks up to `limit` due rows in one batch
claimDueBatch(type: JobType, limit: number, now: number): Promise<Job[]>;
// Extends the locks we still hold. Returns how many were renewed
renewLocks(ids: string[], now: number): Promise<number>;
// Completes a repeating job
recordSuccess(id: string, nextScheduledAt: number, intervalMs: number, now: number): Promise<boolean>;
// Record failure, release lease, bump fail count, record reason, reset nextScheduledAt for retry
recordFailure(id: string, error: unknown, now: number): Promise<boolean>;
recordOneShot(id: string, now: number): Promise<boolean>;
// ********************
// Hand off to evaluator // This is accomplished by creating an "evaluation" type job
// ********************
upsertEvaluate(monitorId: string, now: number): Promise<boolean>;
// ********************
// Job CRUD
// ********************
// Create jobs. Takes a JobSeed
upsertJob(job: JobSeed): Promise<boolean>;
// Creates cleanup jobs, these are executed immediately so need their own method
upsertCleanupJob(job: JobSeed): Promise<boolean>;
// Pause/Resume
setActiveById(refId: string, isActive: boolean): Promise<boolean>;
// Edit job schedule
updateScheduleById(refId: string, type: JobType, intervalMs: number | null): Promise<boolean>;
// Delete, drop all rows
deleteById(refId: string): Promise<boolean>;
// Delete a single job of type
deleteByIdAndType(refId: string, type: JobType): Promise<boolean>;
// Delete jobs that reference monitors that no longer exist
deleteByMonitorIdsNotIn(monitorIds: string[]): Promise<number>;
// Get job
findById(refId: string): Promise<Job[]>;
// ********************
// Observability
// ********************
findPage(pagination: JobPageQuery): Promise<JobPage>;
findAll(): Promise<Job[]>;
}
@@ -0,0 +1,287 @@
import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js";
import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js";
class MongoJobsRepository implements IJobsRepository {
constructor(private readonly workerId: string) {}
private toEntity(doc: JobDocument): Job {
return {
id: doc._id,
type: doc.type,
refId: doc.refId,
isActive: doc.isActive,
nextScheduledAt: doc.nextScheduledAt,
intervalMs: doc.intervalMs,
lockedBy: doc.lockedBy,
lockedUntil: doc.lockedUntil,
runCount: doc.runCount,
failCount: doc.failCount,
lastFinishedAt: doc.lastFinishedAt,
lastFailReason: doc.lastFailReason,
};
}
claimDue = async (type: JobType, now: number) => {
const doc = await JobModel.findOneAndUpdate(
{
type,
isActive: true,
nextScheduledAt: { $lte: now }, // Find due jobs
$or: [
{ lockedUntil: null }, // Not locked
{ lockedUntil: { $lt: now } }, // Free or expired lock
],
},
{
$set: {
lockedBy: this.workerId,
lockedUntil: now + LOCK_MS,
},
},
{
new: true,
sort: { nextScheduledAt: 1 }, // Get the oldest due job
}
).lean<JobDocument>();
return doc ? this.toEntity(doc) : null;
};
claimDueBatch = async (type: JobType, limit: number, now: number) => {
if (limit <= 0) return [];
const dueAndFree = {
type,
isActive: true,
nextScheduledAt: { $lte: now }, // due
$or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }], // free or expired lock
};
// 1. Pick the oldest-due candidates
const candidates = await JobModel.find(dueAndFree).sort({ nextScheduledAt: 1 }).limit(limit).select({ _id: 1 }).lean<{ _id: string }[]>();
if (candidates.length === 0) return [];
const ids = candidates.map((d) => d._id);
// 2. lock them. Re-check free/expired so we never steal another worker's live lock
// only the rows this updateMany actually flips end up with claimed
const lockedUntil = now + LOCK_MS;
await JobModel.updateMany(
{ _id: { $in: ids }, $or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }] },
{ $set: { lockedBy: this.workerId, lockedUntil } }
);
// 3. Read back exactly the rows we now hold from this batch, keyed by the lockedUntil time
const docs = await JobModel.find({ _id: { $in: ids }, lockedBy: this.workerId, lockedUntil }).lean<JobDocument[]>();
return docs.map(this.toEntity);
};
renewLocks = async (ids: string[], now: number) => {
if (ids.length === 0) return 0;
const res = await JobModel.updateMany(
{ _id: { $in: ids }, lockedBy: this.workerId }, // only locks held by this worker can be renewed
{ $set: { lockedUntil: now + LOCK_MS } }
);
return res.modifiedCount;
};
recordSuccess = async (id: string, scheduledRunAt: number, intervalMs: number, now: number) => {
// Fixed rate job scheduling
let nextScheduledAt = scheduledRunAt + intervalMs;
// If the job fell behind, skip the missed runs. Spread the next run with jitter to avoid a herd
if (nextScheduledAt <= now) {
nextScheduledAt = now + intervalMs + Math.floor(Math.random() * intervalMs);
}
const res = await JobModel.updateOne(
{
_id: id,
lockedBy: this.workerId, // Only the worker that claimed can record success
},
{
$set: {
nextScheduledAt,
lockedBy: null, // Remove lock
lockedUntil: null,
lastFinishedAt: now,
},
$inc: {
runCount: 1, // Increment run count
},
}
);
return res.modifiedCount === 1;
};
recordFailure = async (id: string, error: unknown, now: number) => {
const res = await JobModel.updateOne(
{
_id: id,
lockedBy: this.workerId, // Only the worker that claimed can record success
},
{
$set: {
nextScheduledAt: now + BACKOFF_MS,
lockedBy: null, // Remove lock
lockedUntil: null,
lastFailReason: error instanceof Error ? error.message : String(error),
},
$inc: {
failCount: 1, // Increment fail count
},
}
);
return res.modifiedCount === 1;
};
recordOneShot = async (id: string, now: number) => {
const res = await JobModel.updateOne(
{
_id: id,
lockedBy: this.workerId, // Only the worker that claimed can record success
},
{
$set: {
nextScheduledAt: PARKED,
lockedBy: null, // Remove lock
lockedUntil: null,
lastFinishedAt: now,
},
$inc: {
runCount: 1, // Increment run count
},
}
);
return res.modifiedCount === 1;
};
upsertEvaluate = async (monitorId: string, now: number) => {
const filter = { type: "evaluate", refId: monitorId };
const update = {
$setOnInsert: {
_id: jobId("evaluate", monitorId),
// refId and Type inferred from filter
isActive: true,
intervalMs: null,
lockedBy: null,
lockedUntil: null,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
},
$min: {
nextScheduledAt: now, // no op if there is already a pending evaluation that is older
},
};
try {
const res = await JobModel.updateOne(filter, update, { upsert: true });
return res.acknowledged;
} catch (error) {
// Two callers can race the first insert for the same monitor, so the
// loser hits a duplicate _id. Retry once, the row now exists, so this becomes the $min.
if (error && typeof error === "object" && (error as { code?: number }).code === 11000) {
const res = await JobModel.updateOne(filter, update, { upsert: true });
return res.acknowledged;
}
throw error;
}
};
upsertJob = async (job: JobSeed) => {
const res = await JobModel.updateOne(
{ _id: job.id },
{
$set: {
// These fields are owned by the monitor, always safe to set
type: job.type,
refId: job.refId,
isActive: job.isActive,
intervalMs: job.intervalMs,
},
$setOnInsert: {
// These fields are scheduling related, only set on insert so they aren't clobbered
nextScheduledAt: job.nextScheduledAt,
lockedBy: null,
lockedUntil: null,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
},
},
{ upsert: true }
);
return res.acknowledged;
};
upsertCleanupJob = async (job: JobSeed) => {
const res = await JobModel.updateOne(
{ _id: job.id },
{
$set: {
type: job.type,
refId: job.refId,
isActive: job.isActive,
intervalMs: job.intervalMs,
nextScheduledAt: job.nextScheduledAt,
},
$setOnInsert: {
lockedBy: null,
lockedUntil: null,
runCount: 0,
failCount: 0,
lastFinishedAt: null,
lastFailReason: null,
},
},
{ upsert: true }
);
return res.acknowledged;
};
setActiveById = async (refId: string, isActive: boolean) => {
const res = await JobModel.updateMany({ refId }, { $set: { isActive } }); // inverts check and geo check rows together
return res.modifiedCount > 0;
};
updateScheduleById = async (refId: string, type: JobType, intervalMs: number | null) => {
const res = await JobModel.updateOne({ refId, type }, { $set: { intervalMs } });
return res.modifiedCount === 1;
};
deleteById = async (refId: string) => {
const res = await JobModel.deleteMany({ refId }); // Delete all jobs for monitor
return res.deletedCount > 0;
};
deleteByIdAndType = async (refId: string, type: JobType) => {
const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row
return res.deletedCount > 0;
};
deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise<number> => {
// remove jobs who'se refId is not null (cleanup jobs) and not in the list of monitor IDs (orphaned monitor jobs)
const res = await JobModel.deleteMany({ refId: { $ne: null, $nin: monitorIds } });
return res.deletedCount ?? 0;
};
findById = async (refId: string) => {
const docs = await JobModel.find({ refId }).lean<JobDocument[]>();
return docs.map(this.toEntity);
};
findPage = async ({ page = 0, rowsPerPage = 0 }: JobPageQuery) => {
const count = await JobModel.countDocuments();
const query = JobModel.find().sort({ nextScheduledAt: 1 });
if (rowsPerPage > 0) query.skip(Math.max(page, 0) * rowsPerPage).limit(rowsPerPage);
const docs = await query.lean<JobDocument[]>();
return { jobs: docs.map(this.toEntity), count };
};
findAll = async () => {
const docs = await JobModel.find().lean<JobDocument[]>();
return docs.map(this.toEntity);
};
}
export default MongoJobsRepository;
+32
View File
@@ -0,0 +1,32 @@
export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"] as const;
export type JobType = (typeof JOB_TYPES)[number];
export const LOCK_MS = 60_000;
export const BACKOFF_MS = 5_000;
export const PARKED = Number.MAX_SAFE_INTEGER;
// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null) are just the type.
export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`);
export type Job = {
id: string;
type: JobType;
refId: string | null;
isActive: boolean;
// Scheduling fields
nextScheduledAt: number; // epoch ms; due when nextScheduledAt <= now
intervalMs: number | null; // repeat period in ms; null = one time job
// Lock fields
lockedBy: string | null; // workerId holding the lease
lockedUntil: number | null; // epoch ms; lease expiry => reclaimable after a crash
// observability (replaces what getMetrics/getJobs)
runCount: number;
failCount: number;
lastFinishedAt: number | null;
lastFailReason: string | null;
};
// These are the only values to set when creating a job. Repo owns the rest
export type JobSeed = Pick<Job, "id" | "type" | "refId" | "isActive" | "intervalMs" | "nextScheduledAt">;
@@ -34,6 +34,7 @@ interface MonitorDocument extends MonitorDocumentBase {
teamId: Types.ObjectId;
createdAt: Date;
updatedAt: Date;
lastEvaluatedAt: number; // epoch ms
}
const snapshotTimingPhasesSchema = new Schema<GotTimings["phases"]>(
@@ -377,6 +378,10 @@ const MonitorSchema = new Schema<MonitorDocument>(
type: [checkSnapshotSchema],
default: [],
},
lastEvaluatedAt: {
type: Number,
default: 0,
},
},
{
timestamps: true,
@@ -571,6 +571,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
dnsRecordType: doc.dnsRecordType ?? undefined,
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
lastEvaluatedAt: doc.lastEvaluatedAt,
};
};
@@ -636,6 +637,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
dnsRecordType: doc.dnsRecordType ?? undefined,
createdAt: toDateString(doc.createdAt),
updatedAt: toDateString(doc.updatedAt),
lastEvaluatedAt: doc.lastEvaluatedAt,
};
};
@@ -597,6 +597,7 @@ export class MonitorService implements IMonitorService {
recentChecks: [],
createdAt: "",
updatedAt: "",
lastEvaluatedAt: 0,
}));
const createdMonitors = await this.createMonitors(cleanedMonitors);
@@ -88,6 +88,7 @@ export interface Monitor {
recentChecks: CheckSnapshot[];
createdAt: string;
updatedAt: string;
lastEvaluatedAt: number; // epoch ms
}
export interface MonitorsSummary {
@@ -2,11 +2,15 @@ import { Schema, model } from "mongoose";
import { QueueModes } from "@/domain/app-settings/app-settings.type.js";
import { QueueWorkerDocument } from "@/domain/queue-workers/queue-worker.type.js";
// Single source of truth for worker liveness: the TTL that GCs stale records and
// the window readers use to count a worker as alive. Heartbeat cadence must stay well below it.
export const WORKER_TTL_SECONDS = 180;
const QueueWorkerSchema = new Schema<QueueWorkerDocument>(
{
_id: { type: String, required: true },
mode: { type: String, enum: QueueModes, required: true },
lastSeenAt: { type: Date, default: Date.now, expires: 30 },
lastSeenAt: { type: Date, default: Date.now, expires: WORKER_TTL_SECONDS },
},
{ timestamps: true }
);