Merge pull request #3728 from bluewave-labs/feat/db-queue
feat: db queue
This commit is contained in:
42 files changed
+3499
-3514
No files matched your search
@@ -1,5 +1,5 @@
|
||||
import { ISettingsRepository } from "@/domain/app-settings/app-settings-repository.interface.js";
|
||||
import { Settings, SettingsUpdate, type DbType, type QueueMode, type QueueType } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { Settings, SettingsUpdate, type DbType, type QueueMode } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { AppError } from "@/utils/AppError.js";
|
||||
import { ValidatedEnv } from "@/config/envValidation.js";
|
||||
import type { StringValue } from "ms";
|
||||
@@ -13,8 +13,8 @@ export type EnvConfig = {
|
||||
clientHost: string;
|
||||
dbConnectionString: string;
|
||||
dbType: DbType;
|
||||
queueType: QueueType;
|
||||
queueMode: QueueMode;
|
||||
queuePrimaryProcesses: boolean;
|
||||
statusPageThemesEnabled: boolean;
|
||||
};
|
||||
|
||||
@@ -41,8 +41,8 @@ export class SettingsService implements ISettingsService {
|
||||
clientHost: env.CLIENT_HOST,
|
||||
dbConnectionString: env.DB_CONNECTION_STRING,
|
||||
dbType: env.DB_TYPE,
|
||||
queueType: env.QUEUE_TYPE,
|
||||
queueMode: env.QUEUE_MODE,
|
||||
queuePrimaryProcesses: env.QUEUE_PRIMARY_PROCESSES,
|
||||
statusPageThemesEnabled: env.STATUS_PAGE_THEMES_ENABLED,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
export const DbTypes = ["mongodb"] as const;
|
||||
export type DbType = (typeof DbTypes)[number];
|
||||
|
||||
export const QueueTypes = ["lessSimpleQueue", "superSimpleQueue"] as const;
|
||||
export type QueueType = (typeof QueueTypes)[number];
|
||||
|
||||
export const QueueModes = ["primary", "worker"] as const;
|
||||
export type QueueMode = (typeof QueueModes)[number];
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ export interface IChecksRepository {
|
||||
options?: { type?: MonitorType }
|
||||
): Promise<UptimeChecksResult | HardwareChecksResult | PageSpeedChecksResult>;
|
||||
findSummaryByTeamId(teamId: string, dateRange: string): Promise<ChecksSummary>;
|
||||
findUnevaluatedByMonitorId(monitorId: string, since: number): Promise<Check[]>;
|
||||
// update
|
||||
//delete
|
||||
deleteByMonitorId(monitorId: string): Promise<number>;
|
||||
|
||||
@@ -386,6 +386,16 @@ class MongoChecksRepository implements IChecksRepository {
|
||||
};
|
||||
};
|
||||
|
||||
findUnevaluatedByMonitorId = async (monitorId: string, since: number) => {
|
||||
const docs = await CheckModel.find({
|
||||
"metadata.monitorId": new mongoose.Types.ObjectId(monitorId),
|
||||
createdAt: { $gt: new Date(since) },
|
||||
})
|
||||
.sort({ createdAt: 1 })
|
||||
.lean<CheckDocument[]>();
|
||||
return docs.map(this.toEntity);
|
||||
};
|
||||
|
||||
deleteByMonitorId = async (monitorId: string): Promise<number> => {
|
||||
const result = await CheckModel.deleteMany({ "metadata.monitorId": new mongoose.Types.ObjectId(monitorId) });
|
||||
return result.deletedCount;
|
||||
|
||||
@@ -11,7 +11,9 @@ const SERVICE_NAME = "checkService";
|
||||
|
||||
export interface ICheckService {
|
||||
createChecks(checks: Check[]): Promise<Check[]>;
|
||||
buildCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
|
||||
toLastEvaluatedAt(check: Check): number;
|
||||
toCheck(statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined;
|
||||
toStatusResponse(check: Check): MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>;
|
||||
getChecksByMonitor(params: {
|
||||
monitorId: string;
|
||||
teamId: string;
|
||||
@@ -56,7 +58,11 @@ export class CheckService implements ICheckService {
|
||||
return this.checksRepository.createChecks(checks);
|
||||
};
|
||||
|
||||
buildCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
|
||||
toLastEvaluatedAt = (check: Check) => {
|
||||
return new Date(check.createdAt).getTime();
|
||||
};
|
||||
|
||||
toCheck = (statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]>): Check | undefined => {
|
||||
const { monitorId, teamId, type, status, responseTime, code, message, payload, timings } = statusResponse;
|
||||
|
||||
const now = new Date().toISOString();
|
||||
@@ -82,7 +88,7 @@ export class CheckService implements ICheckService {
|
||||
this.logger.warn({
|
||||
message: "Failed to build check",
|
||||
service: SERVICE_NAME,
|
||||
method: "buildCheck",
|
||||
method: "toCheck",
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
@@ -131,6 +137,24 @@ export class CheckService implements ICheckService {
|
||||
return check;
|
||||
};
|
||||
|
||||
toStatusResponse = (check: Check) => {
|
||||
const statusResponse: MonitorStatusResponse<MonitorPayloadMap[keyof MonitorPayloadMap]> = {
|
||||
monitorId: check.metadata.monitorId,
|
||||
teamId: check.metadata.teamId,
|
||||
type: check.metadata.type,
|
||||
status: check.status,
|
||||
code: check.statusCode,
|
||||
message: check.message,
|
||||
responseTime: check.responseTime,
|
||||
// re-nest hardware metrics so updateMonitorStatus / extractThresholdBreaches see payload.data
|
||||
payload:
|
||||
check.metadata.type === "hardware"
|
||||
? ({ data: { cpu: check.cpu, memory: check.memory, disk: check.disk, host: check.host, net: check.net } } as HardwareStatusPayload)
|
||||
: undefined,
|
||||
};
|
||||
return statusResponse;
|
||||
};
|
||||
|
||||
getChecksByMonitor = async ({
|
||||
monitorId,
|
||||
teamId,
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import { Schema, model } from "mongoose";
|
||||
import { JOB_TYPES, type Job } from "@/domain/jobs/job.type.js";
|
||||
|
||||
export interface JobDocument extends Omit<Job, "id"> {
|
||||
_id: string;
|
||||
}
|
||||
|
||||
const JobSchema = new Schema<JobDocument>(
|
||||
{
|
||||
_id: {
|
||||
type: String,
|
||||
required: true,
|
||||
},
|
||||
type: {
|
||||
type: String,
|
||||
enum: JOB_TYPES,
|
||||
required: true,
|
||||
},
|
||||
// monitorId for monitor-bound jobs; null for global cleanup jobs. Kept as a String pointer (no populate).
|
||||
refId: {
|
||||
type: String,
|
||||
default: null,
|
||||
},
|
||||
// paused monitor => false; the claim scan skips inactive rows (replaces pauseJob/resumeJob)
|
||||
isActive: {
|
||||
type: Boolean,
|
||||
default: true,
|
||||
},
|
||||
|
||||
// Scheduling — epoch ms, so the claim/$min queries operate on plain numbers
|
||||
nextScheduledAt: {
|
||||
type: Number,
|
||||
required: true,
|
||||
},
|
||||
intervalMs: {
|
||||
type: Number,
|
||||
default: null,
|
||||
},
|
||||
|
||||
// Lock / lease
|
||||
lockedBy: {
|
||||
type: String,
|
||||
default: null,
|
||||
},
|
||||
lockedUntil: {
|
||||
type: Number,
|
||||
default: null,
|
||||
},
|
||||
|
||||
// Observability
|
||||
runCount: {
|
||||
type: Number,
|
||||
default: 0,
|
||||
},
|
||||
failCount: {
|
||||
type: Number,
|
||||
default: 0,
|
||||
},
|
||||
lastFinishedAt: {
|
||||
type: Number,
|
||||
default: null,
|
||||
},
|
||||
lastFailReason: {
|
||||
type: String,
|
||||
default: null,
|
||||
},
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
|
||||
// Used for scanning for claims
|
||||
JobSchema.index({ type: 1, isActive: 1, nextScheduledAt: 1 });
|
||||
// for findById, upsertEvaluate, deleteById
|
||||
JobSchema.index({ refId: 1, type: 1 });
|
||||
|
||||
const JobModel = model<JobDocument>("Job", JobSchema);
|
||||
|
||||
export { JobModel };
|
||||
export default JobModel;
|
||||
@@ -0,0 +1,72 @@
|
||||
import { Job, JobSeed, JobType } from "@/domain/jobs/job.type.js";
|
||||
|
||||
export type JobPageQuery = {
|
||||
page?: number;
|
||||
rowsPerPage?: number;
|
||||
};
|
||||
export type JobPage = {
|
||||
jobs: Job[];
|
||||
count: number;
|
||||
};
|
||||
|
||||
export interface IJobsRepository {
|
||||
// ********************
|
||||
// Claims
|
||||
// ********************
|
||||
|
||||
// Atomically claim a job
|
||||
claimDue(type: JobType, now: number): Promise<Job | null>;
|
||||
|
||||
// locks up to `limit` due rows in one batch
|
||||
claimDueBatch(type: JobType, limit: number, now: number): Promise<Job[]>;
|
||||
|
||||
// Extends the locks we still hold. Returns how many were renewed
|
||||
renewLocks(ids: string[], now: number): Promise<number>;
|
||||
|
||||
// Completes a repeating job
|
||||
recordSuccess(id: string, nextScheduledAt: number, intervalMs: number, now: number): Promise<boolean>;
|
||||
|
||||
// Record failure, release lease, bump fail count, record reason, reset nextScheduledAt for retry
|
||||
recordFailure(id: string, error: unknown, now: number): Promise<boolean>;
|
||||
|
||||
recordOneShot(id: string, now: number): Promise<boolean>;
|
||||
|
||||
// ********************
|
||||
// Hand off to evaluator // This is accomplished by creating an "evaluation" type job
|
||||
// ********************
|
||||
upsertEvaluate(monitorId: string, now: number): Promise<boolean>;
|
||||
|
||||
// ********************
|
||||
// Job CRUD
|
||||
// ********************
|
||||
|
||||
// Create jobs. Takes a JobSeed
|
||||
upsertJob(job: JobSeed): Promise<boolean>;
|
||||
|
||||
// Creates cleanup jobs, these are executed immediately so need their own method
|
||||
upsertCleanupJob(job: JobSeed): Promise<boolean>;
|
||||
|
||||
// Pause/Resume
|
||||
setActiveById(refId: string, isActive: boolean): Promise<boolean>;
|
||||
|
||||
// Edit job schedule
|
||||
updateScheduleById(refId: string, type: JobType, intervalMs: number | null): Promise<boolean>;
|
||||
|
||||
// Delete, drop all rows
|
||||
deleteById(refId: string): Promise<boolean>;
|
||||
|
||||
// Delete a single job of type
|
||||
deleteByIdAndType(refId: string, type: JobType): Promise<boolean>;
|
||||
|
||||
// Delete jobs that reference monitors that no longer exist
|
||||
deleteByMonitorIdsNotIn(monitorIds: string[]): Promise<number>;
|
||||
|
||||
// Get job
|
||||
findById(refId: string): Promise<Job[]>;
|
||||
|
||||
// ********************
|
||||
// Observability
|
||||
// ********************
|
||||
findPage(pagination: JobPageQuery): Promise<JobPage>;
|
||||
findAll(): Promise<Job[]>;
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
import { IJobsRepository, JobPageQuery } from "@/domain/jobs/job.repository.interface.js";
|
||||
import JobModel, { JobDocument } from "@/domain/jobs/job.model.js";
|
||||
import { JobType, Job, JobSeed, BACKOFF_MS, LOCK_MS, PARKED, jobId } from "@/domain/jobs/job.type.js";
|
||||
|
||||
class MongoJobsRepository implements IJobsRepository {
|
||||
constructor(private readonly workerId: string) {}
|
||||
|
||||
private toEntity(doc: JobDocument): Job {
|
||||
return {
|
||||
id: doc._id,
|
||||
type: doc.type,
|
||||
refId: doc.refId,
|
||||
isActive: doc.isActive,
|
||||
nextScheduledAt: doc.nextScheduledAt,
|
||||
intervalMs: doc.intervalMs,
|
||||
lockedBy: doc.lockedBy,
|
||||
lockedUntil: doc.lockedUntil,
|
||||
runCount: doc.runCount,
|
||||
failCount: doc.failCount,
|
||||
lastFinishedAt: doc.lastFinishedAt,
|
||||
lastFailReason: doc.lastFailReason,
|
||||
};
|
||||
}
|
||||
|
||||
claimDue = async (type: JobType, now: number) => {
|
||||
const doc = await JobModel.findOneAndUpdate(
|
||||
{
|
||||
type,
|
||||
isActive: true,
|
||||
nextScheduledAt: { $lte: now }, // Find due jobs
|
||||
$or: [
|
||||
{ lockedUntil: null }, // Not locked
|
||||
{ lockedUntil: { $lt: now } }, // Free or expired lock
|
||||
],
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
lockedBy: this.workerId,
|
||||
lockedUntil: now + LOCK_MS,
|
||||
},
|
||||
},
|
||||
{
|
||||
new: true,
|
||||
sort: { nextScheduledAt: 1 }, // Get the oldest due job
|
||||
}
|
||||
).lean<JobDocument>();
|
||||
return doc ? this.toEntity(doc) : null;
|
||||
};
|
||||
|
||||
claimDueBatch = async (type: JobType, limit: number, now: number) => {
|
||||
if (limit <= 0) return [];
|
||||
const dueAndFree = {
|
||||
type,
|
||||
isActive: true,
|
||||
nextScheduledAt: { $lte: now }, // due
|
||||
$or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }], // free or expired lock
|
||||
};
|
||||
|
||||
// 1. Pick the oldest-due candidates
|
||||
const candidates = await JobModel.find(dueAndFree).sort({ nextScheduledAt: 1 }).limit(limit).select({ _id: 1 }).lean<{ _id: string }[]>();
|
||||
if (candidates.length === 0) return [];
|
||||
const ids = candidates.map((d) => d._id);
|
||||
|
||||
// 2. lock them. Re-check free/expired so we never steal another worker's live lock
|
||||
// only the rows this updateMany actually flips end up with claimed
|
||||
const lockedUntil = now + LOCK_MS;
|
||||
await JobModel.updateMany(
|
||||
{ _id: { $in: ids }, $or: [{ lockedUntil: null }, { lockedUntil: { $lt: now } }] },
|
||||
{ $set: { lockedBy: this.workerId, lockedUntil } }
|
||||
);
|
||||
|
||||
// 3. Read back exactly the rows we now hold from this batch, keyed by the lockedUntil time
|
||||
const docs = await JobModel.find({ _id: { $in: ids }, lockedBy: this.workerId, lockedUntil }).lean<JobDocument[]>();
|
||||
return docs.map(this.toEntity);
|
||||
};
|
||||
|
||||
renewLocks = async (ids: string[], now: number) => {
|
||||
if (ids.length === 0) return 0;
|
||||
const res = await JobModel.updateMany(
|
||||
{ _id: { $in: ids }, lockedBy: this.workerId }, // only locks held by this worker can be renewed
|
||||
{ $set: { lockedUntil: now + LOCK_MS } }
|
||||
);
|
||||
return res.modifiedCount;
|
||||
};
|
||||
|
||||
recordSuccess = async (id: string, scheduledRunAt: number, intervalMs: number, now: number) => {
|
||||
// Fixed rate job scheduling
|
||||
|
||||
let nextScheduledAt = scheduledRunAt + intervalMs;
|
||||
|
||||
// If the job fell behind, skip the missed runs. Spread the next run with jitter to avoid a herd
|
||||
if (nextScheduledAt <= now) {
|
||||
nextScheduledAt = now + intervalMs + Math.floor(Math.random() * intervalMs);
|
||||
}
|
||||
|
||||
const res = await JobModel.updateOne(
|
||||
{
|
||||
_id: id,
|
||||
lockedBy: this.workerId, // Only the worker that claimed can record success
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
nextScheduledAt,
|
||||
lockedBy: null, // Remove lock
|
||||
lockedUntil: null,
|
||||
lastFinishedAt: now,
|
||||
},
|
||||
$inc: {
|
||||
runCount: 1, // Increment run count
|
||||
},
|
||||
}
|
||||
);
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
recordFailure = async (id: string, error: unknown, now: number) => {
|
||||
const res = await JobModel.updateOne(
|
||||
{
|
||||
_id: id,
|
||||
lockedBy: this.workerId, // Only the worker that claimed can record success
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
nextScheduledAt: now + BACKOFF_MS,
|
||||
lockedBy: null, // Remove lock
|
||||
lockedUntil: null,
|
||||
lastFailReason: error instanceof Error ? error.message : String(error),
|
||||
},
|
||||
$inc: {
|
||||
failCount: 1, // Increment fail count
|
||||
},
|
||||
}
|
||||
);
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
recordOneShot = async (id: string, now: number) => {
|
||||
const res = await JobModel.updateOne(
|
||||
{
|
||||
_id: id,
|
||||
lockedBy: this.workerId, // Only the worker that claimed can record success
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
nextScheduledAt: PARKED,
|
||||
lockedBy: null, // Remove lock
|
||||
lockedUntil: null,
|
||||
lastFinishedAt: now,
|
||||
},
|
||||
$inc: {
|
||||
runCount: 1, // Increment run count
|
||||
},
|
||||
}
|
||||
);
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
upsertEvaluate = async (monitorId: string, now: number) => {
|
||||
const filter = { type: "evaluate", refId: monitorId };
|
||||
const update = {
|
||||
$setOnInsert: {
|
||||
_id: jobId("evaluate", monitorId),
|
||||
// refId and Type inferred from filter
|
||||
isActive: true,
|
||||
intervalMs: null,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
},
|
||||
$min: {
|
||||
nextScheduledAt: now, // no op if there is already a pending evaluation that is older
|
||||
},
|
||||
};
|
||||
try {
|
||||
const res = await JobModel.updateOne(filter, update, { upsert: true });
|
||||
return res.acknowledged;
|
||||
} catch (error) {
|
||||
// Two callers can race the first insert for the same monitor, so the
|
||||
// loser hits a duplicate _id. Retry once, the row now exists, so this becomes the $min.
|
||||
if (error && typeof error === "object" && (error as { code?: number }).code === 11000) {
|
||||
const res = await JobModel.updateOne(filter, update, { upsert: true });
|
||||
return res.acknowledged;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
upsertJob = async (job: JobSeed) => {
|
||||
const res = await JobModel.updateOne(
|
||||
{ _id: job.id },
|
||||
{
|
||||
$set: {
|
||||
// These fields are owned by the monitor, always safe to set
|
||||
type: job.type,
|
||||
refId: job.refId,
|
||||
isActive: job.isActive,
|
||||
intervalMs: job.intervalMs,
|
||||
},
|
||||
$setOnInsert: {
|
||||
// These fields are scheduling related, only set on insert so they aren't clobbered
|
||||
nextScheduledAt: job.nextScheduledAt,
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
},
|
||||
},
|
||||
{ upsert: true }
|
||||
);
|
||||
return res.acknowledged;
|
||||
};
|
||||
|
||||
upsertCleanupJob = async (job: JobSeed) => {
|
||||
const res = await JobModel.updateOne(
|
||||
{ _id: job.id },
|
||||
{
|
||||
$set: {
|
||||
type: job.type,
|
||||
refId: job.refId,
|
||||
isActive: job.isActive,
|
||||
intervalMs: job.intervalMs,
|
||||
nextScheduledAt: job.nextScheduledAt,
|
||||
},
|
||||
$setOnInsert: {
|
||||
lockedBy: null,
|
||||
lockedUntil: null,
|
||||
runCount: 0,
|
||||
failCount: 0,
|
||||
lastFinishedAt: null,
|
||||
lastFailReason: null,
|
||||
},
|
||||
},
|
||||
{ upsert: true }
|
||||
);
|
||||
return res.acknowledged;
|
||||
};
|
||||
|
||||
setActiveById = async (refId: string, isActive: boolean) => {
|
||||
const res = await JobModel.updateMany({ refId }, { $set: { isActive } }); // inverts check and geo check rows together
|
||||
return res.modifiedCount > 0;
|
||||
};
|
||||
|
||||
updateScheduleById = async (refId: string, type: JobType, intervalMs: number | null) => {
|
||||
const res = await JobModel.updateOne({ refId, type }, { $set: { intervalMs } });
|
||||
return res.modifiedCount === 1;
|
||||
};
|
||||
|
||||
deleteById = async (refId: string) => {
|
||||
const res = await JobModel.deleteMany({ refId }); // Delete all jobs for monitor
|
||||
return res.deletedCount > 0;
|
||||
};
|
||||
|
||||
deleteByIdAndType = async (refId: string, type: JobType) => {
|
||||
const res = await JobModel.deleteOne({ refId, type }); // Delete a single typed row, e.g. just the geo row
|
||||
return res.deletedCount > 0;
|
||||
};
|
||||
|
||||
deleteByMonitorIdsNotIn = async (monitorIds: string[]): Promise<number> => {
|
||||
// remove jobs who'se refId is not null (cleanup jobs) and not in the list of monitor IDs (orphaned monitor jobs)
|
||||
const res = await JobModel.deleteMany({ refId: { $ne: null, $nin: monitorIds } });
|
||||
return res.deletedCount ?? 0;
|
||||
};
|
||||
|
||||
findById = async (refId: string) => {
|
||||
const docs = await JobModel.find({ refId }).lean<JobDocument[]>();
|
||||
return docs.map(this.toEntity);
|
||||
};
|
||||
|
||||
findPage = async ({ page = 0, rowsPerPage = 0 }: JobPageQuery) => {
|
||||
const count = await JobModel.countDocuments();
|
||||
const query = JobModel.find().sort({ nextScheduledAt: 1 });
|
||||
if (rowsPerPage > 0) query.skip(Math.max(page, 0) * rowsPerPage).limit(rowsPerPage);
|
||||
const docs = await query.lean<JobDocument[]>();
|
||||
return { jobs: docs.map(this.toEntity), count };
|
||||
};
|
||||
|
||||
findAll = async () => {
|
||||
const docs = await JobModel.find().lean<JobDocument[]>();
|
||||
return docs.map(this.toEntity);
|
||||
};
|
||||
}
|
||||
|
||||
export default MongoJobsRepository;
|
||||
@@ -0,0 +1,32 @@
|
||||
export const JOB_TYPES = ["check", "geo-check", "evaluate", "cleanup-orphaned", "cleanup-retention"] as const;
|
||||
export type JobType = (typeof JOB_TYPES)[number];
|
||||
export const LOCK_MS = 60_000;
|
||||
export const BACKOFF_MS = 5_000;
|
||||
export const PARKED = Number.MAX_SAFE_INTEGER;
|
||||
|
||||
// Canonical _id scheme: monitor-bound rows are `${type}:${refId}`; global rows (refId null) are just the type.
|
||||
export const jobId = (type: JobType, refId: string | null): string => (refId === null ? type : `${type}:${refId}`);
|
||||
|
||||
export type Job = {
|
||||
id: string;
|
||||
type: JobType;
|
||||
refId: string | null;
|
||||
isActive: boolean;
|
||||
|
||||
// Scheduling fields
|
||||
nextScheduledAt: number; // epoch ms; due when nextScheduledAt <= now
|
||||
intervalMs: number | null; // repeat period in ms; null = one time job
|
||||
|
||||
// Lock fields
|
||||
lockedBy: string | null; // workerId holding the lease
|
||||
lockedUntil: number | null; // epoch ms; lease expiry => reclaimable after a crash
|
||||
|
||||
// observability (replaces what getMetrics/getJobs)
|
||||
runCount: number;
|
||||
failCount: number;
|
||||
lastFinishedAt: number | null;
|
||||
lastFailReason: string | null;
|
||||
};
|
||||
|
||||
// These are the only values to set when creating a job. Repo owns the rest
|
||||
export type JobSeed = Pick<Job, "id" | "type" | "refId" | "isActive" | "intervalMs" | "nextScheduledAt">;
|
||||
@@ -34,6 +34,7 @@ interface MonitorDocument extends MonitorDocumentBase {
|
||||
teamId: Types.ObjectId;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
lastEvaluatedAt: number; // epoch ms
|
||||
}
|
||||
|
||||
const snapshotTimingPhasesSchema = new Schema<GotTimings["phases"]>(
|
||||
@@ -377,6 +378,10 @@ const MonitorSchema = new Schema<MonitorDocument>(
|
||||
type: [checkSnapshotSchema],
|
||||
default: [],
|
||||
},
|
||||
lastEvaluatedAt: {
|
||||
type: Number,
|
||||
default: 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
timestamps: true,
|
||||
|
||||
@@ -571,6 +571,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
|
||||
dnsRecordType: doc.dnsRecordType ?? undefined,
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
lastEvaluatedAt: doc.lastEvaluatedAt,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -636,6 +637,7 @@ class MongoMonitorsRepository implements IMonitorsRepository {
|
||||
dnsRecordType: doc.dnsRecordType ?? undefined,
|
||||
createdAt: toDateString(doc.createdAt),
|
||||
updatedAt: toDateString(doc.updatedAt),
|
||||
lastEvaluatedAt: doc.lastEvaluatedAt,
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -597,6 +597,7 @@ export class MonitorService implements IMonitorService {
|
||||
recentChecks: [],
|
||||
createdAt: "",
|
||||
updatedAt: "",
|
||||
lastEvaluatedAt: 0,
|
||||
}));
|
||||
|
||||
const createdMonitors = await this.createMonitors(cleanedMonitors);
|
||||
|
||||
@@ -88,6 +88,7 @@ export interface Monitor {
|
||||
recentChecks: CheckSnapshot[];
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
lastEvaluatedAt: number; // epoch ms
|
||||
}
|
||||
|
||||
export interface MonitorsSummary {
|
||||
|
||||
@@ -2,11 +2,15 @@ import { Schema, model } from "mongoose";
|
||||
import { QueueModes } from "@/domain/app-settings/app-settings.type.js";
|
||||
import { QueueWorkerDocument } from "@/domain/queue-workers/queue-worker.type.js";
|
||||
|
||||
// Single source of truth for worker liveness: the TTL that GCs stale records and
|
||||
// the window readers use to count a worker as alive. Heartbeat cadence must stay well below it.
|
||||
export const WORKER_TTL_SECONDS = 180;
|
||||
|
||||
const QueueWorkerSchema = new Schema<QueueWorkerDocument>(
|
||||
{
|
||||
_id: { type: String, required: true },
|
||||
mode: { type: String, enum: QueueModes, required: true },
|
||||
lastSeenAt: { type: Date, default: Date.now, expires: 30 },
|
||||
lastSeenAt: { type: Date, default: Date.now, expires: WORKER_TTL_SECONDS },
|
||||
},
|
||||
{ timestamps: true }
|
||||
);
|
||||
|
||||
Reference in new issue
Block a user