incident on initializing && down

This commit is contained in:
Alex Holliday committed 2026-04-28 10:09:42 -07:00
1 parent 959e2ff7b8
commit 54c0e089a8
2 files changed
+66 -19

No files matched your search

@@ -232,10 +232,17 @@ export class StatusService implements IStatusService {
// Build the status patch — computed against the projected window
const patch: Partial<Monitor> = {};
// Not enough data points yet
if (projectedWindow.length < monitor.statusWindowSize) {
patch.status = status === true ? "up" : "down";
// Resolve "initializing" up-front, incidents should be created on initialization && down
let newStatus: MonitorStatus = monitor.status;
let statusChanged = false;
if (monitor.status === "initializing") {
newStatus = status === true ? "up" : "down";
patch.status = newStatus;
statusChanged = newStatus === "down";
}
// Not enough data points yet — record the check and return
if (projectedWindow.length < monitor.statusWindowSize) {
const updated = await this.monitorsRepository.updateStatusWindowAndChecks(
monitor.id,
monitor.teamId,
@@ -248,24 +255,13 @@ export class StatusService implements IStatusService {
return {
monitor: updated,
statusChanged: false,
statusChanged,
prevStatus,
code,
timestamp: Date.now(),
};
}
// With a full window, a single raw check must not change UNLESS we are initializing.
// Otherwise, only the sliding-window threshold can trigger a transition.
let newStatus: MonitorStatus;
if (monitor.status === "initializing") {
newStatus = status === true ? "up" : "down";
} else {
newStatus = monitor.status;
}
let statusChanged = false;
// First evaluate reachability-based status changes, which apply to all monitor types
// and take precedence over hardware breaches.
const reachabilityResult = this.computeReachability(newStatus, projectedWindow, monitor.statusWindowThreshold);
@@ -502,9 +502,12 @@ describe("StatusService", () => {
expect(result.monitor.status).toBe("down");
});
it("during warmup (window not full) the stored status tracks the raw check", async () => {
// Early-return branch: monitor.status is written to reflect the current check and
// no status-changed notification is emitted. This is the service's documented warmup behavior.
it("during warmup, sub-threshold raw checks no longer flip stored status (regression: down-during-init incident not resolving)", async () => {
// Once the monitor has left 'initializing', the warmup branch must NOT silently
// flip status from raw checks. Otherwise a down check during warmup writes
// status='down' without statusChanged, and the next up check leaks status='up'
// the same way — by the time the window fills, monitor.status is already 'up'
// and computeReachability cannot fire down→up to resolve the incident.
const monitor = makeMonitor({ statusWindow: [true, true], statusWindowSize: 5, status: "up" });
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
@@ -513,10 +516,58 @@ describe("StatusService", () => {
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("down");
expect(result.monitor.status).toBe("up");
expect(monitor.statusWindow).toEqual([true, true, false]);
});
it("creates incident on initial down check when statusWindowSize is 1 (regression: alert on initial down skips warmup branch)", async () => {
// statusWindowSize=1 means the very first check skips the warmup branch entirely
// (1 < 1 is false) and lands in the full-window branch. The 'initializing' override
// must still surface statusChanged=true on a down result, otherwise computeReachability
// sees currentStatus='down' and refuses to transition, and no incident is opened.
const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 1, statusWindowThreshold: 60, status: "initializing" });
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("down");
expect(result.prevStatus).toBe("initializing");
});
it("creates incident on initial down check for a brand-new monitor (regression: alert on initial down)", async () => {
// Brand-new monitor: status='initializing', empty statusWindow. First check is down.
// Must surface statusChanged=true with monitor.status='down' so the helper opens an incident.
const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "initializing" });
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
expect(result.statusChanged).toBe(true);
expect(result.monitor.status).toBe("down");
expect(result.prevStatus).toBe("initializing");
});
it("does not flip status on subsequent checks during warmup once 'initializing' has been left (regression: down→up incident resolves once window fills)", async () => {
// After a 'initializing'→'down' transition on check #1, monitor.status is now 'down'
// and statusWindow has one false. Subsequent up checks during warmup must leave
// monitor.status='down' so the sliding window can detect the down→up transition
// once the window fills.
const monitor = makeMonitor({ statusWindow: [false], statusWindowSize: 5, status: "down" });
const { service, monitorsRepository } = createService();
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
expect(result.statusChanged).toBe(false);
expect(result.monitor.status).toBe("down");
});
it("logs a warning but still succeeds when running-stats update fails mid-flight", async () => {
const monitor = makeMonitor({ statusWindow: [true, true, true, true], statusWindowSize: 5, status: "up" });
const { service, logger, monitorsRepository, monitorStatsRepository } = createService();