incident on initializing && down
This commit is contained in:
1 parent
959e2ff7b8
commit
54c0e089a8
2 files changed
+66
-19
No files matched your search
@@ -232,10 +232,17 @@ export class StatusService implements IStatusService {
|
||||
// Build the status patch — computed against the projected window
|
||||
const patch: Partial<Monitor> = {};
|
||||
|
||||
// Not enough data points yet
|
||||
if (projectedWindow.length < monitor.statusWindowSize) {
|
||||
patch.status = status === true ? "up" : "down";
|
||||
// Resolve "initializing" up-front, incidents should be created on initialization && down
|
||||
let newStatus: MonitorStatus = monitor.status;
|
||||
let statusChanged = false;
|
||||
if (monitor.status === "initializing") {
|
||||
newStatus = status === true ? "up" : "down";
|
||||
patch.status = newStatus;
|
||||
statusChanged = newStatus === "down";
|
||||
}
|
||||
|
||||
// Not enough data points yet — record the check and return
|
||||
if (projectedWindow.length < monitor.statusWindowSize) {
|
||||
const updated = await this.monitorsRepository.updateStatusWindowAndChecks(
|
||||
monitor.id,
|
||||
monitor.teamId,
|
||||
@@ -248,24 +255,13 @@ export class StatusService implements IStatusService {
|
||||
|
||||
return {
|
||||
monitor: updated,
|
||||
statusChanged: false,
|
||||
statusChanged,
|
||||
prevStatus,
|
||||
code,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
// With a full window, a single raw check must not change UNLESS we are initializing.
|
||||
// Otherwise, only the sliding-window threshold can trigger a transition.
|
||||
let newStatus: MonitorStatus;
|
||||
if (monitor.status === "initializing") {
|
||||
newStatus = status === true ? "up" : "down";
|
||||
} else {
|
||||
newStatus = monitor.status;
|
||||
}
|
||||
|
||||
let statusChanged = false;
|
||||
|
||||
// First evaluate reachability-based status changes, which apply to all monitor types
|
||||
// and take precedence over hardware breaches.
|
||||
const reachabilityResult = this.computeReachability(newStatus, projectedWindow, monitor.statusWindowThreshold);
|
||||
|
||||
@@ -502,9 +502,12 @@ describe("StatusService", () => {
|
||||
expect(result.monitor.status).toBe("down");
|
||||
});
|
||||
|
||||
it("during warmup (window not full) the stored status tracks the raw check", async () => {
|
||||
// Early-return branch: monitor.status is written to reflect the current check and
|
||||
// no status-changed notification is emitted. This is the service's documented warmup behavior.
|
||||
it("during warmup, sub-threshold raw checks no longer flip stored status (regression: down-during-init incident not resolving)", async () => {
|
||||
// Once the monitor has left 'initializing', the warmup branch must NOT silently
|
||||
// flip status from raw checks. Otherwise a down check during warmup writes
|
||||
// status='down' without statusChanged, and the next up check leaks status='up'
|
||||
// the same way — by the time the window fills, monitor.status is already 'up'
|
||||
// and computeReachability cannot fire down→up to resolve the incident.
|
||||
const monitor = makeMonitor({ statusWindow: [true, true], statusWindowSize: 5, status: "up" });
|
||||
const { service, monitorsRepository } = createService();
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
@@ -513,10 +516,58 @@ describe("StatusService", () => {
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("down");
|
||||
expect(result.monitor.status).toBe("up");
|
||||
expect(monitor.statusWindow).toEqual([true, true, false]);
|
||||
});
|
||||
|
||||
it("creates incident on initial down check when statusWindowSize is 1 (regression: alert on initial down skips warmup branch)", async () => {
|
||||
// statusWindowSize=1 means the very first check skips the warmup branch entirely
|
||||
// (1 < 1 is false) and lands in the full-window branch. The 'initializing' override
|
||||
// must still surface statusChanged=true on a down result, otherwise computeReachability
|
||||
// sees currentStatus='down' and refuses to transition, and no incident is opened.
|
||||
const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 1, statusWindowThreshold: 60, status: "initializing" });
|
||||
const { service, monitorsRepository } = createService();
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("down");
|
||||
expect(result.prevStatus).toBe("initializing");
|
||||
});
|
||||
|
||||
it("creates incident on initial down check for a brand-new monitor (regression: alert on initial down)", async () => {
|
||||
// Brand-new monitor: status='initializing', empty statusWindow. First check is down.
|
||||
// Must surface statusChanged=true with monitor.status='down' so the helper opens an incident.
|
||||
const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "initializing" });
|
||||
const { service, monitorsRepository } = createService();
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
|
||||
|
||||
expect(result.statusChanged).toBe(true);
|
||||
expect(result.monitor.status).toBe("down");
|
||||
expect(result.prevStatus).toBe("initializing");
|
||||
});
|
||||
|
||||
it("does not flip status on subsequent checks during warmup once 'initializing' has been left (regression: down→up incident resolves once window fills)", async () => {
|
||||
// After a 'initializing'→'down' transition on check #1, monitor.status is now 'down'
|
||||
// and statusWindow has one false. Subsequent up checks during warmup must leave
|
||||
// monitor.status='down' so the sliding window can detect the down→up transition
|
||||
// once the window fills.
|
||||
const monitor = makeMonitor({ statusWindow: [false], statusWindowSize: 5, status: "down" });
|
||||
const { service, monitorsRepository } = createService();
|
||||
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||
(monitorsRepository.updateById as jest.Mock).mockImplementation((_id: unknown, _tid: unknown, m: unknown) => Promise.resolve(m));
|
||||
|
||||
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
|
||||
|
||||
expect(result.statusChanged).toBe(false);
|
||||
expect(result.monitor.status).toBe("down");
|
||||
});
|
||||
|
||||
it("logs a warning but still succeeds when running-stats update fails mid-flight", async () => {
|
||||
const monitor = makeMonitor({ statusWindow: [true, true, true, true], statusWindowSize: 5, status: "up" });
|
||||
const { service, logger, monitorsRepository, monitorStatsRepository } = createService();
|
||||
|
||||
Reference in new issue
Block a user