Merge pull request #3734 from bluewave-labs/fix/maintenance-window-exit
fix: change status on exit maintenance window
This commit is contained in:
4 files changed
+54
-4
No files matched your search
@@ -232,10 +232,10 @@ export class StatusService implements IStatusService {
|
|||||||
// Build the status patch — computed against the projected window
|
// Build the status patch — computed against the projected window
|
||||||
const patch: Partial<Monitor> = {};
|
const patch: Partial<Monitor> = {};
|
||||||
|
|
||||||
// Resolve "initializing" up-front, incidents should be created on initialization && down
|
// Resolve "initializing" and "maintenance" up front, incidents should be created on initialization && down
|
||||||
let newStatus: MonitorStatus = monitor.status;
|
let newStatus: MonitorStatus = monitor.status;
|
||||||
let statusChanged = false;
|
let statusChanged = false;
|
||||||
if (monitor.status === "initializing") {
|
if (monitor.status === "initializing" || monitor.status === "maintenance") {
|
||||||
newStatus = status === true ? "up" : "down";
|
newStatus = status === true ? "up" : "down";
|
||||||
patch.status = newStatus;
|
patch.status = newStatus;
|
||||||
statusChanged = newStatus === "down";
|
statusChanged = newStatus === "down";
|
||||||
|
|||||||
@@ -49,7 +49,8 @@ export class CheckProducer implements ICheckProducer {
|
|||||||
method: "produce",
|
method: "produce",
|
||||||
});
|
});
|
||||||
if (monitor.status !== "maintenance") {
|
if (monitor.status !== "maintenance") {
|
||||||
await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance" });
|
// Clear the status window to avoid incidents being created on next check
|
||||||
|
await this.monitorsRepository.updateById(monitor.id, monitor.teamId, { status: "maintenance", statusWindow: [] });
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -67,7 +67,7 @@ describe("CheckPipeline", () => {
|
|||||||
|
|
||||||
expect(result).toBeNull();
|
expect(result).toBeNull();
|
||||||
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
expect(defaults.networkService.requestStatus).not.toHaveBeenCalled();
|
||||||
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance" });
|
expect(defaults.monitorsRepository.updateById).toHaveBeenCalledWith("m1", "team", { status: "maintenance", statusWindow: [] });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not re-write status when already in maintenance", async () => {
|
it("does not re-write status when already in maintenance", async () => {
|
||||||
|
|||||||
@@ -557,6 +557,55 @@ describe("StatusService", () => {
|
|||||||
expect(result.prevStatus).toBe("initializing");
|
expect(result.prevStatus).toBe("initializing");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("leaves 'maintenance' for 'up' on a passing check after the window closes (empty statusWindow)", async () => {
|
||||||
|
// On entry to a maintenance window the producer clears statusWindow, so the first
|
||||||
|
// check after the window closes sees an empty window and lands in the warmup
|
||||||
|
// early-return branch. A passing check must move the monitor out of 'maintenance'
|
||||||
|
// to 'up' without flagging a status change (no incident).
|
||||||
|
const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "maintenance" });
|
||||||
|
const { service, monitorsRepository } = createService();
|
||||||
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||||
|
|
||||||
|
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
|
||||||
|
|
||||||
|
expect(result.statusChanged).toBe(false);
|
||||||
|
expect(result.monitor.status).toBe("up");
|
||||||
|
expect(result.prevStatus).toBe("maintenance");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves 'maintenance' for 'down' on a failing check after the window closes (empty statusWindow)", async () => {
|
||||||
|
// Mirror of the recovery case: a failing first check after the window closes must
|
||||||
|
// flip the monitor straight to 'down' and surface statusChanged so an incident opens,
|
||||||
|
// rather than leave it stuck in 'maintenance'.
|
||||||
|
const monitor = makeMonitor({ statusWindow: [], statusWindowSize: 5, status: "maintenance" });
|
||||||
|
const { service, monitorsRepository } = createService();
|
||||||
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||||
|
|
||||||
|
const result = await service.updateMonitorStatus(makeStatusResponse({ status: false }), makeCheck({ status: false }));
|
||||||
|
|
||||||
|
expect(result.statusChanged).toBe(true);
|
||||||
|
expect(result.monitor.status).toBe("down");
|
||||||
|
expect(result.prevStatus).toBe("maintenance");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("unsticks from 'maintenance' to 'up' on a passing check even with a full window (regression: stuck in maintenance)", async () => {
|
||||||
|
// Guards the up-front override directly: even if a full window survives (e.g. the
|
||||||
|
// entry-clear is bypassed), computeReachability only transitions out to 'up' from
|
||||||
|
// 'down', so without the override the monitor would stay stuck in 'maintenance'.
|
||||||
|
const monitor = makeMonitor({
|
||||||
|
statusWindow: [true, true, true, true, true],
|
||||||
|
statusWindowSize: 5,
|
||||||
|
statusWindowThreshold: 80,
|
||||||
|
status: "maintenance",
|
||||||
|
});
|
||||||
|
const { service, monitorsRepository } = createService();
|
||||||
|
(monitorsRepository.findById as jest.Mock).mockResolvedValue(monitor);
|
||||||
|
|
||||||
|
const result = await service.updateMonitorStatus(makeStatusResponse({ status: true }), makeCheck({ status: true }));
|
||||||
|
|
||||||
|
expect(result.monitor.status).toBe("up");
|
||||||
|
});
|
||||||
|
|
||||||
it("does not flip status on subsequent checks during warmup once 'initializing' has been left (regression: down→up incident resolves once window fills)", async () => {
|
it("does not flip status on subsequent checks during warmup once 'initializing' has been left (regression: down→up incident resolves once window fills)", async () => {
|
||||||
// After a 'initializing'→'down' transition on check #1, monitor.status is now 'down'
|
// After a 'initializing'→'down' transition on check #1, monitor.status is now 'down'
|
||||||
// and statusWindow has one false. Subsequent up checks during warmup must leave
|
// and statusWindow has one false. Subsequent up checks during warmup must leave
|
||||||
|
|||||||
Reference in new issue
Block a user