feat(server): add request timeout middleware to prevent hung connections

This commit is contained in:
Brad Groux 2026-01-29 04:33:53 -06:00
parent 5cfd9d88c2
commit 16d52c25e5
5 changed files with 433 additions and 0 deletions

View file

@ -1143,3 +1143,66 @@
{"type":"task.status_changed","taskId":"task_20260126_arch1","status":"blocked","previousStatus":"done","id":"evt_OWd9OwvFmOc_","timestamp":"2026-01-29T10:32:03.538Z"}
{"type":"task.archived","taskId":"task_20260126_arch1","status":"blocked","id":"evt_AlUAXFJQxDag","timestamp":"2026-01-29T10:32:03.540Z"}
{"type":"task.status_changed","taskId":"task_20260129_R0B5gI","status":"done","previousStatus":"todo","id":"evt_8teIusF6hd4O","timestamp":"2026-01-29T10:32:14.350Z"}
{"type":"task.created","taskId":"task_20260129_fm4h60","project":"my-project","status":"todo","id":"evt_UXdE2GMMIFWT","timestamp":"2026-01-29T10:33:33.771Z"}
{"type":"task.status_changed","taskId":"task_20260126_legacy1","status":"blocked","previousStatus":"review","id":"evt_Ki6oqOd4-Btu","timestamp":"2026-01-29T10:33:33.776Z"}
{"type":"task.created","taskId":"task_20260129_yjFpM0","status":"todo","id":"evt_2smhLKFps8xw","timestamp":"2026-01-29T10:33:33.777Z"}
{"type":"task.created","taskId":"task_20260129_Hyq6Jr","status":"todo","id":"evt_hZjDZVxmv6d1","timestamp":"2026-01-29T10:33:33.791Z"}
{"type":"task.created","taskId":"task_20260129_C7hpMu","status":"todo","id":"evt_ZGlBX-Eqg_hn","timestamp":"2026-01-29T10:33:33.795Z"}
{"type":"task.created","taskId":"task_20260129_dMib0D","status":"todo","id":"evt_KE2Gk8tiz0MP","timestamp":"2026-01-29T10:33:33.797Z"}
{"type":"task.created","taskId":"task_20260129__jPR_j","status":"todo","id":"evt_2OvTevPNoD3b","timestamp":"2026-01-29T10:33:33.801Z"}
{"type":"task.status_changed","taskId":"task_20260129_dMib0D","status":"in-progress","previousStatus":"todo","id":"evt__aanWNw6oAVw","timestamp":"2026-01-29T10:33:33.802Z"}
{"type":"task.created","taskId":"task_20260129_qBSxGa","status":"todo","id":"evt_hxLx2SxWOWMX","timestamp":"2026-01-29T10:33:33.804Z"}
{"type":"task.created","taskId":"task_20260129_ESWtxf","status":"todo","id":"evt_ZHPEW5p4pyzl","timestamp":"2026-01-29T10:33:33.805Z"}
{"type":"task.created","taskId":"task_20260129_BMRGLF","status":"todo","id":"evt_6acLz10UuRJ7","timestamp":"2026-01-29T10:33:33.805Z"}
{"type":"task.status_changed","taskId":"task_20260129_ESWtxf","status":"blocked","previousStatus":"todo","id":"evt_f_RczBBK8FSb","timestamp":"2026-01-29T10:33:33.807Z"}
{"type":"task.created","taskId":"task_20260129_FtlQUr","status":"todo","id":"evt_LRhKlVmlvP6A","timestamp":"2026-01-29T10:33:33.810Z"}
{"type":"task.status_changed","taskId":"task_20260129_FtlQUr","status":"done","previousStatus":"todo","id":"evt_TyNj43UblAQR","timestamp":"2026-01-29T10:33:33.812Z"}
{"type":"task.created","taskId":"task_20260129_66i8MI","status":"todo","id":"evt_ApxTz_Ypakz7","timestamp":"2026-01-29T10:33:33.818Z"}
{"type":"task.created","taskId":"task_20260129_OFI4II","status":"todo","id":"evt_CstZtNJD-SQb","timestamp":"2026-01-29T10:33:33.822Z"}
{"type":"task.status_changed","taskId":"task_20260126_idem1","status":"blocked","previousStatus":"review","id":"evt_fiKTyQ9RpyT5","timestamp":"2026-01-29T10:33:33.829Z"}
{"type":"task.status_changed","taskId":"task_20260129_66i8MI","status":"in-progress","previousStatus":"todo","id":"evt_BMbf1Jl9HjGG","timestamp":"2026-01-29T10:33:33.837Z"}
{"type":"task.created","taskId":"task_20260129_h2VSAn","project":"test-project","status":"todo","id":"evt_R4odCOTuqdGg","timestamp":"2026-01-29T10:33:33.843Z"}
{"type":"task.status_changed","taskId":"task_20260126_multi2","status":"blocked","previousStatus":"review","id":"evt_vtJDvT8DlQPN","timestamp":"2026-01-29T10:33:33.846Z"}
{"type":"task.status_changed","taskId":"task_20260126_multi1","status":"blocked","previousStatus":"review","id":"evt_E5hkfdHFwYuW","timestamp":"2026-01-29T10:33:33.849Z"}
{"type":"task.created","taskId":"task_20260129_dMHKHk","status":"todo","id":"evt_vyj6HocKG7Hp","timestamp":"2026-01-29T10:33:33.857Z"}
{"type":"task.created","taskId":"task_20260129_v0v_ne","status":"todo","id":"evt_aJjO3uppuvUj","timestamp":"2026-01-29T10:33:33.860Z"}
{"type":"task.created","taskId":"task_20260129_T4KR7z","status":"todo","id":"evt_gtv6iiFOjLfb","timestamp":"2026-01-29T10:33:33.861Z"}
{"type":"task.created","taskId":"task_20260129_SMgdmJ","status":"todo","id":"evt_ritLBthvus1u","timestamp":"2026-01-29T10:33:33.859Z"}
{"type":"task.created","taskId":"task_20260129_0LWtpl","status":"todo","id":"evt_6uHWYomXxxM7","timestamp":"2026-01-29T10:33:33.876Z"}
{"type":"task.created","taskId":"task_20260129_CQVlUO","status":"todo","id":"evt_0y8AHXVQWrwB","timestamp":"2026-01-29T10:33:33.876Z"}
{"type":"task.created","taskId":"task_20260129_MtMpCF","status":"todo","id":"evt_jGQ3dg_ky1DO","timestamp":"2026-01-29T10:33:33.921Z"}
{"type":"task.created","taskId":"task_20260129_75y1MC","status":"todo","id":"evt_JLXZUo6eNFw-","timestamp":"2026-01-29T10:33:33.923Z"}
{"type":"task.status_changed","taskId":"task_20260129_MtMpCF","status":"in-progress","previousStatus":"todo","id":"evt_k9Xc-X4zHUg7","timestamp":"2026-01-29T10:33:33.925Z"}
{"type":"task.created","taskId":"task_20260129_Rv3hI6","status":"todo","id":"evt_N1abO0M5bHXw","timestamp":"2026-01-29T10:33:33.927Z"}
{"type":"task.created","taskId":"task_20260129_4qO-Ya","status":"todo","id":"evt_-Rz_nOlzfYnd","timestamp":"2026-01-29T10:33:33.933Z"}
{"type":"task.created","taskId":"task_20260129_cadKov","status":"todo","id":"evt_sqf9y6LYS7sV","timestamp":"2026-01-29T10:33:33.935Z"}
{"type":"task.created","taskId":"task_20260129_LHf40j","status":"todo","id":"evt_-VONwbVK60sB","timestamp":"2026-01-29T10:33:33.948Z"}
{"type":"task.created","taskId":"task_20260129_cfEb8-","status":"todo","id":"evt_ZLKiHGTpzLfr","timestamp":"2026-01-29T10:33:33.956Z"}
{"type":"task.created","taskId":"task_20260129_WmIhed","status":"todo","id":"evt_KRxuj3zYhROE","timestamp":"2026-01-29T10:33:33.957Z"}
{"type":"task.created","taskId":"task_20260129_SV273U","status":"todo","id":"evt_z4u6xXRnp_YO","timestamp":"2026-01-29T10:33:33.959Z"}
{"type":"task.created","taskId":"task_20260129_yO12G_","status":"todo","id":"evt_OjaXRhm-y58j","timestamp":"2026-01-29T10:33:33.964Z"}
{"type":"task.created","taskId":"task_20260129_A8TE1n","status":"todo","id":"evt_XNTwjXN73dYh","timestamp":"2026-01-29T10:33:33.993Z"}
{"type":"task.created","taskId":"task_20260129_86rb77","status":"todo","id":"evt_YnOO5HsDOSSy","timestamp":"2026-01-29T10:33:33.997Z"}
{"type":"task.restored","taskId":"task_20260126_arch1","status":"done","id":"evt_oLkoYURYuSn1","timestamp":"2026-01-29T10:33:34.020Z"}
{"type":"task.status_changed","taskId":"task_20260126_arch1","status":"blocked","previousStatus":"done","id":"evt_FLOztAFgpE-_","timestamp":"2026-01-29T10:33:34.021Z"}
{"type":"task.archived","taskId":"task_20260126_arch1","status":"blocked","id":"evt_bLEe4qpYnCCU","timestamp":"2026-01-29T10:33:34.022Z"}
{"type":"task.created","taskId":"task_20260129_nMwnYU","status":"todo","id":"evt_FQQazz3PPjvk","timestamp":"2026-01-29T10:33:34.028Z"}
{"type":"task.archived","taskId":"task_20260129_nMwnYU","status":"todo","id":"evt_w1KGlvpPj0An","timestamp":"2026-01-29T10:33:34.031Z"}
{"type":"task.created","taskId":"task_20260129_ihD1Vl","status":"todo","id":"evt_fGYZdTkK0JsH","timestamp":"2026-01-29T10:33:34.033Z"}
{"type":"task.created","taskId":"task_20260129_bQECq-","status":"todo","id":"evt_bpeFP6dmBQRU","timestamp":"2026-01-29T10:33:34.051Z"}
{"type":"task.created","taskId":"task_20260129_9uM9UP","status":"todo","id":"evt_er4EgWGelUBN","timestamp":"2026-01-29T10:33:34.052Z"}
{"type":"task.created","taskId":"task_20260129_gH9ggX","status":"todo","id":"evt_t_4HQbtY_Zkj","timestamp":"2026-01-29T10:33:34.061Z"}
{"type":"task.created","taskId":"task_20260129_rK4D0Z","status":"todo","id":"evt_TKWqA-1g6LZG","timestamp":"2026-01-29T10:33:34.062Z"}
{"type":"task.created","taskId":"task_20260129_TTQNQh","status":"todo","id":"evt_qy30fKYvWQ2Q","timestamp":"2026-01-29T10:33:34.066Z"}
{"type":"task.created","taskId":"task_20260129_52sle1","status":"todo","id":"evt_dIBOIyTNxNUU","timestamp":"2026-01-29T10:33:34.067Z"}
{"type":"task.created","taskId":"task_20260129_Kmjex-","status":"todo","id":"evt_ra9w1uV3oCZc","timestamp":"2026-01-29T10:33:34.067Z"}
{"type":"task.created","taskId":"task_20260129_zKeFS7","status":"todo","id":"evt_enL6hrQPJoED","timestamp":"2026-01-29T10:33:34.076Z"}
{"type":"task.created","taskId":"task_20260129_gMurfl","status":"todo","id":"evt_b_Im-8Pu61og","timestamp":"2026-01-29T10:33:34.080Z"}
{"type":"task.created","taskId":"task_20260129_Rg0MWs","status":"todo","id":"evt_xZdsQbBYbk-p","timestamp":"2026-01-29T10:33:34.094Z"}
{"type":"task.created","taskId":"task_20260129_4dJvia","status":"todo","id":"evt_YrM-51f-p7Dk","timestamp":"2026-01-29T10:33:34.102Z"}
{"type":"task.created","taskId":"task_20260129_ctjT9o","status":"todo","id":"evt_WiIFs9gTWg02","timestamp":"2026-01-29T10:33:34.116Z"}
{"type":"task.created","taskId":"task_20260129_ty5XV3","project":"project-a","status":"todo","id":"evt__vTL1W0EY4Wv","timestamp":"2026-01-29T10:33:34.121Z"}
{"type":"task.created","taskId":"task_20260129_MvSnjn","project":"project-a","status":"todo","id":"evt_d9Zsc_HAwlNS","timestamp":"2026-01-29T10:33:34.122Z"}
{"type":"task.created","taskId":"task_20260129_WrgTN4","project":"project-b","status":"todo","id":"evt_SlDWSF__DcE3","timestamp":"2026-01-29T10:33:34.122Z"}
{"type":"task.created","taskId":"task_20260129_MPAR69","status":"todo","id":"evt_n5fRvj7bfa8X","timestamp":"2026-01-29T10:33:34.181Z"}
{"type":"task.status_changed","taskId":"task_20260129_hyFAp2","status":"done","previousStatus":"in-progress","id":"evt_mXnTwEr2rajA","timestamp":"2026-01-29T10:33:43.338Z"}

View file

@ -1,4 +1,26 @@
[
{
"id": "activity_1769682830010_ysgu6hsic",
"type": "comment_added",
"taskId": "task_20260129_hyFAp2",
"taskTitle": "STABILITY: Add request timeout middleware to prevent hung connections",
"details": {
"author": "Veritas",
"preview": "Added request timeout middleware (server/src/middl..."
},
"timestamp": "2026-01-29T10:33:50.010Z"
},
{
"id": "activity_1769682823339_croin6b3q",
"type": "status_changed",
"taskId": "task_20260129_hyFAp2",
"taskTitle": "STABILITY: Add request timeout middleware to prevent hung connections",
"details": {
"from": "in-progress",
"status": "done"
},
"timestamp": "2026-01-29T10:33:43.339Z"
},
{
"id": "activity_1769682740659_goqu3nhgt",
"type": "comment_added",

View file

@ -0,0 +1,274 @@
/**
* Request Timeout Middleware Tests
*
* Verifies that hung connections are terminated with 408 and that
* well-behaved requests, WebSocket upgrades, and streaming responses
* are left alone.
*/
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import type { Request, Response, NextFunction } from 'express';
import { requestTimeout } from '../../middleware/request-timeout.js';
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
type EventMap = Record<string, Array<() => void>>;
function mockRequest(
headers: Record<string, string | string[] | undefined> = {},
overrides: Record<string, unknown> = {}
): Request {
return {
headers,
originalUrl: '/api/v1/tasks',
url: '/api/v1/tasks',
path: '/api/v1/tasks',
destroy: vi.fn(),
...overrides,
} as unknown as Request;
}
interface MockRes extends Response {
_statusCode: number;
_json: unknown;
_events: EventMap;
_emit: (event: string) => void;
}
function mockResponse(): MockRes {
const events: EventMap = {};
const res = {
_statusCode: 0,
_json: null,
_events: events,
headersSent: false,
locals: {},
status(code: number) {
res._statusCode = code;
return res;
},
json(body: unknown) {
res._json = body;
return res;
},
on(event: string, cb: () => void) {
if (!events[event]) events[event] = [];
events[event].push(cb);
return res;
},
/** Simulate emitting a response event (finish / close) */
_emit(event: string) {
(events[event] || []).forEach((cb) => cb());
},
} as unknown as MockRes;
return res;
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
describe('Request Timeout Middleware', () => {
beforeEach(() => {
vi.useFakeTimers();
});
afterEach(() => {
vi.useRealTimers();
});
// -----------------------------------------------------------------------
// Normal operation
// -----------------------------------------------------------------------
it('should call next() immediately and not block the request', () => {
const middleware = requestTimeout();
const req = mockRequest();
const res = mockResponse();
const next: NextFunction = vi.fn();
middleware(req, res, next);
expect(next).toHaveBeenCalledOnce();
});
it('should not send a timeout response when the request completes within the limit', () => {
const middleware = requestTimeout(5_000);
const req = mockRequest();
const res = mockResponse();
const next: NextFunction = vi.fn();
middleware(req, res, next);
// Simulate the response finishing before the timeout
res._emit('finish');
// Advance past the timeout window
vi.advanceTimersByTime(6_000);
expect(res._statusCode).toBe(0);
expect(res._json).toBeNull();
expect(req.destroy).not.toHaveBeenCalled();
});
// -----------------------------------------------------------------------
// Timeout behaviour
// -----------------------------------------------------------------------
it('should respond with 408 when the default 30 s timeout is exceeded', () => {
const middleware = requestTimeout();
const req = mockRequest();
const res = mockResponse();
const next: NextFunction = vi.fn();
middleware(req, res, next);
vi.advanceTimersByTime(30_000);
expect(res._statusCode).toBe(408);
expect(res._json).toEqual({
error: 'Request Timeout',
message: 'Request exceeded the 30s timeout',
code: 'REQUEST_TIMEOUT',
});
expect(req.destroy).toHaveBeenCalledOnce();
});
it('should respect a custom timeout value', () => {
const middleware = requestTimeout(10_000);
const req = mockRequest();
const res = mockResponse();
const next: NextFunction = vi.fn();
middleware(req, res, next);
// Not timed out yet
vi.advanceTimersByTime(9_999);
expect(res._statusCode).toBe(0);
// Now it fires
vi.advanceTimersByTime(1);
expect(res._statusCode).toBe(408);
expect(res._json).toEqual({
error: 'Request Timeout',
message: 'Request exceeded the 10s timeout',
code: 'REQUEST_TIMEOUT',
});
});
// -----------------------------------------------------------------------
// Upload / attachment routes get 120 s
// -----------------------------------------------------------------------
it('should use 120 s timeout for attachment upload routes', () => {
const middleware = requestTimeout(); // default 30 s
const req = mockRequest(
{},
{
originalUrl: '/api/v1/tasks/task_abc123/attachments',
url: '/api/v1/tasks/task_abc123/attachments',
path: '/tasks/task_abc123/attachments',
}
);
const res = mockResponse();
const next: NextFunction = vi.fn();
middleware(req, res, next);
// Should NOT fire at the 30 s mark
vi.advanceTimersByTime(30_000);
expect(res._statusCode).toBe(0);
// Should fire at 120 s
vi.advanceTimersByTime(90_000);
expect(res._statusCode).toBe(408);
expect(res._json).toEqual({
error: 'Request Timeout',
message: 'Request exceeded the 120s timeout',
code: 'REQUEST_TIMEOUT',
});
});
// -----------------------------------------------------------------------
// WebSocket upgrade — skip
// -----------------------------------------------------------------------
it('should skip WebSocket upgrade requests', () => {
const middleware = requestTimeout(100);
const req = mockRequest({ upgrade: 'websocket' });
const res = mockResponse();
const next: NextFunction = vi.fn();
middleware(req, res, next);
expect(next).toHaveBeenCalledOnce();
vi.advanceTimersByTime(200);
// No timeout response should have been sent
expect(res._statusCode).toBe(0);
expect(req.destroy).not.toHaveBeenCalled();
});
it('should skip WebSocket upgrade requests (case-insensitive)', () => {
const middleware = requestTimeout(100);
const req = mockRequest({ upgrade: 'WebSocket' });
const res = mockResponse();
const next: NextFunction = vi.fn();
middleware(req, res, next);
vi.advanceTimersByTime(200);
expect(res._statusCode).toBe(0);
});
// -----------------------------------------------------------------------
// SSE / streaming — skip if headers already sent
// -----------------------------------------------------------------------
it('should not send timeout response if headers are already sent (SSE/streaming)', () => {
const middleware = requestTimeout(100);
const req = mockRequest();
const res = mockResponse();
const next: NextFunction = vi.fn();
middleware(req, res, next);
// Simulate headers already flushed (e.g. SSE or chunked transfer)
(res as unknown as { headersSent: boolean }).headersSent = true;
vi.advanceTimersByTime(100);
// No duplicate response, no socket destruction
expect(res._statusCode).toBe(0);
expect(req.destroy).not.toHaveBeenCalled();
});
// -----------------------------------------------------------------------
// Cleanup on client disconnect
// -----------------------------------------------------------------------
it('should clear timeout when the client disconnects (close event)', () => {
const middleware = requestTimeout(5_000);
const req = mockRequest();
const res = mockResponse();
const next: NextFunction = vi.fn();
middleware(req, res, next);
// Client drops the connection
res._emit('close');
vi.advanceTimersByTime(5_000);
expect(res._statusCode).toBe(0);
expect(req.destroy).not.toHaveBeenCalled();
});
});

View file

@ -30,6 +30,7 @@ import { initBroadcast } from './services/broadcast-service.js';
import { runStartupMigrations } from './services/migration-service.js';
import { errorHandler, AppError } from './middleware/error-handler.js';
import { requestIdMiddleware } from './middleware/request-id.js';
import { requestTimeout } from './middleware/request-timeout.js';
import {
authenticate,
authorize,
@ -266,6 +267,14 @@ const corsOptions: cors.CorsOptions = {
// to all downstream middleware and route handlers.
app.use(requestIdMiddleware);
// ============================================
// Stability: Request Timeout (30 s default, 120 s uploads)
// ============================================
// Prevents hung connections from piling up and exhausting server
// resources. Must be registered after request-id (so timeout
// responses include the trace ID) and before routes.
app.use(requestTimeout());
// Middleware
app.use(cors(corsOptions));
app.use(cookieParser());

View file

@ -0,0 +1,65 @@
import { Request, Response, NextFunction } from 'express';
/**
* Request Timeout Middleware
*
* Enforces a maximum duration for each request. If the handler hasn't
* finished within the allotted time, the client receives a 408 response
* and the underlying socket is destroyed to free resources.
*
* Upload/attachment routes automatically receive a longer timeout (120 s)
* since file transfers are expected to take more time.
*
* Skipped for:
* - WebSocket upgrade requests (long-lived by design)
* - Responses that have already begun streaming (headersSent === true)
*
* Usage:
* app.use(requestTimeout()); // 30 s default
* app.use(requestTimeout(60_000)); // custom 60 s
*/
const DEFAULT_TIMEOUT_MS = 30_000;
const UPLOAD_TIMEOUT_MS = 120_000;
/** Matches attachment/upload routes: /tasks/:id/attachments */
const UPLOAD_PATH_RE = /\/tasks\/[^/]+\/attachments/;
export function requestTimeout(ms: number = DEFAULT_TIMEOUT_MS) {
return (req: Request, res: Response, next: NextFunction): void => {
// Skip WebSocket upgrade requests — they are long-lived by design
if (req.headers.upgrade && req.headers.upgrade.toLowerCase() === 'websocket') {
next();
return;
}
// Use longer timeout for upload/attachment routes
const url = req.originalUrl || req.url || req.path;
const effectiveMs = UPLOAD_PATH_RE.test(url) ? UPLOAD_TIMEOUT_MS : ms;
const timeoutId = setTimeout(() => {
// Don't send a response if headers were already sent (SSE / streaming)
if (res.headersSent) {
return;
}
const seconds = Math.round(effectiveMs / 1000);
res.status(408).json({
error: 'Request Timeout',
message: `Request exceeded the ${seconds}s timeout`,
code: 'REQUEST_TIMEOUT',
});
// Tear down the underlying socket to release resources
req.destroy();
}, effectiveMs);
// Clear the timeout when the response finishes normally
const cleanup = () => clearTimeout(timeoutId);
res.on('finish', cleanup);
res.on('close', cleanup);
next();
};
}