Enhancement: Cache and reuse keep-alive HTTP(S) agents (#6536)

src/utils/proxy/http.js

@@ -224,23 +224,43 @@ function homepageDNSLookupFn() {
   };
 }
 
+const homepageLookup = homepageDNSLookupFn();
+const agentCache = new Map();
+
+function getAgent(protocol, disableIpv6) {
+  const cacheKey = `${protocol}:${disableIpv6 ? "ipv4" : "auto"}`;
+  const cachedAgent = agentCache.get(cacheKey);
+  if (cachedAgent) {
+    return cachedAgent;
+  }
+
+  const agentOptions = {
+    keepAlive: true,
+    ...(disableIpv6 ? { family: 4, autoSelectFamily: false } : { autoSelectFamilyAttemptTimeout: 500 }),
+    lookup: homepageLookup,
+  };
+
+  const agent =
+    protocol === "https:"
+      ? new https.Agent({ ...agentOptions, rejectUnauthorized: false })
+      : new http.Agent(agentOptions);
+
+  agentCache.set(cacheKey, agent);
+  return agent;
+}
+
 export async function httpProxy(url, params = {}) {
   const constructedUrl = new URL(url);
   const disableIpv6 = process.env.HOMEPAGE_PROXY_DISABLE_IPV6 === "true";
-  const agentOptions = {
-    ...(disableIpv6 ? { family: 4, autoSelectFamily: false } : { autoSelectFamilyAttemptTimeout: 500 }),
-    lookup: homepageDNSLookupFn(),
-  };
-
   let request = null;
   if (constructedUrl.protocol === "https:") {
     request = httpsRequest(constructedUrl, {
-      agent: new https.Agent({ ...agentOptions, rejectUnauthorized: false }),
+      agent: getAgent(constructedUrl.protocol, disableIpv6),
       ...params,
     });
   } else {
     request = httpRequest(constructedUrl, {
-      agent: new http.Agent(agentOptions),
+      agent: getAgent(constructedUrl.protocol, disableIpv6),
       ...params,
     });
   }

src/utils/proxy/http.test.js

@@ -8,6 +8,7 @@ const { state, cache, logger, dns, net, cookieJar } = vi.hoisted(() => ({
       body: Buffer.from(""),
     },
     error: null,
+    lastAgent: null,
     lastAgentOptions: null,
     lastRequestParams: null,
     lastWrittenBody: null,
@@ -59,6 +60,7 @@ vi.mock("follow-redirects", async () => {
         state.lastWrittenBody = chunk;
       });
       req.end = vi.fn(() => {
+        state.lastAgent = params?.agent ?? null;
         state.lastAgentOptions = params?.agent?.opts ?? null;
         if (state.error) {
           req.emit("error", state.error);
@@ -104,6 +106,7 @@ describe("utils/proxy/http cachedRequest", () => {
       headers: { "content-type": "application/json" },
       body: Buffer.from(""),
     };
+    state.lastAgent = null;
     state.lastAgentOptions = null;
     state.lastRequestParams = null;
     state.lastWrittenBody = null;
@@ -307,6 +310,7 @@ describe("utils/proxy/http httpProxy", () => {
       headers: { "content-type": "application/json" },
       body: Buffer.from("ok"),
     };
+    state.lastAgent = null;
     state.lastAgentOptions = null;
     state.lastRequestParams = null;
     state.lastWrittenBody = null;
@@ -397,6 +401,7 @@ describe("utils/proxy/http httpProxy", () => {
 
     await httpMod.httpProxy("http://example.com");
 
+    expect(state.lastAgentOptions.keepAlive).toBe(true);
     expect(state.lastAgentOptions.family).toBe(4);
     expect(state.lastAgentOptions.autoSelectFamily).toBe(false);
   });
@@ -409,6 +414,17 @@ describe("utils/proxy/http httpProxy", () => {
     expect(state.lastAgentOptions.rejectUnauthorized).toBe(false);
   });
 
+  it("reuses the same keep-alive agent for repeated http requests", async () => {
+    const httpMod = await import("./http");
+
+    await httpMod.httpProxy("http://example.com/first");
+    const firstAgent = state.lastAgent;
+    await httpMod.httpProxy("http://example.com/second");
+
+    expect(state.lastAgentOptions.keepAlive).toBe(true);
+    expect(state.lastAgent).toBe(firstAgent);
+  });
+
   it("returns a sanitized error response when the request fails", async () => {
     state.error = Object.assign(new Error("boom"), { code: "EHOSTUNREACH" });
     const httpMod = await import("./http");