Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions lib/cluster/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,7 @@ class Cluster<
this.status
);
reject(new RedisError("Connection is aborted"));
this.handleAbortedConnect();
return;
}
this.connectionPool.reset(nodes);
Expand Down Expand Up @@ -1049,6 +1050,28 @@ class Cluster<
});
}

/**
* Called when connect() gives up because the status changed while it was
* resolving the startup nodes. connect() has not registered its "close"
* listeners yet, so in two cases nothing else would move the cluster out of
* its current status:
* - "close": the late "end" of a node removed by a previous disconnection
* made the pool emit "drain";
* - "disconnecting" with an empty pool: disconnect() or quit() was called,
* and no node is left to emit "drain".
*/
private handleAbortedConnect(): void {
if (
this.status === "disconnecting" &&
!this.connectionPool.getNodes().length
) {
this.setStatus("close");
}
if (this.status === "close") {
this.handleCloseEvent();
}
}
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved

/**
* Called when closed to check whether a reconnection should be made
*/
Expand Down
30 changes: 29 additions & 1 deletion test/functional/cluster/dnsLookup.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import MockServer from "../../helpers/mock_server";
import { Cluster } from "../../../lib";
import { expect } from "chai";
import { expect, use } from "chai";

use(require("chai-as-promised"));

describe("cluster:dnsLookup", () => {
it("resolve hostnames to IPs", (done) => {
Expand Down Expand Up @@ -111,4 +113,30 @@ describe("cluster:dnsLookup", () => {
done();
});
});

["disconnect", "quit"].forEach((method) => {
it(`ends when ${method}() is called while resolving the startup nodes`, async () => {
const slotTable = [[0, 16383, ["127.0.0.1", 30001]]];
new MockServer(30001, () => {}, slotTable);

let resolveLookup: () => void;
const cluster = new Cluster([{ host: "a.com", port: "30001" }], {
dnsLookup(_, callback) {
resolveLookup = () => callback(null, "127.0.0.1");
},
});
const pending = cluster.get("foo");
const end = new Promise((resolve) => cluster.once("end", resolve));

await new Promise((resolve) => setImmediate(resolve));
cluster[method]();
resolveLookup();

await end;
expect(cluster.status).to.eql("end");
await expect(pending).to.be.rejectedWith(
"None of startup nodes is available"
);
});
});
});
109 changes: 109 additions & 0 deletions test/functional/cluster/lateNodeEnd.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
import MockServer from "../../helpers/mock_server";
import { expect } from "chai";
import { Cluster } from "../../../lib";

const DISCONNECT_TIMEOUT = 200;

/**
* Resolves on the first `event`, or rejects after `ms` with the cluster state.
*/
function waitFor(cluster: Cluster, event: string, ms: number) {
return new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => {
cluster.removeListener(event, onEvent);
reject(
new Error(
`no "${event}" within ${ms}ms: status=${cluster.status}, ` +
// @ts-expect-error
`reconnectTimeout=${Boolean(cluster.reconnectTimeout)}, ` +
// @ts-expect-error
`offlineQueue=${cluster.offlineQueue.length}`
)
);
}, ms);
const onEvent = () => {
clearTimeout(timer);
resolve();
};
cluster.once(event, onEvent);
});
}

describe("cluster:late node end", () => {
const PORT_NODE_1 = 30001;
const PORT_NODE_2 = 30002;
const slotTable = [
[0, 8000, ["127.0.0.1", PORT_NODE_1]],
[8001, 16383, ["127.0.0.1", PORT_NODE_2]],
];

it("should reconnect when a node ends while resolving the startup nodes", async () => {
// [Setup] Configure Redis servers (2 mock nodes)
// The first ready check fails, and the server keeps that socket half-open
let readyChecks = 0;
let slowPort: number | undefined;
const redisServerHandler = (argv, c) => {
if (argv[0] === "cluster" && argv[1].toLowerCase() === "info") {
readyChecks++;
// First ready check → failure
if (readyChecks === 1) {
slowPort = c.localPort;
c.allowHalfOpen = true;
return "cluster_state:fail";
}
// Other ready checks → success
return "cluster_state:ok";
}
if (argv[0] === "get") {
return "bar";
}
};
new MockServer(PORT_NODE_1, redisServerHandler, slotTable);
new MockServer(PORT_NODE_2, redisServerHandler, slotTable);

// [Setup] Configure DNS lookup
// The lookup made by the reconnection only resolves once the slow node has ended.
let lookups = 0;
let slowNodeEndedDuringLookup = false;
const cluster = new Cluster(
[
{
host: "redis.test", // Use a hostname instead of an IP to force one dnsLookup() per connect()
port: PORT_NODE_1,
},
],
{
redisOptions: { disconnectTimeout: DISCONNECT_TIMEOUT },
dnsLookup(hostname, callback) {
lookups++;

// Only the 2nd lookup resolves late (after the slow node has ended)
if (lookups === 2) {
const onNodeEnd = (redis) => {
if (redis.options.port !== slowPort) return;
cluster.removeListener("-node", onNodeEnd);
slowNodeEndedDuringLookup = true;
setImmediate(() => callback(null, "127.0.0.1"));
};
cluster.on("-node", onNodeEnd);
} else {
callback(null, "127.0.0.1");
}
},
}
);

// Sent during the incident: queued until the cluster is ready.
const pending = cluster.get("foo");

try {
await waitFor(cluster, "ready", 5 * DISCONNECT_TIMEOUT);

expect(slowNodeEndedDuringLookup).to.eql(true);
expect(readyChecks).to.eql(2);
expect(await pending).to.eql("bar");
} finally {
cluster.disconnect();
}
});
});