From 344b93b6b2c1f5d2fb8cc632ee7af8d08c18f62f Mon Sep 17 00:00:00 2001 From: Jacob Ley Date: Thu, 22 Oct 2026 01:42:23 +0000 Subject: [PATCH 1/4] fix(cluster): vendor cluster-key-slot to fix empty hash tag slot bug Valkey treats an empty hash tag ("{}") as no tag and hashes the whole key. cluster-key-slot keeps scanning for the next "}" instead, so "foo{}{bar}" maps to slot 4015 (the slot of "{bar") rather than 8363, and commands for such keys go to the wrong node and come back MOVED. The package's repository is archived, so it cannot be fixed upstream. Vendor the implementation into lib/utils/calculateSlot.ts with the fix, point Command, Pipeline and autoPipelining at it, and drop the dependency. Ported from redis/ioredis#2172. Signed-off-by: Jacob Ley --- benchmarks/fixtures/generate.ts | 2 +- lib/Command.ts | 2 +- lib/Pipeline.ts | 2 +- lib/autoPipelining.ts | 2 +- lib/utils/calculateSlot.ts | 189 +++++++++++++++++++++ package-lock.json | 9 +- package.json | 1 - test/functional/cluster/ask.ts | 2 +- test/functional/cluster/autopipelining.ts | 2 +- test/functional/cluster/maxRedirections.ts | 2 +- test/functional/cluster/moved.ts | 2 +- test/functional/cluster/nat.ts | 2 +- test/functional/cluster/pipeline.ts | 2 +- test/functional/cluster/transaction.ts | 2 +- test/unit/calculateSlot.ts | 120 +++++++++++++ 15 files changed, 325 insertions(+), 16 deletions(-) create mode 100644 lib/utils/calculateSlot.ts create mode 100644 test/unit/calculateSlot.ts diff --git a/benchmarks/fixtures/generate.ts b/benchmarks/fixtures/generate.ts index 21f0adf4..e6101ce1 100644 --- a/benchmarks/fixtures/generate.ts +++ b/benchmarks/fixtures/generate.ts @@ -2,7 +2,7 @@ const start = process.hrtime.bigint(); -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "../../lib/utils/calculateSlot"; import { writeFileSync } from "fs"; import { join } from "path"; import { v4 as uuid } from "uuid"; diff --git a/lib/Command.ts b/lib/Command.ts index 4e13bbdb..9c592ec7 100644 --- a/lib/Command.ts +++ b/lib/Command.ts @@ -1,5 +1,5 @@ import { exists, getKeyIndexes } from "@iovalkey/commands"; -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "./utils/calculateSlot"; import asCallback from "standard-as-callback"; import { convertBufferToString, diff --git a/lib/Pipeline.ts b/lib/Pipeline.ts index 49af22f3..8176f0d1 100644 --- a/lib/Pipeline.ts +++ b/lib/Pipeline.ts @@ -1,4 +1,4 @@ -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "./utils/calculateSlot"; import { exists, hasFlag } from "@iovalkey/commands"; import asCallback from "standard-as-callback"; import { deprecate } from "util"; diff --git a/lib/autoPipelining.ts b/lib/autoPipelining.ts index 68adc2a4..43301680 100644 --- a/lib/autoPipelining.ts +++ b/lib/autoPipelining.ts @@ -1,5 +1,5 @@ import { isArguments, noop } from "./utils/lodash"; -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "./utils/calculateSlot"; import asCallback from "standard-as-callback"; import { ArgumentType } from "./Command"; diff --git a/lib/utils/calculateSlot.ts b/lib/utils/calculateSlot.ts new file mode 100644 index 00000000..bb73dd91 --- /dev/null +++ b/lib/utils/calculateSlot.ts @@ -0,0 +1,189 @@ +/* + * Vendored from `cluster-key-slot@1.1.1` + * (https://github.com/invertase/cluster-key-slot), patched to fix a + * disagreement with real Redis servers over how an empty hash tag ("{}") is + * handled. + * + * Redis treats an empty hash tag as no tag at all: given `{}` it hashes the + * whole key and stops searching (see `keyHashSlot()` in Redis's + * `src/cluster.c`). The upstream single-pass scanner instead falls through + * and keeps scanning for the next `}`, hashing whatever sits between -- which + * can silently misroute commands. Vendoring was necessary because the + * upstream repository has been archived and is no longer maintained. Ported + * from ioredis (https://github.com/redis/ioredis/pull/2172). See + * https://github.com/mishan/intraslot/blob/main/contrib/empty-hashtag-slot-bug/README.md + * for the full writeup, live-server measurements, and suggested test cases. + * + * The fix, marked below, latches the whole-key fallback once the empty-tag + * case is seen: one extra boolean, no extra pass, no change to the hot path + * for keys without braces. + * + * Original copyright notices, preserved from upstream: + * + * Copyright 2001-2010 Georges Menie (www.menie.org) + * Copyright 2010 Salvatore Sanfilippo (adapted to Redis coding style) + * Copyright 2015 Zihua Li (http://zihua.li) (ported to JavaScript) + * Copyright 2016 Mike Diarmid (http://github.com/salakar) (re-write for + * performance, ~700% perf inc) + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* CRC16 implementation according to CCITT standards. + * + * Note by @antirez: this is actually the XMODEM CRC 16 algorithm, using the + * following parameters: + * + * Name : "XMODEM", also known as "ZMODEM", "CRC-16/ACORN" + * Width : 16 bit + * Poly : 1021 (That is actually x^16 + x^12 + x^5 + 1) + * Initialization : 0000 + * Reflect Input byte : False + * Reflect Output CRC : False + * Xor constant to output CRC : 0000 + * Output for "123456789" : 31C3 + */ +const lookup = [ + 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, + 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, 0x1231, 0x0210, + 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, 0x9339, 0x8318, 0xb37b, + 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, 0x2462, 0x3443, 0x0420, 0x1401, + 0x64e6, 0x74c7, 0x44a4, 0x5485, 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, + 0xf5cf, 0xc5ac, 0xd58d, 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, + 0x5695, 0x46b4, 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, + 0xc7bc, 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, + 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, 0x5af5, + 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, 0xdbfd, 0xcbdc, + 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, 0x6ca6, 0x7c87, 0x4ce4, + 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, 0xedae, 0xfd8f, 0xcdec, 0xddcd, + 0xad2a, 0xbd0b, 0x8d68, 0x9d49, 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, + 0x2e32, 0x1e51, 0x0e70, 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, + 0x9f59, 0x8f78, 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, + 0xe16f, 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, + 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, 0x02b1, + 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, 0xb5ea, 0xa5cb, + 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, 0x34e2, 0x24c3, 0x14a0, + 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, 0xa7db, 0xb7fa, 0x8799, 0x97b8, + 0xe75f, 0xf77e, 0xc71d, 0xd73c, 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, + 0x7676, 0x4615, 0x5634, 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, + 0xb98a, 0xa9ab, 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, + 0x28a3, 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, + 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, 0xfd2e, + 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, 0x7c26, 0x6c07, + 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, 0xef1f, 0xff3e, 0xcf5d, + 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 0x6e17, 0x7e36, 0x4e55, 0x5e74, + 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0, +]; + +/** + * Convert a string to a UTF8 array - faster than via buffer + */ +function toUTF8Array(str: string): number[] { + let char: number; + let i = 0; + let p = 0; + const utf8: number[] = []; + const len = str.length; + + for (; i < len; i++) { + char = str.charCodeAt(i); + if (char < 128) { + utf8[p++] = char; + } else if (char < 2048) { + utf8[p++] = (char >> 6) | 192; + utf8[p++] = (char & 63) | 128; + } else if ( + (char & 0xfc00) === 0xd800 && + i + 1 < str.length && + (str.charCodeAt(i + 1) & 0xfc00) === 0xdc00 + ) { + char = 0x10000 + ((char & 0x03ff) << 10) + (str.charCodeAt(++i) & 0x03ff); + utf8[p++] = (char >> 18) | 240; + utf8[p++] = ((char >> 12) & 63) | 128; + utf8[p++] = ((char >> 6) & 63) | 128; + utf8[p++] = (char & 63) | 128; + } else { + utf8[p++] = (char >> 12) | 224; + utf8[p++] = ((char >> 6) & 63) | 128; + utf8[p++] = (char & 63) | 128; + } + } + + return utf8; +} + +/** + * Convert a string into a redis slot hash. + */ +function generate(str: string | ArrayLike): number { + let char: number; + let i = 0; + let start = -1; + // Latched once an empty hash tag ("{}") is seen: Redis hashes the whole + // key and stops looking for a tag in that case, so once `done` is set we + // must stop treating any later "{" as the start of a new tag. + let done = false; + let result = 0; + let resultHash = 0; + const utf8 = typeof str === "string" ? toUTF8Array(str) : str; + const len = utf8.length; + + while (i < len) { + char = utf8[i++]; + if (done || start === -1) { + if (!done && char === 0x7b) { + start = i; + } + } else if (char !== 0x7d) { + resultHash = + lookup[(char ^ (resultHash >> 8)) & 0xff] ^ (resultHash << 8); + } else if (i - 1 !== start) { + return resultHash & 0x3fff; + } else { + // Empty hash tag ("{}"): Redis falls back to hashing the whole key. + done = true; + } + + result = lookup[(char ^ (result >> 8)) & 0xff] ^ (result << 8); + } + + return result & 0x3fff; +} + +interface CalculateSlot { + (str: string | ArrayLike): number; + /** + * Convert an array of multiple strings into a redis slot hash. + * Returns -1 if one of the keys is not for the same slot as the others + */ + generateMulti(keys: (string | ArrayLike)[]): number; +} + +const calculateSlot = generate as CalculateSlot; + +calculateSlot.generateMulti = function generateMulti( + keys: (string | ArrayLike)[] +): number { + let i = 1; + const len = keys.length; + const base = generate(keys[0]); + + while (i < len) { + if (generate(keys[i++]) !== base) return -1; + } + + return base; +}; + +export = calculateSlot; diff --git a/package-lock.json b/package-lock.json index 0c346b2d..f7e09f72 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,15 @@ { "name": "iovalkey", - "version": "0.3.3", + "version": "0.4.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "iovalkey", - "version": "0.3.3", + "version": "0.4.0", "license": "MIT", "dependencies": { "@iovalkey/commands": "^0.1.0", - "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", @@ -1547,6 +1546,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==", + "dev": true, "engines": { "node": ">=0.10.0" } @@ -6076,7 +6076,8 @@ "cluster-key-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", - "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==", + "dev": true }, "color-convert": { "version": "2.0.1", diff --git a/package.json b/package.json index d1839b27..0fe9dd13 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,6 @@ "license": "MIT", "dependencies": { "@iovalkey/commands": "^0.1.0", - "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", diff --git a/test/functional/cluster/ask.ts b/test/functional/cluster/ask.ts index 2ff647a1..d5d98cbb 100644 --- a/test/functional/cluster/ask.ts +++ b/test/functional/cluster/ask.ts @@ -1,5 +1,5 @@ import MockServer from "../../helpers/mock_server"; -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "../../../lib/utils/calculateSlot"; import { expect } from "chai"; import { Cluster } from "../../../lib"; diff --git a/test/functional/cluster/autopipelining.ts b/test/functional/cluster/autopipelining.ts index 2dcc63c0..03edef9c 100644 --- a/test/functional/cluster/autopipelining.ts +++ b/test/functional/cluster/autopipelining.ts @@ -1,5 +1,5 @@ import { expect, use } from "chai"; -import * as calculateKeySlot from "cluster-key-slot"; +import * as calculateKeySlot from "../../../lib/utils/calculateSlot"; import { default as Cluster } from "../../../lib/cluster"; import MockServer from "../../helpers/mock_server"; diff --git a/test/functional/cluster/maxRedirections.ts b/test/functional/cluster/maxRedirections.ts index 183a6282..01171654 100644 --- a/test/functional/cluster/maxRedirections.ts +++ b/test/functional/cluster/maxRedirections.ts @@ -1,4 +1,4 @@ -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "../../../lib/utils/calculateSlot"; import MockServer from "../../helpers/mock_server"; import { expect } from "chai"; import { Cluster } from "../../../lib"; diff --git a/test/functional/cluster/moved.ts b/test/functional/cluster/moved.ts index 14082e36..0523ead4 100644 --- a/test/functional/cluster/moved.ts +++ b/test/functional/cluster/moved.ts @@ -1,4 +1,4 @@ -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "../../../lib/utils/calculateSlot"; import MockServer from "../../helpers/mock_server"; import { expect } from "chai"; import { Cluster } from "../../../lib"; diff --git a/test/functional/cluster/nat.ts b/test/functional/cluster/nat.ts index 10c845b8..238bf43b 100644 --- a/test/functional/cluster/nat.ts +++ b/test/functional/cluster/nat.ts @@ -1,4 +1,4 @@ -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "../../../lib/utils/calculateSlot"; import MockServer from "../../helpers/mock_server"; import { expect } from "chai"; import { Cluster } from "../../../lib"; diff --git a/test/functional/cluster/pipeline.ts b/test/functional/cluster/pipeline.ts index 2bc50a09..cf99bd08 100644 --- a/test/functional/cluster/pipeline.ts +++ b/test/functional/cluster/pipeline.ts @@ -1,4 +1,4 @@ -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "../../../lib/utils/calculateSlot"; import MockServer from "../../helpers/mock_server"; import { expect } from "chai"; import { Cluster } from "../../../lib"; diff --git a/test/functional/cluster/transaction.ts b/test/functional/cluster/transaction.ts index 17d74022..3555f152 100644 --- a/test/functional/cluster/transaction.ts +++ b/test/functional/cluster/transaction.ts @@ -1,4 +1,4 @@ -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "../../../lib/utils/calculateSlot"; import MockServer from "../../helpers/mock_server"; import { expect } from "chai"; import { Cluster } from "../../../lib"; diff --git a/test/unit/calculateSlot.ts b/test/unit/calculateSlot.ts new file mode 100644 index 00000000..c0788c4d --- /dev/null +++ b/test/unit/calculateSlot.ts @@ -0,0 +1,120 @@ +import { expect } from "chai"; +import * as calculateSlot from "../../lib/utils/calculateSlot"; + +describe("calculateSlot", () => { + describe("empty hash tag handling", () => { + // Redis treats an empty hash tag ("{}") as no tag at all: it hashes the + // whole key and stops searching for a tag. The unpatched upstream + // scanner instead falls through on an empty tag without clearing its + // "in a tag" state, so it keeps scanning and latches onto the next "}" + // in the key, hashing whatever sits between them instead of the whole + // key. These values were measured against a live Redis 8.0.6 cluster + // with `CLUSTER KEYSLOT` and confirm the vendored fix matches it. + it("hashes the whole key when it contains only an empty tag", () => { + expect(calculateSlot("a{}b}c")).to.eql(2041); + expect(calculateSlot("{}a}b")).to.eql(5168); + expect(calculateSlot("x{}}y")).to.eql(2083); + expect(calculateSlot("{}{a}")).to.eql(13650); + }); + + it("does not resume tag scanning after an empty tag", () => { + // Regression guard: an empty tag with no later '}' was already correct + // before the fix, since the scanner falls off the end of the string. + expect(calculateSlot("user:{}:1")).to.eql(8272); + }); + }); + + describe("cluster specification examples", () => { + // Examples from the "Hash tags" section of the Redis Cluster + // specification. + it("hashes the whole key when the first tag is empty", () => { + expect(calculateSlot("foo{}{bar}")).to.eql(8363); + }); + + it("hashes up to the first '}' after the first '{'", () => { + expect(calculateSlot("foo{{bar}}zap")).to.eql(calculateSlot("{bar")); + expect(calculateSlot("foo{bar}{zap}")).to.eql(calculateSlot("bar")); + }); + + it("puts keys sharing a tag in the same slot", () => { + expect( + calculateSlot.generateMulti([ + "{user1000}.following", + "{user1000}.followers", + ]) + ).to.eql(calculateSlot("user1000")); + }); + }); + + describe("regression guards for ordinary tags", () => { + it("still hashes the contents of a normal, non-empty tag", () => { + expect(calculateSlot("a{b}c")).to.eql(3300); + }); + + it("still stops at the first '}' after the first '{'", () => { + expect(calculateSlot("a{b{c}d")).to.eql(15725); + }); + + it("still hashes the whole key when there are no braces", () => { + expect(calculateSlot("foo")).to.eql(12182); + }); + }); + + describe("generateMulti()", () => { + it("returns the shared slot when all keys hash to the same slot", () => { + // "a{}b}c" and "42703" both land on slot 2041 under the corrected + // whole-key hashing behaviour. + expect(calculateSlot.generateMulti(["a{}b}c", "42703"])).to.eql(2041); + }); + + it("returns -1 when keys hash to different slots", () => { + expect(calculateSlot.generateMulti(["x{}}y", "z{}}w"])).to.eql(-1); + }); + }); + + describe("parity with upstream cluster-key-slot", () => { + // Ported from cluster-key-slot's own test suite (test/hash.spec.js) at + // https://github.com/invertase/cluster-key-slot/blob/master/test/hash.spec.js + // to guard against regressions in the vendored implementation. None of + // these keys trigger the empty-hash-tag bug, so every expected value is + // unchanged from upstream. + const tests: Array<[string | Buffer, number]> = [ + ["123465", 1492], + ["foobar", 12325], + ["abcdefghijklmnopqrstuvwxyz", 9132], + ["gsdfhan$%^&*(sdgsdnhshcs", 15532], + ["abc{foobar}", 12325], + ["{foobar}", 12325], + ["h8a9sd{foobar}}{asd}}", 12325], + ["{foobar", 16235], + ["foobar{}", 4435], + ["{{foobar}", 16235], + ["éêe", 13690], + ["àâa", 3872], + ["漢字", 14191], + ["汉字", 16196], + ["호텔", 4350], + ["💀", 9284], + ["𐀀", 11620], // surrogate pair + ["{}foobar", 14573], + [Buffer.from([0x7b, 0x7d, 0x2a, 0x2]), 3932], + [Buffer.from([0x7b, 0x2a, 0x7d, 0x2]), 1320], + [Buffer.from("汉字"), 16196], + ]; + + it("generates the same hash as upstream for each key", () => { + for (const [key, expectedSlot] of tests) { + expect(calculateSlot(key)).to.eql(expectedSlot, String(key)); + } + }); + + it("generates the same multi-key hash as upstream", () => { + const testsMulti = Array(8).fill("abcdefghijklmnopqrstuvwxyz"); + expect(calculateSlot.generateMulti(testsMulti)).to.eql(9132); + }); + + it("returns -1 for upstream's mixed-slot key set", () => { + expect(calculateSlot.generateMulti(tests.map(([key]) => key))).to.eql(-1); + }); + }); +}); From cadff3f8d2b46ffe55b92142096f4e96e417fdf0 Mon Sep 17 00:00:00 2001 From: Jacob Ley Date: Thu, 22 Oct 2026 01:42:23 +0000 Subject: [PATCH 2/4] fix(cluster): hash autopipelined Buffer keys by their bytes Auto-pipelining built the slot key with a template string, which decodes a Buffer key as UTF-8. Bytes that are not valid UTF-8 become U+FFFD, so different binary keys could share an auto pipeline even when their slots are served by different nodes, and Pipeline#exec rejected the batch. Hash Buffer keys from their bytes (with keyPrefix prepended), as Command does. Ported from redis/ioredis#2208. Signed-off-by: Jacob Ley --- lib/autoPipelining.ts | 17 +++++++++------ test/functional/cluster/autopipelining.ts | 26 +++++++++++++++++++++++ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/lib/autoPipelining.ts b/lib/autoPipelining.ts index 43301680..ec431317 100644 --- a/lib/autoPipelining.ts +++ b/lib/autoPipelining.ts @@ -145,12 +145,17 @@ export function executeWithAutoPipelining( // If we have slot information, we can improve routing by grouping slots served by the same subset of nodes // Note that the first value in args may be a (possibly empty) array. // ioredis will only flatten one level of the array, in the Command constructor. - const prefix = client.options.keyPrefix || ""; - const slotKey = client.isCluster - ? client.slots[ - calculateSlot(`${prefix}${getFirstValueInFlattenedArray(args)}`) - ].join(",") - : "main"; + let slotKey = "main"; + if (client.isCluster) { + const prefix = client.options.keyPrefix || ""; + const key = getFirstValueInFlattenedArray(args); + // Hash Buffer keys by their bytes, as Command#getSlot does; decoding + // them as UTF-8 can change the slot (ported from redis/ioredis#2208). + const slot = Buffer.isBuffer(key) + ? calculateSlot(Buffer.concat([Buffer.from(prefix), key])) + : calculateSlot(`${prefix}${key}`); + slotKey = client.slots[slot].join(","); + } if (!client._autoPipelines.has(slotKey)) { const pipeline = client.pipeline(); diff --git a/test/functional/cluster/autopipelining.ts b/test/functional/cluster/autopipelining.ts index 03edef9c..59bb4caf 100644 --- a/test/functional/cluster/autopipelining.ts +++ b/test/functional/cluster/autopipelining.ts @@ -36,6 +36,11 @@ describe("autoPipelining for cluster", () => { return "bar6"; } + // Buffer.from([0xfe]) hashes to slot 3793. + if (argv[0] === "get" && argv[1] === "\ufffd") { + return "binary-fe"; + } + if (argv[0] === "get" && argv[1] === "baz:foo10") { return "bar10"; } @@ -53,6 +58,11 @@ describe("autoPipelining for cluster", () => { if (argv[0] === "get" && argv[1] === "foo4") { return "bar4"; } + + // Buffer.from([0xff]) hashes to slot 7920. + if (argv[0] === "get" && argv[1] === "\ufffd") { + return "binary-ff"; + } }); new MockServer(30003, (argv) => { @@ -219,6 +229,22 @@ describe("autoPipelining for cluster", () => { cluster.disconnect(); }); + it("should bucket binary Buffer keys by their raw bytes", async () => { + const cluster = new Cluster(hosts, { enableAutoPipelining: true }); + await new Promise((resolve) => cluster.once("connect", resolve)); + + // Both keys decode to "\ufffd" as UTF-8 strings, but their bytes hash to + // slots served by different nodes. + expect( + await Promise.all([ + cluster.get(Buffer.from([0xff])), + cluster.get(Buffer.from([0xfe])), + ]) + ).to.eql(["binary-ff", "binary-fe"]); + + cluster.disconnect(); + }); + it("should support commands queued after a pipeline is already queued for execution", (done) => { const cluster = new Cluster(hosts, { enableAutoPipelining: true }); From 1c4b7513479a20a06c42da0810a62d3ca76aa93a Mon Sep 17 00:00:00 2001 From: Jacob Ley Date: Thu, 22 Oct 2026 01:55:52 +0000 Subject: [PATCH 3/4] fix(cluster): hash lone surrogates as the UTF-8 Node sends Ported from redis/ioredis#2242. Signed-off-by: Jacob Ley --- lib/utils/calculateSlot.ts | 9 +++++++++ test/unit/calculateSlot.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/lib/utils/calculateSlot.ts b/lib/utils/calculateSlot.ts index bb73dd91..1444e679 100644 --- a/lib/utils/calculateSlot.ts +++ b/lib/utils/calculateSlot.ts @@ -114,6 +114,15 @@ function toUTF8Array(str: string): number[] { utf8[p++] = ((char >> 6) & 63) | 128; utf8[p++] = (char & 63) | 128; } else { + // A lone surrogate has no UTF-8 encoding, so Node writes it as U+FFFD + // (EF BF BD), both to a socket and in `Buffer.from(str)`. Encoding it + // the same way hashes the bytes the server receives. Node's UTF-8 write + // passes V8 `kReplaceInvalidUtf8`, which replaces orphan surrogates: + // https://github.com/nodejs/node/blob/59e46cac2943/src/string_bytes.cc#L305-L316 + // https://github.com/nodejs/node/blob/59e46cac2943/deps/v8/include/v8-primitive.h#L222-L225 + if ((char & 0xf800) === 0xd800) { + char = 0xfffd; + } utf8[p++] = (char >> 12) | 224; utf8[p++] = ((char >> 6) & 63) | 128; utf8[p++] = (char & 63) | 128; diff --git a/test/unit/calculateSlot.ts b/test/unit/calculateSlot.ts index c0788c4d..97122d80 100644 --- a/test/unit/calculateSlot.ts +++ b/test/unit/calculateSlot.ts @@ -60,6 +60,34 @@ describe("calculateSlot", () => { }); }); + describe("lone UTF-16 surrogates", () => { + // Node writes string keys to the socket as UTF-8, replacing any lone + // surrogate with U+FFFD (EF BF BD). The slot must be computed from those + // same bytes, or the server answers with MOVED on every command. + const keys: Array<[string, string]> = [ + ["a lone high surrogate", "x\ud800y"], + ["a lone low surrogate", "x\udc00"], + ["a high surrogate at the end", "abc\ud83d"], + ["a low surrogate followed by a high one", "a\udc00\ud800b"], + ]; + + for (const [name, key] of keys) { + it(`hashes ${name} as the UTF-8 bytes Node sends`, () => { + expect(calculateSlot(key)).to.eql(calculateSlot(Buffer.from(key))); + }); + } + + it("matches the server's slot for a lone low surrogate", () => { + // Measured with `CLUSTER KEYSLOT` against a live Redis 8 server. + expect(calculateSlot("x\udc00")).to.eql(13299); + }); + + it("still hashes a valid surrogate pair as one code point", () => { + const key = "x\ud83d\ude00"; + expect(calculateSlot(key)).to.eql(calculateSlot(Buffer.from(key))); + }); + }); + describe("generateMulti()", () => { it("returns the shared slot when all keys hash to the same slot", () => { // "a{}b}c" and "42703" both land on slot 2041 under the corrected From e732339710c30f9944070cbf721b47c88d808d55 Mon Sep 17 00:00:00 2001 From: Jacob Ley Date: Thu, 22 Oct 2026 04:42:28 +0000 Subject: [PATCH 4/4] fix(cluster): hash autopipelined keys by their bytes under a Buffer keyPrefix keyPrefix is typed as a string, but a Buffer prefix is supported: keys are sent prefixed with its bytes. Auto-pipelining joined a Buffer prefix and a string key with a template string, which decodes the prefix as UTF-8, so a prefix that isn't valid UTF-8 could put commands for different nodes in one pipeline. Hash the bytes whenever the key or the prefix is a Buffer. Ported from redis/ioredis#2243. Signed-off-by: Jacob Ley --- lib/autoPipelining.ts | 16 +++++++++++----- test/unit/autoPipelining.ts | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/lib/autoPipelining.ts b/lib/autoPipelining.ts index ec431317..5e1e3347 100644 --- a/lib/autoPipelining.ts +++ b/lib/autoPipelining.ts @@ -111,6 +111,10 @@ export function getFirstValueInFlattenedArray( return undefined; } +function toBytes(value: string | Buffer | number | null | undefined): Buffer { + return Buffer.isBuffer(value) ? value : Buffer.from(String(value)); +} + export function executeWithAutoPipelining( client, functionName: string, @@ -149,11 +153,13 @@ export function executeWithAutoPipelining( if (client.isCluster) { const prefix = client.options.keyPrefix || ""; const key = getFirstValueInFlattenedArray(args); - // Hash Buffer keys by their bytes, as Command#getSlot does; decoding - // them as UTF-8 can change the slot (ported from redis/ioredis#2208). - const slot = Buffer.isBuffer(key) - ? calculateSlot(Buffer.concat([Buffer.from(prefix), key])) - : calculateSlot(`${prefix}${key}`); + // Hash Buffer keys and prefixes by their bytes, as Command#getSlot does; + // decoding them as UTF-8 can change the slot (ported from + // redis/ioredis#2208 and redis/ioredis#2243). + const slot = + Buffer.isBuffer(key) || Buffer.isBuffer(prefix) + ? calculateSlot(Buffer.concat([toBytes(prefix), toBytes(key)])) + : calculateSlot(`${prefix}${key}`); slotKey = client.slots[slot].join(","); } diff --git a/test/unit/autoPipelining.ts b/test/unit/autoPipelining.ts index c2e09478..74983dfa 100644 --- a/test/unit/autoPipelining.ts +++ b/test/unit/autoPipelining.ts @@ -1,4 +1,7 @@ import { expect } from "chai"; +import * as sinon from "sinon"; +import Pipeline from "../../lib/Pipeline"; +import Redis from "../../lib/Redis"; import { getFirstValueInFlattenedArray } from "../../lib/autoPipelining"; describe("autoPipelining", function () { @@ -26,4 +29,36 @@ describe("autoPipelining", function () { // @ts-expect-error expectGetFirstValueIs([createArguments("")], ""); }); + + it("should bucket string keys under a Buffer keyPrefix by their bytes", async function () { + const redis = new Redis(30000, { + enableAutoPipelining: true, + lazyConnect: true, + // keyPrefix is typed as a string, but a Buffer is supported + keyPrefix: Buffer.from([0xff]) as unknown as string, + }); + Object.defineProperty(redis, "isCluster", { value: true }); + // The prefixed keys' bytes hash to slots 12000 (node a) and 7811 (node + // b). Every other slot is node b's, so only hashing those bytes splits + // them. + (redis as any).slots = new Array(16384).fill(["b"]); + (redis as any).slots[12000] = ["a"]; + (redis as any)._groupsBySlot = new Array(16384); + // Only the buckets are read here; the pipelines themselves are not run. + const exec = sinon.stub(Pipeline.prototype, "exec").callsFake(function () { + return Promise.resolve([]); + }); + + try { + (redis as any).get("k0").catch(() => {}); + (redis as any).get("k3").catch(() => {}); + // One pipeline per node. + expect([...(redis as any)._autoPipelines.keys()]).to.have.lengthOf(2); + // Let the pipelines scheduled above run while exec is still stubbed. + await new Promise((resolve) => setImmediate(resolve)); + } finally { + exec.restore(); + redis.disconnect(); + } + }); });