diff --git a/benchmarks/fixtures/generate.ts b/benchmarks/fixtures/generate.ts index 21f0adf4..e6101ce1 100644 --- a/benchmarks/fixtures/generate.ts +++ b/benchmarks/fixtures/generate.ts @@ -2,7 +2,7 @@ const start = process.hrtime.bigint(); -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "../../lib/utils/calculateSlot"; import { writeFileSync } from "fs"; import { join } from "path"; import { v4 as uuid } from "uuid"; diff --git a/lib/Command.ts b/lib/Command.ts index 4e13bbdb..9c592ec7 100644 --- a/lib/Command.ts +++ b/lib/Command.ts @@ -1,5 +1,5 @@ import { exists, getKeyIndexes } from "@iovalkey/commands"; -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "./utils/calculateSlot"; import asCallback from "standard-as-callback"; import { convertBufferToString, diff --git a/lib/Pipeline.ts b/lib/Pipeline.ts index 49af22f3..8176f0d1 100644 --- a/lib/Pipeline.ts +++ b/lib/Pipeline.ts @@ -1,4 +1,4 @@ -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "./utils/calculateSlot"; import { exists, hasFlag } from "@iovalkey/commands"; import asCallback from "standard-as-callback"; import { deprecate } from "util"; diff --git a/lib/autoPipelining.ts b/lib/autoPipelining.ts index 68adc2a4..5e1e3347 100644 --- a/lib/autoPipelining.ts +++ b/lib/autoPipelining.ts @@ -1,5 +1,5 @@ import { isArguments, noop } from "./utils/lodash"; -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "./utils/calculateSlot"; import asCallback from "standard-as-callback"; import { ArgumentType } from "./Command"; @@ -111,6 +111,10 @@ export function getFirstValueInFlattenedArray( return undefined; } +function toBytes(value: string | Buffer | number | null | undefined): Buffer { + return Buffer.isBuffer(value) ? value : Buffer.from(String(value)); +} + export function executeWithAutoPipelining( client, functionName: string, @@ -145,12 +149,19 @@ export function executeWithAutoPipelining( // If we have slot information, we can improve routing by grouping slots served by the same subset of nodes // Note that the first value in args may be a (possibly empty) array. // ioredis will only flatten one level of the array, in the Command constructor. - const prefix = client.options.keyPrefix || ""; - const slotKey = client.isCluster - ? client.slots[ - calculateSlot(`${prefix}${getFirstValueInFlattenedArray(args)}`) - ].join(",") - : "main"; + let slotKey = "main"; + if (client.isCluster) { + const prefix = client.options.keyPrefix || ""; + const key = getFirstValueInFlattenedArray(args); + // Hash Buffer keys and prefixes by their bytes, as Command#getSlot does; + // decoding them as UTF-8 can change the slot (ported from + // redis/ioredis#2208 and redis/ioredis#2243). + const slot = + Buffer.isBuffer(key) || Buffer.isBuffer(prefix) + ? calculateSlot(Buffer.concat([toBytes(prefix), toBytes(key)])) + : calculateSlot(`${prefix}${key}`); + slotKey = client.slots[slot].join(","); + } if (!client._autoPipelines.has(slotKey)) { const pipeline = client.pipeline(); diff --git a/lib/utils/calculateSlot.ts b/lib/utils/calculateSlot.ts new file mode 100644 index 00000000..1444e679 --- /dev/null +++ b/lib/utils/calculateSlot.ts @@ -0,0 +1,198 @@ +/* + * Vendored from `cluster-key-slot@1.1.1` + * (https://github.com/invertase/cluster-key-slot), patched to fix a + * disagreement with real Redis servers over how an empty hash tag ("{}") is + * handled. + * + * Redis treats an empty hash tag as no tag at all: given `{}` it hashes the + * whole key and stops searching (see `keyHashSlot()` in Redis's + * `src/cluster.c`). The upstream single-pass scanner instead falls through + * and keeps scanning for the next `}`, hashing whatever sits between -- which + * can silently misroute commands. Vendoring was necessary because the + * upstream repository has been archived and is no longer maintained. Ported + * from ioredis (https://github.com/redis/ioredis/pull/2172). See + * https://github.com/mishan/intraslot/blob/main/contrib/empty-hashtag-slot-bug/README.md + * for the full writeup, live-server measurements, and suggested test cases. + * + * The fix, marked below, latches the whole-key fallback once the empty-tag + * case is seen: one extra boolean, no extra pass, no change to the hot path + * for keys without braces. + * + * Original copyright notices, preserved from upstream: + * + * Copyright 2001-2010 Georges Menie (www.menie.org) + * Copyright 2010 Salvatore Sanfilippo (adapted to Redis coding style) + * Copyright 2015 Zihua Li (http://zihua.li) (ported to JavaScript) + * Copyright 2016 Mike Diarmid (http://github.com/salakar) (re-write for + * performance, ~700% perf inc) + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* CRC16 implementation according to CCITT standards. + * + * Note by @antirez: this is actually the XMODEM CRC 16 algorithm, using the + * following parameters: + * + * Name : "XMODEM", also known as "ZMODEM", "CRC-16/ACORN" + * Width : 16 bit + * Poly : 1021 (That is actually x^16 + x^12 + x^5 + 1) + * Initialization : 0000 + * Reflect Input byte : False + * Reflect Output CRC : False + * Xor constant to output CRC : 0000 + * Output for "123456789" : 31C3 + */ +const lookup = [ + 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108, + 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, 0x1231, 0x0210, + 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, 0x9339, 0x8318, 0xb37b, + 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, 0x2462, 0x3443, 0x0420, 0x1401, + 0x64e6, 0x74c7, 0x44a4, 0x5485, 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, + 0xf5cf, 0xc5ac, 0xd58d, 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, + 0x5695, 0x46b4, 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, + 0xc7bc, 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, + 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, 0x5af5, + 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, 0xdbfd, 0xcbdc, + 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, 0x6ca6, 0x7c87, 0x4ce4, + 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, 0xedae, 0xfd8f, 0xcdec, 0xddcd, + 0xad2a, 0xbd0b, 0x8d68, 0x9d49, 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, + 0x2e32, 0x1e51, 0x0e70, 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, + 0x9f59, 0x8f78, 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, + 0xe16f, 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, + 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, 0x02b1, + 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, 0xb5ea, 0xa5cb, + 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, 0x34e2, 0x24c3, 0x14a0, + 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, 0xa7db, 0xb7fa, 0x8799, 0x97b8, + 0xe75f, 0xf77e, 0xc71d, 0xd73c, 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, + 0x7676, 0x4615, 0x5634, 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, + 0xb98a, 0xa9ab, 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, + 0x28a3, 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, + 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, 0xfd2e, + 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, 0x7c26, 0x6c07, + 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, 0xef1f, 0xff3e, 0xcf5d, + 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 0x6e17, 0x7e36, 0x4e55, 0x5e74, + 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0, +]; + +/** + * Convert a string to a UTF8 array - faster than via buffer + */ +function toUTF8Array(str: string): number[] { + let char: number; + let i = 0; + let p = 0; + const utf8: number[] = []; + const len = str.length; + + for (; i < len; i++) { + char = str.charCodeAt(i); + if (char < 128) { + utf8[p++] = char; + } else if (char < 2048) { + utf8[p++] = (char >> 6) | 192; + utf8[p++] = (char & 63) | 128; + } else if ( + (char & 0xfc00) === 0xd800 && + i + 1 < str.length && + (str.charCodeAt(i + 1) & 0xfc00) === 0xdc00 + ) { + char = 0x10000 + ((char & 0x03ff) << 10) + (str.charCodeAt(++i) & 0x03ff); + utf8[p++] = (char >> 18) | 240; + utf8[p++] = ((char >> 12) & 63) | 128; + utf8[p++] = ((char >> 6) & 63) | 128; + utf8[p++] = (char & 63) | 128; + } else { + // A lone surrogate has no UTF-8 encoding, so Node writes it as U+FFFD + // (EF BF BD), both to a socket and in `Buffer.from(str)`. Encoding it + // the same way hashes the bytes the server receives. Node's UTF-8 write + // passes V8 `kReplaceInvalidUtf8`, which replaces orphan surrogates: + // https://github.com/nodejs/node/blob/59e46cac2943/src/string_bytes.cc#L305-L316 + // https://github.com/nodejs/node/blob/59e46cac2943/deps/v8/include/v8-primitive.h#L222-L225 + if ((char & 0xf800) === 0xd800) { + char = 0xfffd; + } + utf8[p++] = (char >> 12) | 224; + utf8[p++] = ((char >> 6) & 63) | 128; + utf8[p++] = (char & 63) | 128; + } + } + + return utf8; +} + +/** + * Convert a string into a redis slot hash. + */ +function generate(str: string | ArrayLike): number { + let char: number; + let i = 0; + let start = -1; + // Latched once an empty hash tag ("{}") is seen: Redis hashes the whole + // key and stops looking for a tag in that case, so once `done` is set we + // must stop treating any later "{" as the start of a new tag. + let done = false; + let result = 0; + let resultHash = 0; + const utf8 = typeof str === "string" ? toUTF8Array(str) : str; + const len = utf8.length; + + while (i < len) { + char = utf8[i++]; + if (done || start === -1) { + if (!done && char === 0x7b) { + start = i; + } + } else if (char !== 0x7d) { + resultHash = + lookup[(char ^ (resultHash >> 8)) & 0xff] ^ (resultHash << 8); + } else if (i - 1 !== start) { + return resultHash & 0x3fff; + } else { + // Empty hash tag ("{}"): Redis falls back to hashing the whole key. + done = true; + } + + result = lookup[(char ^ (result >> 8)) & 0xff] ^ (result << 8); + } + + return result & 0x3fff; +} + +interface CalculateSlot { + (str: string | ArrayLike): number; + /** + * Convert an array of multiple strings into a redis slot hash. + * Returns -1 if one of the keys is not for the same slot as the others + */ + generateMulti(keys: (string | ArrayLike)[]): number; +} + +const calculateSlot = generate as CalculateSlot; + +calculateSlot.generateMulti = function generateMulti( + keys: (string | ArrayLike)[] +): number { + let i = 1; + const len = keys.length; + const base = generate(keys[0]); + + while (i < len) { + if (generate(keys[i++]) !== base) return -1; + } + + return base; +}; + +export = calculateSlot; diff --git a/package-lock.json b/package-lock.json index 0c346b2d..f7e09f72 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,15 @@ { "name": "iovalkey", - "version": "0.3.3", + "version": "0.4.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "iovalkey", - "version": "0.3.3", + "version": "0.4.0", "license": "MIT", "dependencies": { "@iovalkey/commands": "^0.1.0", - "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", @@ -1547,6 +1546,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==", + "dev": true, "engines": { "node": ">=0.10.0" } @@ -6076,7 +6076,8 @@ "cluster-key-slot": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.0.tgz", - "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==" + "integrity": "sha512-2Nii8p3RwAPiFwsnZvukotvow2rIHM+yQ6ZcBXGHdniadkYGZYiGmkHJIbZPIV9nfv7m/U1IPMVVcAhoWFeklw==", + "dev": true }, "color-convert": { "version": "2.0.1", diff --git a/package.json b/package.json index d1839b27..0fe9dd13 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,6 @@ "license": "MIT", "dependencies": { "@iovalkey/commands": "^0.1.0", - "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", diff --git a/test/functional/cluster/ask.ts b/test/functional/cluster/ask.ts index 2ff647a1..d5d98cbb 100644 --- a/test/functional/cluster/ask.ts +++ b/test/functional/cluster/ask.ts @@ -1,5 +1,5 @@ import MockServer from "../../helpers/mock_server"; -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "../../../lib/utils/calculateSlot"; import { expect } from "chai"; import { Cluster } from "../../../lib"; diff --git a/test/functional/cluster/autopipelining.ts b/test/functional/cluster/autopipelining.ts index 2dcc63c0..59bb4caf 100644 --- a/test/functional/cluster/autopipelining.ts +++ b/test/functional/cluster/autopipelining.ts @@ -1,5 +1,5 @@ import { expect, use } from "chai"; -import * as calculateKeySlot from "cluster-key-slot"; +import * as calculateKeySlot from "../../../lib/utils/calculateSlot"; import { default as Cluster } from "../../../lib/cluster"; import MockServer from "../../helpers/mock_server"; @@ -36,6 +36,11 @@ describe("autoPipelining for cluster", () => { return "bar6"; } + // Buffer.from([0xfe]) hashes to slot 3793. + if (argv[0] === "get" && argv[1] === "\ufffd") { + return "binary-fe"; + } + if (argv[0] === "get" && argv[1] === "baz:foo10") { return "bar10"; } @@ -53,6 +58,11 @@ describe("autoPipelining for cluster", () => { if (argv[0] === "get" && argv[1] === "foo4") { return "bar4"; } + + // Buffer.from([0xff]) hashes to slot 7920. + if (argv[0] === "get" && argv[1] === "\ufffd") { + return "binary-ff"; + } }); new MockServer(30003, (argv) => { @@ -219,6 +229,22 @@ describe("autoPipelining for cluster", () => { cluster.disconnect(); }); + it("should bucket binary Buffer keys by their raw bytes", async () => { + const cluster = new Cluster(hosts, { enableAutoPipelining: true }); + await new Promise((resolve) => cluster.once("connect", resolve)); + + // Both keys decode to "\ufffd" as UTF-8 strings, but their bytes hash to + // slots served by different nodes. + expect( + await Promise.all([ + cluster.get(Buffer.from([0xff])), + cluster.get(Buffer.from([0xfe])), + ]) + ).to.eql(["binary-ff", "binary-fe"]); + + cluster.disconnect(); + }); + it("should support commands queued after a pipeline is already queued for execution", (done) => { const cluster = new Cluster(hosts, { enableAutoPipelining: true }); diff --git a/test/functional/cluster/maxRedirections.ts b/test/functional/cluster/maxRedirections.ts index 183a6282..01171654 100644 --- a/test/functional/cluster/maxRedirections.ts +++ b/test/functional/cluster/maxRedirections.ts @@ -1,4 +1,4 @@ -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "../../../lib/utils/calculateSlot"; import MockServer from "../../helpers/mock_server"; import { expect } from "chai"; import { Cluster } from "../../../lib"; diff --git a/test/functional/cluster/moved.ts b/test/functional/cluster/moved.ts index 14082e36..0523ead4 100644 --- a/test/functional/cluster/moved.ts +++ b/test/functional/cluster/moved.ts @@ -1,4 +1,4 @@ -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "../../../lib/utils/calculateSlot"; import MockServer from "../../helpers/mock_server"; import { expect } from "chai"; import { Cluster } from "../../../lib"; diff --git a/test/functional/cluster/nat.ts b/test/functional/cluster/nat.ts index 10c845b8..238bf43b 100644 --- a/test/functional/cluster/nat.ts +++ b/test/functional/cluster/nat.ts @@ -1,4 +1,4 @@ -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "../../../lib/utils/calculateSlot"; import MockServer from "../../helpers/mock_server"; import { expect } from "chai"; import { Cluster } from "../../../lib"; diff --git a/test/functional/cluster/pipeline.ts b/test/functional/cluster/pipeline.ts index 2bc50a09..cf99bd08 100644 --- a/test/functional/cluster/pipeline.ts +++ b/test/functional/cluster/pipeline.ts @@ -1,4 +1,4 @@ -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "../../../lib/utils/calculateSlot"; import MockServer from "../../helpers/mock_server"; import { expect } from "chai"; import { Cluster } from "../../../lib"; diff --git a/test/functional/cluster/transaction.ts b/test/functional/cluster/transaction.ts index 17d74022..3555f152 100644 --- a/test/functional/cluster/transaction.ts +++ b/test/functional/cluster/transaction.ts @@ -1,4 +1,4 @@ -import * as calculateSlot from "cluster-key-slot"; +import * as calculateSlot from "../../../lib/utils/calculateSlot"; import MockServer from "../../helpers/mock_server"; import { expect } from "chai"; import { Cluster } from "../../../lib"; diff --git a/test/unit/autoPipelining.ts b/test/unit/autoPipelining.ts index c2e09478..74983dfa 100644 --- a/test/unit/autoPipelining.ts +++ b/test/unit/autoPipelining.ts @@ -1,4 +1,7 @@ import { expect } from "chai"; +import * as sinon from "sinon"; +import Pipeline from "../../lib/Pipeline"; +import Redis from "../../lib/Redis"; import { getFirstValueInFlattenedArray } from "../../lib/autoPipelining"; describe("autoPipelining", function () { @@ -26,4 +29,36 @@ describe("autoPipelining", function () { // @ts-expect-error expectGetFirstValueIs([createArguments("")], ""); }); + + it("should bucket string keys under a Buffer keyPrefix by their bytes", async function () { + const redis = new Redis(30000, { + enableAutoPipelining: true, + lazyConnect: true, + // keyPrefix is typed as a string, but a Buffer is supported + keyPrefix: Buffer.from([0xff]) as unknown as string, + }); + Object.defineProperty(redis, "isCluster", { value: true }); + // The prefixed keys' bytes hash to slots 12000 (node a) and 7811 (node + // b). Every other slot is node b's, so only hashing those bytes splits + // them. + (redis as any).slots = new Array(16384).fill(["b"]); + (redis as any).slots[12000] = ["a"]; + (redis as any)._groupsBySlot = new Array(16384); + // Only the buckets are read here; the pipelines themselves are not run. + const exec = sinon.stub(Pipeline.prototype, "exec").callsFake(function () { + return Promise.resolve([]); + }); + + try { + (redis as any).get("k0").catch(() => {}); + (redis as any).get("k3").catch(() => {}); + // One pipeline per node. + expect([...(redis as any)._autoPipelines.keys()]).to.have.lengthOf(2); + // Let the pipelines scheduled above run while exec is still stubbed. + await new Promise((resolve) => setImmediate(resolve)); + } finally { + exec.restore(); + redis.disconnect(); + } + }); }); diff --git a/test/unit/calculateSlot.ts b/test/unit/calculateSlot.ts new file mode 100644 index 00000000..97122d80 --- /dev/null +++ b/test/unit/calculateSlot.ts @@ -0,0 +1,148 @@ +import { expect } from "chai"; +import * as calculateSlot from "../../lib/utils/calculateSlot"; + +describe("calculateSlot", () => { + describe("empty hash tag handling", () => { + // Redis treats an empty hash tag ("{}") as no tag at all: it hashes the + // whole key and stops searching for a tag. The unpatched upstream + // scanner instead falls through on an empty tag without clearing its + // "in a tag" state, so it keeps scanning and latches onto the next "}" + // in the key, hashing whatever sits between them instead of the whole + // key. These values were measured against a live Redis 8.0.6 cluster + // with `CLUSTER KEYSLOT` and confirm the vendored fix matches it. + it("hashes the whole key when it contains only an empty tag", () => { + expect(calculateSlot("a{}b}c")).to.eql(2041); + expect(calculateSlot("{}a}b")).to.eql(5168); + expect(calculateSlot("x{}}y")).to.eql(2083); + expect(calculateSlot("{}{a}")).to.eql(13650); + }); + + it("does not resume tag scanning after an empty tag", () => { + // Regression guard: an empty tag with no later '}' was already correct + // before the fix, since the scanner falls off the end of the string. + expect(calculateSlot("user:{}:1")).to.eql(8272); + }); + }); + + describe("cluster specification examples", () => { + // Examples from the "Hash tags" section of the Redis Cluster + // specification. + it("hashes the whole key when the first tag is empty", () => { + expect(calculateSlot("foo{}{bar}")).to.eql(8363); + }); + + it("hashes up to the first '}' after the first '{'", () => { + expect(calculateSlot("foo{{bar}}zap")).to.eql(calculateSlot("{bar")); + expect(calculateSlot("foo{bar}{zap}")).to.eql(calculateSlot("bar")); + }); + + it("puts keys sharing a tag in the same slot", () => { + expect( + calculateSlot.generateMulti([ + "{user1000}.following", + "{user1000}.followers", + ]) + ).to.eql(calculateSlot("user1000")); + }); + }); + + describe("regression guards for ordinary tags", () => { + it("still hashes the contents of a normal, non-empty tag", () => { + expect(calculateSlot("a{b}c")).to.eql(3300); + }); + + it("still stops at the first '}' after the first '{'", () => { + expect(calculateSlot("a{b{c}d")).to.eql(15725); + }); + + it("still hashes the whole key when there are no braces", () => { + expect(calculateSlot("foo")).to.eql(12182); + }); + }); + + describe("lone UTF-16 surrogates", () => { + // Node writes string keys to the socket as UTF-8, replacing any lone + // surrogate with U+FFFD (EF BF BD). The slot must be computed from those + // same bytes, or the server answers with MOVED on every command. + const keys: Array<[string, string]> = [ + ["a lone high surrogate", "x\ud800y"], + ["a lone low surrogate", "x\udc00"], + ["a high surrogate at the end", "abc\ud83d"], + ["a low surrogate followed by a high one", "a\udc00\ud800b"], + ]; + + for (const [name, key] of keys) { + it(`hashes ${name} as the UTF-8 bytes Node sends`, () => { + expect(calculateSlot(key)).to.eql(calculateSlot(Buffer.from(key))); + }); + } + + it("matches the server's slot for a lone low surrogate", () => { + // Measured with `CLUSTER KEYSLOT` against a live Redis 8 server. + expect(calculateSlot("x\udc00")).to.eql(13299); + }); + + it("still hashes a valid surrogate pair as one code point", () => { + const key = "x\ud83d\ude00"; + expect(calculateSlot(key)).to.eql(calculateSlot(Buffer.from(key))); + }); + }); + + describe("generateMulti()", () => { + it("returns the shared slot when all keys hash to the same slot", () => { + // "a{}b}c" and "42703" both land on slot 2041 under the corrected + // whole-key hashing behaviour. + expect(calculateSlot.generateMulti(["a{}b}c", "42703"])).to.eql(2041); + }); + + it("returns -1 when keys hash to different slots", () => { + expect(calculateSlot.generateMulti(["x{}}y", "z{}}w"])).to.eql(-1); + }); + }); + + describe("parity with upstream cluster-key-slot", () => { + // Ported from cluster-key-slot's own test suite (test/hash.spec.js) at + // https://github.com/invertase/cluster-key-slot/blob/master/test/hash.spec.js + // to guard against regressions in the vendored implementation. None of + // these keys trigger the empty-hash-tag bug, so every expected value is + // unchanged from upstream. + const tests: Array<[string | Buffer, number]> = [ + ["123465", 1492], + ["foobar", 12325], + ["abcdefghijklmnopqrstuvwxyz", 9132], + ["gsdfhan$%^&*(sdgsdnhshcs", 15532], + ["abc{foobar}", 12325], + ["{foobar}", 12325], + ["h8a9sd{foobar}}{asd}}", 12325], + ["{foobar", 16235], + ["foobar{}", 4435], + ["{{foobar}", 16235], + ["éêe", 13690], + ["àâa", 3872], + ["漢字", 14191], + ["汉字", 16196], + ["호텔", 4350], + ["💀", 9284], + ["𐀀", 11620], // surrogate pair + ["{}foobar", 14573], + [Buffer.from([0x7b, 0x7d, 0x2a, 0x2]), 3932], + [Buffer.from([0x7b, 0x2a, 0x7d, 0x2]), 1320], + [Buffer.from("汉字"), 16196], + ]; + + it("generates the same hash as upstream for each key", () => { + for (const [key, expectedSlot] of tests) { + expect(calculateSlot(key)).to.eql(expectedSlot, String(key)); + } + }); + + it("generates the same multi-key hash as upstream", () => { + const testsMulti = Array(8).fill("abcdefghijklmnopqrstuvwxyz"); + expect(calculateSlot.generateMulti(testsMulti)).to.eql(9132); + }); + + it("returns -1 for upstream's mixed-slot key set", () => { + expect(calculateSlot.generateMulti(tests.map(([key]) => key))).to.eql(-1); + }); + }); +});