Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion benchmarks/fixtures/generate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

const start = process.hrtime.bigint();

import * as calculateSlot from "cluster-key-slot";
import * as calculateSlot from "../../lib/utils/calculateSlot";
import { writeFileSync } from "fs";
import { join } from "path";
import { v4 as uuid } from "uuid";
Expand Down
2 changes: 1 addition & 1 deletion lib/Command.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { exists, getKeyIndexes } from "@iovalkey/commands";
import * as calculateSlot from "cluster-key-slot";
import * as calculateSlot from "./utils/calculateSlot";
import asCallback from "standard-as-callback";
import {
convertBufferToString,
Expand Down
2 changes: 1 addition & 1 deletion lib/Pipeline.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import * as calculateSlot from "cluster-key-slot";
import * as calculateSlot from "./utils/calculateSlot";
import { exists, hasFlag } from "@iovalkey/commands";
import asCallback from "standard-as-callback";
import { deprecate } from "util";
Expand Down
25 changes: 18 additions & 7 deletions lib/autoPipelining.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { isArguments, noop } from "./utils/lodash";
import * as calculateSlot from "cluster-key-slot";
import * as calculateSlot from "./utils/calculateSlot";
import asCallback from "standard-as-callback";
import { ArgumentType } from "./Command";

Expand Down Expand Up @@ -111,6 +111,10 @@ export function getFirstValueInFlattenedArray(
return undefined;
}

function toBytes(value: string | Buffer | number | null | undefined): Buffer {
return Buffer.isBuffer(value) ? value : Buffer.from(String(value));
}

export function executeWithAutoPipelining(
client,
functionName: string,
Expand Down Expand Up @@ -145,12 +149,19 @@ export function executeWithAutoPipelining(
// If we have slot information, we can improve routing by grouping slots served by the same subset of nodes
// Note that the first value in args may be a (possibly empty) array.
// ioredis will only flatten one level of the array, in the Command constructor.
const prefix = client.options.keyPrefix || "";
const slotKey = client.isCluster
? client.slots[
calculateSlot(`${prefix}${getFirstValueInFlattenedArray(args)}`)
].join(",")
: "main";
let slotKey = "main";
if (client.isCluster) {
const prefix = client.options.keyPrefix || "";
const key = getFirstValueInFlattenedArray(args);
// Hash Buffer keys and prefixes by their bytes, as Command#getSlot does;
// decoding them as UTF-8 can change the slot (ported from
// redis/ioredis#2208 and redis/ioredis#2243).
const slot =
Buffer.isBuffer(key) || Buffer.isBuffer(prefix)
? calculateSlot(Buffer.concat([toBytes(prefix), toBytes(key)]))
: calculateSlot(`${prefix}${key}`);
slotKey = client.slots[slot].join(",");
}

if (!client._autoPipelines.has(slotKey)) {
const pipeline = client.pipeline();
Expand Down
198 changes: 198 additions & 0 deletions lib/utils/calculateSlot.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
/*
* Vendored from `cluster-key-slot@1.1.1`
* (https://github.com/invertase/cluster-key-slot), patched to fix a
* disagreement with real Redis servers over how an empty hash tag ("{}") is
* handled.
*
* Redis treats an empty hash tag as no tag at all: given `{}` it hashes the
* whole key and stops searching (see `keyHashSlot()` in Redis's
* `src/cluster.c`). The upstream single-pass scanner instead falls through
* and keeps scanning for the next `}`, hashing whatever sits between -- which
* can silently misroute commands. Vendoring was necessary because the
* upstream repository has been archived and is no longer maintained. Ported
* from ioredis (https://github.com/redis/ioredis/pull/2172). See
* https://github.com/mishan/intraslot/blob/main/contrib/empty-hashtag-slot-bug/README.md
* for the full writeup, live-server measurements, and suggested test cases.
*
* The fix, marked below, latches the whole-key fallback once the empty-tag
* case is seen: one extra boolean, no extra pass, no change to the hot path
* for keys without braces.
*
* Original copyright notices, preserved from upstream:
*
* Copyright 2001-2010 Georges Menie (www.menie.org)
* Copyright 2010 Salvatore Sanfilippo (adapted to Redis coding style)
* Copyright 2015 Zihua Li (http://zihua.li) (ported to JavaScript)
* Copyright 2016 Mike Diarmid (http://github.com/salakar) (re-write for
* performance, ~700% perf inc)
* All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

/* CRC16 implementation according to CCITT standards.
*
* Note by @antirez: this is actually the XMODEM CRC 16 algorithm, using the
* following parameters:
*
* Name : "XMODEM", also known as "ZMODEM", "CRC-16/ACORN"
* Width : 16 bit
* Poly : 1021 (That is actually x^16 + x^12 + x^5 + 1)
* Initialization : 0000
* Reflect Input byte : False
* Reflect Output CRC : False
* Xor constant to output CRC : 0000
* Output for "123456789" : 31C3
*/
const lookup = [
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 0x8108,
0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, 0x1231, 0x0210,
0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, 0x9339, 0x8318, 0xb37b,
0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, 0x2462, 0x3443, 0x0420, 0x1401,
0x64e6, 0x74c7, 0x44a4, 0x5485, 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee,
0xf5cf, 0xc5ac, 0xd58d, 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6,
0x5695, 0x46b4, 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d,
0xc7bc, 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823,
0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, 0x5af5,
0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, 0xdbfd, 0xcbdc,
0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, 0x6ca6, 0x7c87, 0x4ce4,
0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, 0xedae, 0xfd8f, 0xcdec, 0xddcd,
0xad2a, 0xbd0b, 0x8d68, 0x9d49, 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13,
0x2e32, 0x1e51, 0x0e70, 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a,
0x9f59, 0x8f78, 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e,
0xe16f, 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067,
0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, 0x02b1,
0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, 0xb5ea, 0xa5cb,
0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, 0x34e2, 0x24c3, 0x14a0,
0x0481, 0x7466, 0x6447, 0x5424, 0x4405, 0xa7db, 0xb7fa, 0x8799, 0x97b8,
0xe75f, 0xf77e, 0xc71d, 0xd73c, 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657,
0x7676, 0x4615, 0x5634, 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9,
0xb98a, 0xa9ab, 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882,
0x28a3, 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a,
0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, 0xfd2e,
0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, 0x7c26, 0x6c07,
0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, 0xef1f, 0xff3e, 0xcf5d,
0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 0x6e17, 0x7e36, 0x4e55, 0x5e74,
0x2e93, 0x3eb2, 0x0ed1, 0x1ef0,
];

/**
* Convert a string to a UTF8 array - faster than via buffer
*/
function toUTF8Array(str: string): number[] {
let char: number;
let i = 0;
let p = 0;
const utf8: number[] = [];
const len = str.length;

for (; i < len; i++) {
char = str.charCodeAt(i);
if (char < 128) {
utf8[p++] = char;
} else if (char < 2048) {
utf8[p++] = (char >> 6) | 192;
utf8[p++] = (char & 63) | 128;
} else if (
(char & 0xfc00) === 0xd800 &&
i + 1 < str.length &&
(str.charCodeAt(i + 1) & 0xfc00) === 0xdc00
) {
char = 0x10000 + ((char & 0x03ff) << 10) + (str.charCodeAt(++i) & 0x03ff);
utf8[p++] = (char >> 18) | 240;
utf8[p++] = ((char >> 12) & 63) | 128;
utf8[p++] = ((char >> 6) & 63) | 128;
utf8[p++] = (char & 63) | 128;
} else {
// A lone surrogate has no UTF-8 encoding, so Node writes it as U+FFFD
// (EF BF BD), both to a socket and in `Buffer.from(str)`. Encoding it
// the same way hashes the bytes the server receives. Node's UTF-8 write
// passes V8 `kReplaceInvalidUtf8`, which replaces orphan surrogates:
// https://github.com/nodejs/node/blob/59e46cac2943/src/string_bytes.cc#L305-L316
// https://github.com/nodejs/node/blob/59e46cac2943/deps/v8/include/v8-primitive.h#L222-L225
if ((char & 0xf800) === 0xd800) {
char = 0xfffd;
}
utf8[p++] = (char >> 12) | 224;
utf8[p++] = ((char >> 6) & 63) | 128;
utf8[p++] = (char & 63) | 128;
}
}

return utf8;
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

/**
* Convert a string into a redis slot hash.
*/
function generate(str: string | ArrayLike<number>): number {
let char: number;
let i = 0;
let start = -1;
// Latched once an empty hash tag ("{}") is seen: Redis hashes the whole
// key and stops looking for a tag in that case, so once `done` is set we
// must stop treating any later "{" as the start of a new tag.
let done = false;
let result = 0;
let resultHash = 0;
const utf8 = typeof str === "string" ? toUTF8Array(str) : str;
const len = utf8.length;

while (i < len) {
char = utf8[i++];
if (done || start === -1) {
if (!done && char === 0x7b) {
start = i;
}
} else if (char !== 0x7d) {
resultHash =
lookup[(char ^ (resultHash >> 8)) & 0xff] ^ (resultHash << 8);
} else if (i - 1 !== start) {
return resultHash & 0x3fff;
} else {
// Empty hash tag ("{}"): Redis falls back to hashing the whole key.
done = true;
}

result = lookup[(char ^ (result >> 8)) & 0xff] ^ (result << 8);
}

return result & 0x3fff;
}

interface CalculateSlot {
(str: string | ArrayLike<number>): number;
/**
* Convert an array of multiple strings into a redis slot hash.
* Returns -1 if one of the keys is not for the same slot as the others
*/
generateMulti(keys: (string | ArrayLike<number>)[]): number;
}

const calculateSlot = generate as CalculateSlot;

calculateSlot.generateMulti = function generateMulti(
keys: (string | ArrayLike<number>)[]
): number {
let i = 1;
const len = keys.length;
const base = generate(keys[0]);

while (i < len) {
if (generate(keys[i++]) !== base) return -1;
}

return base;
};

export = calculateSlot;
9 changes: 5 additions & 4 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@
"license": "MIT",
"dependencies": {
"@iovalkey/commands": "^0.1.0",
"cluster-key-slot": "^1.1.0",
"debug": "^4.3.4",
"denque": "^2.1.0",
"lodash.defaults": "^4.2.0",
Expand Down
2 changes: 1 addition & 1 deletion test/functional/cluster/ask.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import MockServer from "../../helpers/mock_server";
import * as calculateSlot from "cluster-key-slot";
import * as calculateSlot from "../../../lib/utils/calculateSlot";
import { expect } from "chai";
import { Cluster } from "../../../lib";

Expand Down
28 changes: 27 additions & 1 deletion test/functional/cluster/autopipelining.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { expect, use } from "chai";
import * as calculateKeySlot from "cluster-key-slot";
import * as calculateKeySlot from "../../../lib/utils/calculateSlot";

import { default as Cluster } from "../../../lib/cluster";
import MockServer from "../../helpers/mock_server";
Expand Down Expand Up @@ -36,6 +36,11 @@ describe("autoPipelining for cluster", () => {
return "bar6";
}

// Buffer.from([0xfe]) hashes to slot 3793.
if (argv[0] === "get" && argv[1] === "\ufffd") {
return "binary-fe";
}

if (argv[0] === "get" && argv[1] === "baz:foo10") {
return "bar10";
}
Expand All @@ -53,6 +58,11 @@ describe("autoPipelining for cluster", () => {
if (argv[0] === "get" && argv[1] === "foo4") {
return "bar4";
}

// Buffer.from([0xff]) hashes to slot 7920.
if (argv[0] === "get" && argv[1] === "\ufffd") {
return "binary-ff";
}
});

new MockServer(30003, (argv) => {
Expand Down Expand Up @@ -219,6 +229,22 @@ describe("autoPipelining for cluster", () => {
cluster.disconnect();
});

it("should bucket binary Buffer keys by their raw bytes", async () => {
const cluster = new Cluster(hosts, { enableAutoPipelining: true });
await new Promise((resolve) => cluster.once("connect", resolve));

// Both keys decode to "\ufffd" as UTF-8 strings, but their bytes hash to
// slots served by different nodes.
expect(
await Promise.all([
cluster.get(Buffer.from([0xff])),
cluster.get(Buffer.from([0xfe])),
])
).to.eql(["binary-ff", "binary-fe"]);

cluster.disconnect();
});

it("should support commands queued after a pipeline is already queued for execution", (done) => {
const cluster = new Cluster(hosts, { enableAutoPipelining: true });

Expand Down
2 changes: 1 addition & 1 deletion test/functional/cluster/maxRedirections.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import * as calculateSlot from "cluster-key-slot";
import * as calculateSlot from "../../../lib/utils/calculateSlot";
import MockServer from "../../helpers/mock_server";
import { expect } from "chai";
import { Cluster } from "../../../lib";
Expand Down
2 changes: 1 addition & 1 deletion test/functional/cluster/moved.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import * as calculateSlot from "cluster-key-slot";
import * as calculateSlot from "../../../lib/utils/calculateSlot";
import MockServer from "../../helpers/mock_server";
import { expect } from "chai";
import { Cluster } from "../../../lib";
Expand Down
2 changes: 1 addition & 1 deletion test/functional/cluster/nat.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import * as calculateSlot from "cluster-key-slot";
import * as calculateSlot from "../../../lib/utils/calculateSlot";
import MockServer from "../../helpers/mock_server";
import { expect } from "chai";
import { Cluster } from "../../../lib";
Expand Down
2 changes: 1 addition & 1 deletion test/functional/cluster/pipeline.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import * as calculateSlot from "cluster-key-slot";
import * as calculateSlot from "../../../lib/utils/calculateSlot";
import MockServer from "../../helpers/mock_server";
import { expect } from "chai";
import { Cluster } from "../../../lib";
Expand Down
Loading