diff --git a/.changeset/brave-rocks-lie.md b/.changeset/brave-rocks-lie.md new file mode 100644 index 000000000..f16aea92a --- /dev/null +++ b/.changeset/brave-rocks-lie.md @@ -0,0 +1,6 @@ +--- +"@tanstack/db-ivm": patch +"@tanstack/db": patch +--- + +Add support for Date objects to min/max aggregates and range queries when using an index. diff --git a/packages/db-ivm/src/operators/groupBy.ts b/packages/db-ivm/src/operators/groupBy.ts index afb4dc8d8..ef5810ac6 100644 --- a/packages/db-ivm/src/operators/groupBy.ts +++ b/packages/db-ivm/src/operators/groupBy.ts @@ -214,19 +214,21 @@ export function avg( * Creates a min aggregate function that computes the minimum value in a group * @param valueExtractor Function to extract a numeric value from each data entry */ -export function min( - valueExtractor: (value: T) => number = (v) => v as unknown as number -): AggregateFunction { +export function min( + valueExtractor: (value: T) => V = (v) => v as unknown as V +): AggregateFunction { return { preMap: (data: T) => valueExtractor(data), - reduce: (values: Array<[number, number]>) => { - let minValue = Number.POSITIVE_INFINITY + reduce: (values) => { + let minValue: V | undefined for (const [value, _multiplicity] of values) { - if (value < minValue) { + if (minValue && value < minValue) { + minValue = value + } else if (!minValue) { minValue = value } } - return minValue === Number.POSITIVE_INFINITY ? 0 : minValue + return minValue ?? (0 as V) }, } } @@ -235,19 +237,21 @@ export function min( * Creates a max aggregate function that computes the maximum value in a group * @param valueExtractor Function to extract a numeric value from each data entry */ -export function max( - valueExtractor: (value: T) => number = (v) => v as unknown as number -): AggregateFunction { +export function max( + valueExtractor: (value: T) => V = (v) => v as unknown as V +): AggregateFunction { return { preMap: (data: T) => valueExtractor(data), - reduce: (values: Array<[number, number]>) => { - let maxValue = Number.NEGATIVE_INFINITY + reduce: (values) => { + let maxValue: V | undefined for (const [value, _multiplicity] of values) { - if (value > maxValue) { + if (maxValue && value > maxValue) { + maxValue = value + } else if (!maxValue) { maxValue = value } } - return maxValue === Number.NEGATIVE_INFINITY ? 0 : maxValue + return maxValue ?? (0 as V) }, } } diff --git a/packages/db-ivm/tests/operators/groupBy.test.ts b/packages/db-ivm/tests/operators/groupBy.test.ts index 4a43bfb2a..1a569dcd1 100644 --- a/packages/db-ivm/tests/operators/groupBy.test.ts +++ b/packages/db-ivm/tests/operators/groupBy.test.ts @@ -524,6 +524,7 @@ describe(`Operators`, () => { const input = graph.newInput<{ category: string amount: number + date: Date }>() let latestMessage: any = null @@ -531,6 +532,8 @@ describe(`Operators`, () => { groupBy((data) => ({ category: data.category }), { minimum: min((data) => data.amount), maximum: max((data) => data.amount), + min_date: min((data) => data.date), + max_date: max((data) => data.date), }), output((message) => { latestMessage = message @@ -542,11 +545,11 @@ describe(`Operators`, () => { // Initial data input.sendData( new MultiSet([ - [{ category: `A`, amount: 10 }, 1], - [{ category: `A`, amount: 20 }, 1], - [{ category: `A`, amount: 5 }, 1], - [{ category: `B`, amount: 30 }, 1], - [{ category: `B`, amount: 15 }, 1], + [{ category: `A`, amount: 10, date: new Date(`2025/12/13`) }, 1], + [{ category: `A`, amount: 20, date: new Date(`2025/12/15`) }, 1], + [{ category: `A`, amount: 5, date: new Date(`2025/12/12`) }, 1], + [{ category: `B`, amount: 30, date: new Date(`2025/12/12`) }, 1], + [{ category: `B`, amount: 15, date: new Date(`2025/12/13`) }, 1], ]) ) @@ -563,6 +566,8 @@ describe(`Operators`, () => { category: `A`, minimum: 5, maximum: 20, + min_date: new Date(`2025/12/12`), + max_date: new Date(`2025/12/15`), }, ], 1, @@ -574,6 +579,8 @@ describe(`Operators`, () => { category: `B`, minimum: 15, maximum: 30, + min_date: new Date(`2025/12/12`), + max_date: new Date(`2025/12/13`), }, ], 1, diff --git a/packages/db/src/indexes/btree-index.ts b/packages/db/src/indexes/btree-index.ts index 15b00e8e4..ceda36e1b 100644 --- a/packages/db/src/indexes/btree-index.ts +++ b/packages/db/src/indexes/btree-index.ts @@ -71,15 +71,18 @@ export class BTreeIndex< ) } + // Normalize the value for Map key usage + const normalizedValue = this.normalizeMapKey(indexedValue) + // Check if this value already exists - if (this.valueMap.has(indexedValue)) { + if (this.valueMap.has(normalizedValue)) { // Add to existing set - this.valueMap.get(indexedValue)!.add(key) + this.valueMap.get(normalizedValue)!.add(key) } else { // Create new set for this value const keySet = new Set([key]) - this.valueMap.set(indexedValue, keySet) - this.orderedEntries.set(indexedValue, undefined) + this.valueMap.set(normalizedValue, keySet) + this.orderedEntries.set(normalizedValue, undefined) } this.indexedKeys.add(key) @@ -101,16 +104,19 @@ export class BTreeIndex< return } - if (this.valueMap.has(indexedValue)) { - const keySet = this.valueMap.get(indexedValue)! + // Normalize the value for Map key usage + const normalizedValue = this.normalizeMapKey(indexedValue) + + if (this.valueMap.has(normalizedValue)) { + const keySet = this.valueMap.get(normalizedValue)! keySet.delete(key) // If set is now empty, remove the entry entirely if (keySet.size === 0) { - this.valueMap.delete(indexedValue) + this.valueMap.delete(normalizedValue) // Remove from ordered entries - this.orderedEntries.delete(indexedValue) + this.orderedEntries.delete(normalizedValue) } } @@ -195,7 +201,8 @@ export class BTreeIndex< * Performs an equality lookup */ equalityLookup(value: any): Set { - return new Set(this.valueMap.get(value) ?? []) + const normalizedValue = this.normalizeMapKey(value) + return new Set(this.valueMap.get(normalizedValue) ?? []) } /** @@ -206,8 +213,10 @@ export class BTreeIndex< const { from, to, fromInclusive = true, toInclusive = true } = options const result = new Set() - const fromKey = from ?? this.orderedEntries.minKey() - const toKey = to ?? this.orderedEntries.maxKey() + const normalizedFrom = this.normalizeMapKey(from) + const normalizedTo = this.normalizeMapKey(to) + const fromKey = normalizedFrom ?? this.orderedEntries.minKey() + const toKey = normalizedTo ?? this.orderedEntries.maxKey() this.orderedEntries.forRange( fromKey, @@ -240,7 +249,7 @@ export class BTreeIndex< const keysInResult: Set = new Set() const result: Array = [] const nextKey = (k?: any) => this.orderedEntries.nextHigherKey(k) - let key = from + let key = this.normalizeMapKey(from) while ((key = nextKey(key)) && result.length < n) { const keys = this.valueMap.get(key) @@ -266,7 +275,8 @@ export class BTreeIndex< const result = new Set() for (const value of values) { - const keys = this.valueMap.get(value) + const normalizedValue = this.normalizeMapKey(value) + const keys = this.valueMap.get(normalizedValue) if (keys) { keys.forEach((key) => result.add(key)) } @@ -289,4 +299,15 @@ export class BTreeIndex< get valueMapData(): Map> { return this.valueMap } + + /** + * Normalizes values for use as Map keys to ensure proper equality and range queries + * For Date objects, uses timestamp. + */ + private normalizeMapKey(value: any): any { + if (value instanceof Date) { + return value.valueOf() + } + return value + } } diff --git a/packages/db/src/query/builder/functions.ts b/packages/db/src/query/builder/functions.ts index f1281ae2b..402c6de97 100644 --- a/packages/db/src/query/builder/functions.ts +++ b/packages/db/src/query/builder/functions.ts @@ -53,10 +53,10 @@ type ExtractType = // Helper type to determine aggregate return type based on input nullability type AggregateReturnType = ExtractType extends infer U - ? U extends number | undefined | null + ? U extends number | undefined | null | Date | bigint ? Aggregate - : Aggregate - : Aggregate + : Aggregate + : Aggregate // Helper type to determine string function return type based on input nullability type StringFunctionReturnType = diff --git a/packages/db/src/query/compiler/evaluators.ts b/packages/db/src/query/compiler/evaluators.ts index 0861f6eba..d41881eb9 100644 --- a/packages/db/src/query/compiler/evaluators.ts +++ b/packages/db/src/query/compiler/evaluators.ts @@ -144,6 +144,9 @@ function compileFunction(func: Func, isSingleRow: boolean): (data: any) => any { return (data) => { const a = argA(data) const b = argB(data) + if (a instanceof Date && b instanceof Date) { + return a.valueOf() == b.valueOf() + } return a === b } } diff --git a/packages/db/src/query/compiler/group-by.ts b/packages/db/src/query/compiler/group-by.ts index d67f229a8..e2bd10109 100644 --- a/packages/db/src/query/compiler/group-by.ts +++ b/packages/db/src/query/compiler/group-by.ts @@ -349,6 +349,20 @@ function getAggregateFunction(aggExpr: Aggregate) { return typeof value === `number` ? value : value != null ? Number(value) : 0 } + // Create a value extractor function for the expression to aggregate + const valueExtractorWithDate = ([, namespacedRow]: [ + string, + NamespacedRow, + ]) => { + const value = compiledExpr(namespacedRow) + // Ensure we return a number for numeric aggregate functions + return typeof value === `number` || value instanceof Date + ? value + : value != null + ? Number(value) + : 0 + } + // Create a raw value extractor function for the expression to aggregate const rawValueExtractor = ([, namespacedRow]: [string, NamespacedRow]) => { return compiledExpr(namespacedRow) @@ -363,9 +377,9 @@ function getAggregateFunction(aggExpr: Aggregate) { case `avg`: return avg(valueExtractor) case `min`: - return min(valueExtractor) + return min(valueExtractorWithDate) case `max`: - return max(valueExtractor) + return max(valueExtractorWithDate) default: throw new UnsupportedAggregateFunctionError(aggExpr.name) } diff --git a/packages/db/src/utils/comparison.ts b/packages/db/src/utils/comparison.ts index d8826f104..fa8750610 100644 --- a/packages/db/src/utils/comparison.ts +++ b/packages/db/src/utils/comparison.ts @@ -51,7 +51,7 @@ export const ascComparator = (a: any, b: any, opts: CompareOptions): number => { // If both are dates, compare them if (a instanceof Date && b instanceof Date) { - return a.getTime() - b.getTime() + return a.valueOf() - b.valueOf() } // If at least one of the values is an object, use stable IDs for comparison diff --git a/packages/db/tests/query/group-by.test-d.ts b/packages/db/tests/query/group-by.test-d.ts index f25ee92e2..4d7aac0c9 100644 --- a/packages/db/tests/query/group-by.test-d.ts +++ b/packages/db/tests/query/group-by.test-d.ts @@ -23,6 +23,7 @@ type Order = { amount: number status: string date: string + date_instance: Date product_category: string quantity: number discount: number @@ -37,6 +38,7 @@ const sampleOrders: Array = [ amount: 100, status: `completed`, date: `2023-01-01`, + date_instance: new Date(`2023-01-01`), product_category: `electronics`, quantity: 2, discount: 0, @@ -48,6 +50,7 @@ const sampleOrders: Array = [ amount: 200, status: `completed`, date: `2023-01-15`, + date_instance: new Date(`2023-01-15`), product_category: `electronics`, quantity: 1, discount: 10, @@ -81,6 +84,8 @@ describe(`Query GROUP BY Types`, () => { avg_amount: avg(orders.amount), min_amount: min(orders.amount), max_amount: max(orders.amount), + min_date: min(orders.date_instance), + max_date: max(orders.date_instance), })), }) @@ -93,6 +98,8 @@ describe(`Query GROUP BY Types`, () => { avg_amount: number min_amount: number max_amount: number + min_date: Date + max_date: Date } | undefined >() diff --git a/packages/db/tests/query/group-by.test.ts b/packages/db/tests/query/group-by.test.ts index 452f98f33..aec23a9f8 100644 --- a/packages/db/tests/query/group-by.test.ts +++ b/packages/db/tests/query/group-by.test.ts @@ -24,7 +24,7 @@ type Order = { customer_id: number amount: number status: string - date: string + date: Date product_category: string quantity: number discount: number @@ -64,7 +64,7 @@ const sampleOrders: Array = [ customer_id: 1, amount: 100, status: `completed`, - date: `2023-01-01`, + date: new Date(`2023-01-01`), product_category: `electronics`, quantity: 2, discount: 0, @@ -101,7 +101,7 @@ const sampleOrders: Array = [ customer_id: 1, amount: 200, status: `completed`, - date: `2023-01-15`, + date: new Date(`2023-01-15`), product_category: `electronics`, quantity: 1, discount: 10, @@ -134,7 +134,7 @@ const sampleOrders: Array = [ customer_id: 2, amount: 150, status: `pending`, - date: `2023-01-20`, + date: new Date(`2023-01-20`), product_category: `books`, quantity: 3, discount: 5, @@ -167,7 +167,7 @@ const sampleOrders: Array = [ customer_id: 2, amount: 300, status: `completed`, - date: `2023-02-01`, + date: new Date(`2023-02-01`), product_category: `electronics`, quantity: 1, discount: 0, @@ -178,7 +178,7 @@ const sampleOrders: Array = [ customer_id: 3, amount: 250, status: `pending`, - date: `2023-02-10`, + date: new Date(`2023-02-10`), product_category: `books`, quantity: 5, discount: 15, @@ -189,7 +189,7 @@ const sampleOrders: Array = [ customer_id: 3, amount: 75, status: `cancelled`, - date: `2023-02-15`, + date: new Date(`2023-02-15`), product_category: `electronics`, quantity: 1, discount: 0, @@ -200,7 +200,7 @@ const sampleOrders: Array = [ customer_id: 1, amount: 400, status: `completed`, - date: `2023-03-01`, + date: new Date(`2023-03-01`), product_category: `books`, quantity: 2, discount: 20, @@ -242,6 +242,8 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { avg_amount: avg(orders.amount), min_amount: min(orders.amount), max_amount: max(orders.amount), + min_date: min(orders.date), + max_date: max(orders.date), })), }) @@ -256,6 +258,12 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { expect(customer1?.avg_amount).toBe(233.33333333333334) // (100+200+400)/3 expect(customer1?.min_amount).toBe(100) expect(customer1?.max_amount).toBe(400) + expect(customer1?.min_date.toISOString()).toBe( + new Date(`2023-01-01`).toISOString() + ) + expect(customer1?.max_date.toISOString()).toBe( + new Date(`2023-03-01`).toISOString() + ) // Customer 2: orders 3, 4 (amounts: 150, 300) const customer2 = customerSummary.get(2) @@ -266,6 +274,12 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { expect(customer2?.avg_amount).toBe(225) // (150+300)/2 expect(customer2?.min_amount).toBe(150) expect(customer2?.max_amount).toBe(300) + expect(customer2?.min_date.toISOString()).toBe( + new Date(`2023-01-20`).toISOString() + ) + expect(customer2?.max_date.toISOString()).toBe( + new Date(`2023-02-01`).toISOString() + ) // Customer 3: orders 5, 6 (amounts: 250, 75) const customer3 = customerSummary.get(3) @@ -276,6 +290,12 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { expect(customer3?.avg_amount).toBe(162.5) // (250+75)/2 expect(customer3?.min_amount).toBe(75) expect(customer3?.max_amount).toBe(250) + expect(customer3?.min_date.toISOString()).toBe( + new Date(`2023-02-10`).toISOString() + ) + expect(customer3?.max_date.toISOString()).toBe( + new Date(`2023-02-15`).toISOString() + ) }) test(`group by customer_id with count aggregation (not null only)`, () => { @@ -735,9 +755,14 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { min_amount: min(orders.amount), max_amount: max(orders.amount), spending_range: max(orders.amount), // We'll calculate range in the filter + last_date: max(orders.date), })) .having(({ orders }) => - and(gte(min(orders.amount), 75), gte(max(orders.amount), 300)) + and( + gte(min(orders.amount), 75), + gte(max(orders.amount), 300), + gte(max(orders.date), new Date(`2020-09-17`)) + ) ), }) @@ -829,7 +854,7 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { customer_id: 1, amount: 500, status: `completed`, - date: `2023-03-15`, + date: new Date(`2023-03-15`), product_category: `electronics`, quantity: 2, discount: 0, @@ -850,7 +875,7 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { customer_id: 4, amount: 350, status: `pending`, - date: `2023-03-20`, + date: new Date(`2023-03-20`), product_category: `books`, quantity: 1, discount: 5, @@ -1031,7 +1056,7 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { customer_id: 1, amount: 100, status: `completed`, - date: `2023-01-01`, + date: new Date(`2023-01-01`), product_category: `electronics`, quantity: 1, discount: 0, @@ -1316,7 +1341,7 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { customer_id: 999, amount: 500, status: `completed`, - date: `2023-03-01`, + date: new Date(`2023-03-01`), product_category: `luxury`, quantity: 1, discount: 0, diff --git a/packages/db/tests/query/where.test.ts b/packages/db/tests/query/where.test.ts index a769f71a6..2eb78a38b 100644 --- a/packages/db/tests/query/where.test.ts +++ b/packages/db/tests/query/where.test.ts @@ -30,7 +30,7 @@ type Employee = { department_id: number | null salary: number active: boolean - hire_date: string + hire_date: Date email: string | null first_name: string last_name: string @@ -75,7 +75,7 @@ const sampleEmployees: Array = [ department_id: 1, salary: 75000, active: true, - hire_date: `2020-01-15`, + hire_date: new Date(`2020-01-15`), email: `alice@company.com`, first_name: `Alice`, last_name: `Johnson`, @@ -115,7 +115,7 @@ const sampleEmployees: Array = [ department_id: 2, salary: 65000, active: true, - hire_date: `2019-03-20`, + hire_date: new Date(`2020-01-15`), email: `bob@company.com`, first_name: `Bob`, last_name: `Smith`, @@ -149,7 +149,7 @@ const sampleEmployees: Array = [ department_id: 1, salary: 85000, active: false, - hire_date: `2018-07-10`, + hire_date: new Date(`2018-07-10`), email: null, first_name: `Charlie`, last_name: `Brown`, @@ -175,7 +175,7 @@ const sampleEmployees: Array = [ department_id: 3, salary: 95000, active: true, - hire_date: `2021-11-05`, + hire_date: new Date(`2021-11-05`), email: `diana@company.com`, first_name: `Diana`, last_name: `Miller`, @@ -201,7 +201,7 @@ const sampleEmployees: Array = [ department_id: 2, salary: 55000, active: true, - hire_date: `2022-02-14`, + hire_date: new Date(`2022-02-14`), email: `eve@company.com`, first_name: `Eve`, last_name: `Wilson`, @@ -213,7 +213,7 @@ const sampleEmployees: Array = [ department_id: null, salary: 45000, active: false, - hire_date: `2017-09-30`, + hire_date: new Date(`2017-09-30`), email: `frank@company.com`, first_name: `Frank`, last_name: `Davis`, @@ -242,6 +242,24 @@ function createWhereTests(autoIndex: `off` | `eager`): void { }) test(`eq operator - equality comparison`, () => { + const hireDateEmployee = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ emp: employeesCollection }) + .where(({ emp }) => eq(emp.hire_date, new Date(`2020-01-15`))) + .select(({ emp }) => ({ + id: emp.id, + name: emp.name, + active: emp.active, + hire_date: emp.hire_date, + })), + }) + for (const emp of hireDateEmployee.toArray) { + expect(emp.hire_date).toStrictEqual(new Date(`2020-01-15`)) + } + expect(hireDateEmployee.size).toBe(2) + const activeEmployees = createLiveQueryCollection({ startSync: true, query: (q) => @@ -278,7 +296,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { department_id: 1, salary: 70000, active: true, - hire_date: `2023-01-10`, + hire_date: new Date(`2023-01-10`), email: `grace@company.com`, first_name: `Grace`, last_name: `Lee`, @@ -340,6 +358,25 @@ function createWhereTests(autoIndex: `off` | `eager`): void { expect(seniors.size).toBe(3) // Bob (32), Charlie (35), Frank (40) + // Test with hire date + const recentHires = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ emp: employeesCollection }) + .where(({ emp }) => gt(emp.hire_date, new Date(`2020-01-01`))) + .select(({ emp }) => ({ + id: emp.id, + name: emp.name, + hire_date: emp.hire_date, + })), + }) + + for (const emp of recentHires.toArray) { + expect(emp.hire_date > new Date(`2020-01-01`)).toBe(true) + } + expect(recentHires.size).toBe(4) + // Test live updates const youngerEmployee: Employee = { id: 8, @@ -347,7 +384,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { department_id: 1, salary: 80000, // Above 70k threshold active: true, - hire_date: `2023-01-15`, + hire_date: new Date(`2023-01-15`), email: `henry@company.com`, first_name: `Henry`, last_name: `Young`, @@ -431,6 +468,25 @@ function createWhereTests(autoIndex: `off` | `eager`): void { }) expect(youngEmployees.size).toBe(3) // Alice (28), Diana (29), Eve (25) + + // Test with hire date + const recentHires = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ emp: employeesCollection }) + .where(({ emp }) => lt(emp.hire_date, new Date(`2020-01-01`))) + .select(({ emp }) => ({ + id: emp.id, + name: emp.name, + hire_date: emp.hire_date, + })), + }) + + for (const emp of recentHires.toArray) { + expect(emp.hire_date < new Date(`2020-01-01`)).toBe(true) + } + expect(recentHires.size).toBe(2) }) test(`lte operator - less than or equal comparison`, () => { @@ -999,7 +1055,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { department_id: 1, salary: 80000, // >= 70k active: true, - hire_date: `2023-01-20`, + hire_date: new Date(`2023-01-20`), email: `ian@company.com`, first_name: `Ian`, last_name: `Clark`, @@ -1063,7 +1119,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { department_id: 1, salary: 60000, active: true, - hire_date: `2023-01-25`, + hire_date: new Date(`2023-01-25`), email: `amy@company.com`, first_name: `amy`, // lowercase 'a' last_name: `stone`, @@ -1132,7 +1188,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { department_id: 1, salary: 60000, active: true, - hire_date: `2023-02-01`, + hire_date: new Date(`2023-02-01`), email: null, // null email first_name: `Jack`, last_name: `Null`, @@ -1182,7 +1238,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { department_id: 1, salary: 60000, active: true, - hire_date: `2023-02-05`, + hire_date: new Date(`2023-02-05`), email: `first@company.com`, first_name: `First`, last_name: `Employee`, @@ -1343,7 +1399,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { department_id: 1, salary: 80000, // >= 70k active: true, // active - hire_date: `2023-01-01`, + hire_date: new Date(`2023-01-01`), email: `john@company.com`, first_name: `John`, last_name: `Doe`,