From c7e7c95fa24cb303ed1169064167f6640428dfa6 Mon Sep 17 00:00:00 2001 From: Muhammad Amin Saffari Taheri Date: Thu, 21 Aug 2025 00:28:43 +0330 Subject: [PATCH 1/5] feat: add support for dates to max and min functions. --- packages/db-ivm/src/operators/groupBy.ts | 28 +++++++------- .../db-ivm/tests/operators/groupBy.test.ts | 17 ++++++--- packages/db/src/query/builder/functions.ts | 20 +++------- packages/db/src/query/compiler/evaluators.ts | 10 ++++- packages/db/src/query/compiler/group-by.ts | 18 ++++++++- packages/db/tests/query/group-by.test-d.ts | 7 ++++ packages/db/tests/query/group-by.test.ts | 38 ++++++++++++++++++- packages/db/tests/query/where.test.ts | 16 +++++++- 8 files changed, 115 insertions(+), 39 deletions(-) diff --git a/packages/db-ivm/src/operators/groupBy.ts b/packages/db-ivm/src/operators/groupBy.ts index 344c4b1c2..3f9a28346 100644 --- a/packages/db-ivm/src/operators/groupBy.ts +++ b/packages/db-ivm/src/operators/groupBy.ts @@ -211,19 +211,19 @@ export function avg( * Creates a min aggregate function that computes the minimum value in a group * @param valueExtractor Function to extract a numeric value from each data entry */ -export function min( - valueExtractor: (value: T) => number = (v) => v as unknown as number -): AggregateFunction { +export function min( + valueExtractor: (value: T) => V = (v) => v as unknown as V +): AggregateFunction { return { preMap: (data: T) => valueExtractor(data), - reduce: (values: Array<[number, number]>) => { - let minValue = Number.POSITIVE_INFINITY + reduce: (values) => { + let minValue = Number.POSITIVE_INFINITY as V for (const [value, _multiplicity] of values) { - if (value < minValue) { + if (Number(value) < Number(minValue)) { minValue = value } } - return minValue === Number.POSITIVE_INFINITY ? 0 : minValue + return minValue === Number.POSITIVE_INFINITY ? (0 as V) : minValue }, } } @@ -232,19 +232,19 @@ export function min( * Creates a max aggregate function that computes the maximum value in a group * @param valueExtractor Function to extract a numeric value from each data entry */ -export function max( - valueExtractor: (value: T) => number = (v) => v as unknown as number -): AggregateFunction { +export function max( + valueExtractor: (value: T) => V = (v) => v as unknown as V +): AggregateFunction { return { preMap: (data: T) => valueExtractor(data), - reduce: (values: Array<[number, number]>) => { - let maxValue = Number.NEGATIVE_INFINITY + reduce: (values) => { + let maxValue = Number.NEGATIVE_INFINITY as V for (const [value, _multiplicity] of values) { - if (value > maxValue) { + if (Number(value) > Number(maxValue)) { maxValue = value } } - return maxValue === Number.NEGATIVE_INFINITY ? 0 : maxValue + return maxValue === Number.NEGATIVE_INFINITY ? (0 as V) : maxValue }, } } diff --git a/packages/db-ivm/tests/operators/groupBy.test.ts b/packages/db-ivm/tests/operators/groupBy.test.ts index 5f20b8d08..e3663bfa2 100644 --- a/packages/db-ivm/tests/operators/groupBy.test.ts +++ b/packages/db-ivm/tests/operators/groupBy.test.ts @@ -454,6 +454,7 @@ describe(`Operators`, () => { const input = graph.newInput<{ category: string amount: number + date: Date }>() let latestMessage: any = null @@ -461,6 +462,8 @@ describe(`Operators`, () => { groupBy((data) => ({ category: data.category }), { minimum: min((data) => data.amount), maximum: max((data) => data.amount), + min_date: min((data) => data.date), + max_date: max((data) => data.date), }), output((message) => { latestMessage = message @@ -472,11 +475,11 @@ describe(`Operators`, () => { // Initial data input.sendData( new MultiSet([ - [{ category: `A`, amount: 10 }, 1], - [{ category: `A`, amount: 20 }, 1], - [{ category: `A`, amount: 5 }, 1], - [{ category: `B`, amount: 30 }, 1], - [{ category: `B`, amount: 15 }, 1], + [{ category: `A`, amount: 10, date: new Date(`2025/12/13`) }, 1], + [{ category: `A`, amount: 20, date: new Date(`2025/12/15`) }, 1], + [{ category: `A`, amount: 5, date: new Date(`2025/12/12`) }, 1], + [{ category: `B`, amount: 30, date: new Date(`2025/12/12`) }, 1], + [{ category: `B`, amount: 15, date: new Date(`2025/12/13`) }, 1], ]) ) @@ -493,6 +496,8 @@ describe(`Operators`, () => { category: `A`, minimum: 5, maximum: 20, + min_date: new Date(`2025/12/12`), + max_date: new Date(`2025/12/15`), }, ], 1, @@ -504,6 +509,8 @@ describe(`Operators`, () => { category: `B`, minimum: 15, maximum: 30, + min_date: new Date(`2025/12/12`), + max_date: new Date(`2025/12/13`), }, ], 1, diff --git a/packages/db/src/query/builder/functions.ts b/packages/db/src/query/builder/functions.ts index b5902bd6e..4e1a33504 100644 --- a/packages/db/src/query/builder/functions.ts +++ b/packages/db/src/query/builder/functions.ts @@ -246,23 +246,15 @@ export function sum( return new Aggregate(`sum`, [toExpression(arg)]) } -export function min( - arg: - | RefProxy - | RefProxy - | number - | BasicExpression -): Aggregate { +export function min( + arg: RefProxy | RefProxy | T | BasicExpression +): Aggregate { return new Aggregate(`min`, [toExpression(arg)]) } -export function max( - arg: - | RefProxy - | RefProxy - | number - | BasicExpression -): Aggregate { +export function max( + arg: RefProxy | RefProxy | T | BasicExpression +): Aggregate { return new Aggregate(`max`, [toExpression(arg)]) } diff --git a/packages/db/src/query/compiler/evaluators.ts b/packages/db/src/query/compiler/evaluators.ts index cbeb6ae90..2b355e8f5 100644 --- a/packages/db/src/query/compiler/evaluators.ts +++ b/packages/db/src/query/compiler/evaluators.ts @@ -142,8 +142,14 @@ function compileFunction(func: Func, isSingleRow: boolean): (data: any) => any { const argA = compiledArgs[0]! const argB = compiledArgs[1]! return (data) => { - const a = argA(data) - const b = argB(data) + let a = argA(data) + let b = argB(data) + if (a instanceof Date) { + a = a.getTime() + } + if (b instanceof Date) { + b = b.getTime() + } return a === b } } diff --git a/packages/db/src/query/compiler/group-by.ts b/packages/db/src/query/compiler/group-by.ts index 8366eb009..9069da32c 100644 --- a/packages/db/src/query/compiler/group-by.ts +++ b/packages/db/src/query/compiler/group-by.ts @@ -349,6 +349,20 @@ function getAggregateFunction(aggExpr: Aggregate) { return typeof value === `number` ? value : value != null ? Number(value) : 0 } + // Create a value extractor function for the expression to aggregate + const valueExtractorWithDate = ([, namespacedRow]: [ + string, + NamespacedRow, + ]) => { + const value = compiledExpr(namespacedRow) + // Ensure we return a number for numeric aggregate functions + return typeof value === `number` || value instanceof Date + ? value + : value != null + ? Number(value) + : 0 + } + // Return the appropriate aggregate function switch (aggExpr.name.toLowerCase()) { case `sum`: @@ -358,9 +372,9 @@ function getAggregateFunction(aggExpr: Aggregate) { case `avg`: return avg(valueExtractor) case `min`: - return min(valueExtractor) + return min(valueExtractorWithDate) case `max`: - return max(valueExtractor) + return max(valueExtractorWithDate) default: throw new UnsupportedAggregateFunctionError(aggExpr.name) } diff --git a/packages/db/tests/query/group-by.test-d.ts b/packages/db/tests/query/group-by.test-d.ts index 15e3b5703..de2a2ae96 100644 --- a/packages/db/tests/query/group-by.test-d.ts +++ b/packages/db/tests/query/group-by.test-d.ts @@ -23,6 +23,7 @@ type Order = { amount: number status: string date: string + date_instance: Date product_category: string quantity: number discount: number @@ -37,6 +38,7 @@ const sampleOrders: Array = [ amount: 100, status: `completed`, date: `2023-01-01`, + date_instance: new Date(`2023-01-01`), product_category: `electronics`, quantity: 2, discount: 0, @@ -48,6 +50,7 @@ const sampleOrders: Array = [ amount: 200, status: `completed`, date: `2023-01-15`, + date_instance: new Date(`2023-01-15`), product_category: `electronics`, quantity: 1, discount: 10, @@ -81,6 +84,8 @@ describe(`Query GROUP BY Types`, () => { avg_amount: avg(orders.amount), min_amount: min(orders.amount), max_amount: max(orders.amount), + min_date: min(orders.date_instance), + max_date: max(orders.date_instance), })), }) @@ -93,6 +98,8 @@ describe(`Query GROUP BY Types`, () => { avg_amount: number min_amount: number max_amount: number + min_date: Date + max_date: Date } | undefined >() diff --git a/packages/db/tests/query/group-by.test.ts b/packages/db/tests/query/group-by.test.ts index 684eb4727..34e5a3b5a 100644 --- a/packages/db/tests/query/group-by.test.ts +++ b/packages/db/tests/query/group-by.test.ts @@ -23,6 +23,7 @@ type Order = { amount: number status: string date: string + date_instance: Date product_category: string quantity: number discount: number @@ -37,6 +38,7 @@ const sampleOrders: Array = [ amount: 100, status: `completed`, date: `2023-01-01`, + date_instance: new Date(`2023-01-01`), product_category: `electronics`, quantity: 2, discount: 0, @@ -48,6 +50,7 @@ const sampleOrders: Array = [ amount: 200, status: `completed`, date: `2023-01-15`, + date_instance: new Date(`2023-01-15`), product_category: `electronics`, quantity: 1, discount: 10, @@ -59,6 +62,7 @@ const sampleOrders: Array = [ amount: 150, status: `pending`, date: `2023-01-20`, + date_instance: new Date(`2023-01-20`), product_category: `books`, quantity: 3, discount: 5, @@ -70,6 +74,7 @@ const sampleOrders: Array = [ amount: 300, status: `completed`, date: `2023-02-01`, + date_instance: new Date(`2023-02-01`), product_category: `electronics`, quantity: 1, discount: 0, @@ -81,6 +86,7 @@ const sampleOrders: Array = [ amount: 250, status: `pending`, date: `2023-02-10`, + date_instance: new Date(`2023-02-10`), product_category: `books`, quantity: 5, discount: 15, @@ -92,6 +98,7 @@ const sampleOrders: Array = [ amount: 75, status: `cancelled`, date: `2023-02-15`, + date_instance: new Date(`2023-02-15`), product_category: `electronics`, quantity: 1, discount: 0, @@ -103,6 +110,7 @@ const sampleOrders: Array = [ amount: 400, status: `completed`, date: `2023-03-01`, + date_instance: new Date(`2023-03-01`), product_category: `books`, quantity: 2, discount: 20, @@ -144,6 +152,8 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { avg_amount: avg(orders.amount), min_amount: min(orders.amount), max_amount: max(orders.amount), + min_date: min(orders.date_instance), + max_date: max(orders.date_instance), })), }) @@ -158,6 +168,12 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { expect(customer1?.avg_amount).toBe(233.33333333333334) // (100+200+400)/3 expect(customer1?.min_amount).toBe(100) expect(customer1?.max_amount).toBe(400) + expect(customer1?.min_date.toISOString()).toBe( + new Date(`2023-01-01`).toISOString() + ) + expect(customer1?.max_date.toISOString()).toBe( + new Date(`2023-03-01`).toISOString() + ) // Customer 2: orders 3, 4 (amounts: 150, 300) const customer2 = customerSummary.get(2) @@ -168,6 +184,12 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { expect(customer2?.avg_amount).toBe(225) // (150+300)/2 expect(customer2?.min_amount).toBe(150) expect(customer2?.max_amount).toBe(300) + expect(customer2?.min_date.toISOString()).toBe( + new Date(`2023-01-20`).toISOString() + ) + expect(customer2?.max_date.toISOString()).toBe( + new Date(`2023-02-01`).toISOString() + ) // Customer 3: orders 5, 6 (amounts: 250, 75) const customer3 = customerSummary.get(3) @@ -178,6 +200,12 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { expect(customer3?.avg_amount).toBe(162.5) // (250+75)/2 expect(customer3?.min_amount).toBe(75) expect(customer3?.max_amount).toBe(250) + expect(customer3?.min_date.toISOString()).toBe( + new Date(`2023-02-10`).toISOString() + ) + expect(customer3?.max_date.toISOString()).toBe( + new Date(`2023-02-15`).toISOString() + ) }) test(`group by status`, () => { @@ -603,9 +631,14 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { min_amount: min(orders.amount), max_amount: max(orders.amount), spending_range: max(orders.amount), // We'll calculate range in the filter + last_date: max(orders.date_instance), })) .having(({ orders }) => - and(gte(min(orders.amount), 75), gte(max(orders.amount), 300)) + and( + gte(min(orders.amount), 75), + gte(max(orders.amount), 300), + gte(max(orders.date_instance), new Date(`2020-09-17`)) + ) ), }) @@ -698,6 +731,7 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { amount: 500, status: `completed`, date: `2023-03-15`, + date_instance: new Date(`2023-03-15`), product_category: `electronics`, quantity: 2, discount: 0, @@ -719,6 +753,7 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { amount: 350, status: `pending`, date: `2023-03-20`, + date_instance: new Date(`2023-03-20`), product_category: `books`, quantity: 1, discount: 5, @@ -900,6 +935,7 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { amount: 100, status: `completed`, date: `2023-01-01`, + date_instance: new Date(`2023-01-01`), product_category: `electronics`, quantity: 1, discount: 0, diff --git a/packages/db/tests/query/where.test.ts b/packages/db/tests/query/where.test.ts index 8dc673f3e..c81725c36 100644 --- a/packages/db/tests/query/where.test.ts +++ b/packages/db/tests/query/where.test.ts @@ -29,6 +29,7 @@ type Employee = { salary: number active: boolean hire_date: string + hire_date_instance: Date email: string | null first_name: string last_name: string @@ -44,6 +45,7 @@ const sampleEmployees: Array = [ salary: 75000, active: true, hire_date: `2020-01-15`, + hire_date_instance: new Date(`2020-01-15`), email: `alice@company.com`, first_name: `Alice`, last_name: `Johnson`, @@ -55,7 +57,8 @@ const sampleEmployees: Array = [ department_id: 2, salary: 65000, active: true, - hire_date: `2019-03-20`, + hire_date: `2020-01-15`, + hire_date_instance: new Date(`2020-01-15`), email: `bob@company.com`, first_name: `Bob`, last_name: `Smith`, @@ -68,6 +71,7 @@ const sampleEmployees: Array = [ salary: 85000, active: false, hire_date: `2018-07-10`, + hire_date_instance: new Date(`2018-07-10`), email: null, first_name: `Charlie`, last_name: `Brown`, @@ -80,6 +84,7 @@ const sampleEmployees: Array = [ salary: 95000, active: true, hire_date: `2021-11-05`, + hire_date_instance: new Date(`2021-11-05`), email: `diana@company.com`, first_name: `Diana`, last_name: `Miller`, @@ -92,6 +97,7 @@ const sampleEmployees: Array = [ salary: 55000, active: true, hire_date: `2022-02-14`, + hire_date_instance: new Date(`2022-02-14`), email: `eve@company.com`, first_name: `Eve`, last_name: `Wilson`, @@ -104,6 +110,7 @@ const sampleEmployees: Array = [ salary: 45000, active: false, hire_date: `2017-09-30`, + hire_date_instance: new Date(`2017-09-30`), email: `frank@company.com`, first_name: `Frank`, last_name: `Davis`, @@ -169,6 +176,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { salary: 70000, active: true, hire_date: `2023-01-10`, + hire_date_instance: new Date(`2023-01-10`), email: `grace@company.com`, first_name: `Grace`, last_name: `Lee`, @@ -238,6 +246,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { salary: 80000, // Above 70k threshold active: true, hire_date: `2023-01-15`, + hire_date_instance: new Date(`2023-01-15`), email: `henry@company.com`, first_name: `Henry`, last_name: `Young`, @@ -890,6 +899,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { salary: 80000, // >= 70k active: true, hire_date: `2023-01-20`, + hire_date_instance: new Date(`2023-01-20`), email: `ian@company.com`, first_name: `Ian`, last_name: `Clark`, @@ -954,6 +964,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { salary: 60000, active: true, hire_date: `2023-01-25`, + hire_date_instance: new Date(`2023-01-25`), email: `amy@company.com`, first_name: `amy`, // lowercase 'a' last_name: `stone`, @@ -1023,6 +1034,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { salary: 60000, active: true, hire_date: `2023-02-01`, + hire_date_instance: new Date(`2023-02-01`), email: null, // null email first_name: `Jack`, last_name: `Null`, @@ -1073,6 +1085,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { salary: 60000, active: true, hire_date: `2023-02-05`, + hire_date_instance: new Date(`2023-02-05`), email: `first@company.com`, first_name: `First`, last_name: `Employee`, @@ -1234,6 +1247,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { salary: 80000, // >= 70k active: true, // active hire_date: `2023-01-01`, + hire_date_instance: new Date(`2023-01-01`), email: `john@company.com`, first_name: `John`, last_name: `Doe`, From a8d6e280d05457fc99b8a4e4177d079f580b957e Mon Sep 17 00:00:00 2001 From: Muhammad Amin Saffari Taheri Date: Thu, 21 Aug 2025 00:43:33 +0330 Subject: [PATCH 2/5] feat: enable eq comparison for date and add a tests. --- packages/db/src/indexes/btree-index.ts | 39 ++++++++++++++++++++------ packages/db/tests/query/where.test.ts | 16 +++++++++++ 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/packages/db/src/indexes/btree-index.ts b/packages/db/src/indexes/btree-index.ts index 07edf700c..68ca43a62 100644 --- a/packages/db/src/indexes/btree-index.ts +++ b/packages/db/src/indexes/btree-index.ts @@ -4,6 +4,21 @@ import { BaseIndex } from "./base-index.js" import type { BasicExpression } from "../query/ir.js" import type { IndexOperation } from "./base-index.js" +/** + * Normalizes values for use as Map keys to ensure proper equality comparison + * For Date objects, uses timestamp. For other objects, uses JSON serialization. + */ +function normalizeMapKey(value: any): any { + if (value instanceof Date) { + return value.getTime() + } + if (typeof value === `object` && value !== null) { + // For other objects, use JSON serialization as a fallback + // This ensures objects with same content are treated as equal + return JSON.stringify(value) + } + return value +} /** * Options for Ordered index */ @@ -71,14 +86,17 @@ export class BTreeIndex< ) } + // Normalize the value for Map key usage + const normalizedValue = normalizeMapKey(indexedValue) + // Check if this value already exists - if (this.valueMap.has(indexedValue)) { + if (this.valueMap.has(normalizedValue)) { // Add to existing set - this.valueMap.get(indexedValue)!.add(key) + this.valueMap.get(normalizedValue)!.add(key) } else { // Create new set for this value const keySet = new Set([key]) - this.valueMap.set(indexedValue, keySet) + this.valueMap.set(normalizedValue, keySet) this.orderedEntries.set(indexedValue, undefined) } @@ -101,13 +119,16 @@ export class BTreeIndex< return } - if (this.valueMap.has(indexedValue)) { - const keySet = this.valueMap.get(indexedValue)! + // Normalize the value for Map key usage + const normalizedValue = normalizeMapKey(indexedValue) + + if (this.valueMap.has(normalizedValue)) { + const keySet = this.valueMap.get(normalizedValue)! keySet.delete(key) // If set is now empty, remove the entry entirely if (keySet.size === 0) { - this.valueMap.delete(indexedValue) + this.valueMap.delete(normalizedValue) // Remove from ordered entries this.orderedEntries.delete(indexedValue) @@ -195,7 +216,8 @@ export class BTreeIndex< * Performs an equality lookup */ equalityLookup(value: any): Set { - return new Set(this.valueMap.get(value) ?? []) + const normalizedValue = normalizeMapKey(value) + return new Set(this.valueMap.get(normalizedValue) ?? []) } /** @@ -266,7 +288,8 @@ export class BTreeIndex< const result = new Set() for (const value of values) { - const keys = this.valueMap.get(value) + const normalizedValue = normalizeMapKey(value) + const keys = this.valueMap.get(normalizedValue) if (keys) { keys.forEach((key) => result.add(key)) } diff --git a/packages/db/tests/query/where.test.ts b/packages/db/tests/query/where.test.ts index c81725c36..e11142431 100644 --- a/packages/db/tests/query/where.test.ts +++ b/packages/db/tests/query/where.test.ts @@ -139,6 +139,22 @@ function createWhereTests(autoIndex: `off` | `eager`): void { }) test(`eq operator - equality comparison`, () => { + const hireDateEmployee = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ emp: employeesCollection }) + .where(({ emp }) => + eq(emp.hire_date_instance, new Date(`2020-01-15`)) + ) + .select(({ emp }) => ({ + id: emp.id, + name: emp.name, + active: emp.active, + })), + }) + expect(hireDateEmployee.size).toBe(2) + const activeEmployees = createLiveQueryCollection({ startSync: true, query: (q) => From 6bab62aec5548b49338832297e4ea1190d256a48 Mon Sep 17 00:00:00 2001 From: Muhammad Amin Saffari Taheri Date: Thu, 28 Aug 2025 02:57:31 +0330 Subject: [PATCH 3/5] fix: remove unused generic value. --- packages/db-ivm/src/operators/groupBy.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/db-ivm/src/operators/groupBy.ts b/packages/db-ivm/src/operators/groupBy.ts index 3f9a28346..0b428e5ac 100644 --- a/packages/db-ivm/src/operators/groupBy.ts +++ b/packages/db-ivm/src/operators/groupBy.ts @@ -211,9 +211,9 @@ export function avg( * Creates a min aggregate function that computes the minimum value in a group * @param valueExtractor Function to extract a numeric value from each data entry */ -export function min( +export function min( valueExtractor: (value: T) => V = (v) => v as unknown as V -): AggregateFunction { +): AggregateFunction { return { preMap: (data: T) => valueExtractor(data), reduce: (values) => { @@ -232,9 +232,9 @@ export function min( * Creates a max aggregate function that computes the maximum value in a group * @param valueExtractor Function to extract a numeric value from each data entry */ -export function max( +export function max( valueExtractor: (value: T) => V = (v) => v as unknown as V -): AggregateFunction { +): AggregateFunction { return { preMap: (data: T) => valueExtractor(data), reduce: (values) => { From d7d791471e5ad88ee212586cc3c5190a95404db3 Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 24 Sep 2025 20:42:54 +0100 Subject: [PATCH 4/5] various fixes --- packages/db-ivm/src/operators/groupBy.ts | 24 +++--- packages/db/src/indexes/btree-index.ts | 46 +++++----- packages/db/src/query/builder/functions.ts | 6 +- packages/db/src/query/compiler/evaluators.ts | 11 +-- packages/db/src/query/compiler/group-by.ts | 2 + packages/db/src/utils/comparison.ts | 2 +- packages/db/tests/query/group-by.test.ts | 43 ++++------ packages/db/tests/query/where.test.ts | 88 +++++++++++++------- 8 files changed, 119 insertions(+), 103 deletions(-) diff --git a/packages/db-ivm/src/operators/groupBy.ts b/packages/db-ivm/src/operators/groupBy.ts index 958157dad..ef5810ac6 100644 --- a/packages/db-ivm/src/operators/groupBy.ts +++ b/packages/db-ivm/src/operators/groupBy.ts @@ -214,19 +214,21 @@ export function avg( * Creates a min aggregate function that computes the minimum value in a group * @param valueExtractor Function to extract a numeric value from each data entry */ -export function min( +export function min( valueExtractor: (value: T) => V = (v) => v as unknown as V -): AggregateFunction { +): AggregateFunction { return { preMap: (data: T) => valueExtractor(data), reduce: (values) => { - let minValue = Number.POSITIVE_INFINITY as V + let minValue: V | undefined for (const [value, _multiplicity] of values) { - if (Number(value) < Number(minValue)) { + if (minValue && value < minValue) { + minValue = value + } else if (!minValue) { minValue = value } } - return minValue === Number.POSITIVE_INFINITY ? (0 as V) : minValue + return minValue ?? (0 as V) }, } } @@ -235,19 +237,21 @@ export function min( * Creates a max aggregate function that computes the maximum value in a group * @param valueExtractor Function to extract a numeric value from each data entry */ -export function max( +export function max( valueExtractor: (value: T) => V = (v) => v as unknown as V -): AggregateFunction { +): AggregateFunction { return { preMap: (data: T) => valueExtractor(data), reduce: (values) => { - let maxValue = Number.NEGATIVE_INFINITY as V + let maxValue: V | undefined for (const [value, _multiplicity] of values) { - if (Number(value) > Number(maxValue)) { + if (maxValue && value > maxValue) { + maxValue = value + } else if (!maxValue) { maxValue = value } } - return maxValue === Number.NEGATIVE_INFINITY ? (0 as V) : maxValue + return maxValue ?? (0 as V) }, } } diff --git a/packages/db/src/indexes/btree-index.ts b/packages/db/src/indexes/btree-index.ts index 6394bedcb..ceda36e1b 100644 --- a/packages/db/src/indexes/btree-index.ts +++ b/packages/db/src/indexes/btree-index.ts @@ -4,21 +4,6 @@ import { BaseIndex } from "./base-index.js" import type { BasicExpression } from "../query/ir.js" import type { IndexOperation } from "./base-index.js" -/** - * Normalizes values for use as Map keys to ensure proper equality comparison - * For Date objects, uses timestamp. For other objects, uses JSON serialization. - */ -function normalizeMapKey(value: any): any { - if (value instanceof Date) { - return value.getTime() - } - if (typeof value === `object` && value !== null) { - // For other objects, use JSON serialization as a fallback - // This ensures objects with same content are treated as equal - return JSON.stringify(value) - } - return value -} /** * Options for Ordered index */ @@ -87,7 +72,7 @@ export class BTreeIndex< } // Normalize the value for Map key usage - const normalizedValue = normalizeMapKey(indexedValue) + const normalizedValue = this.normalizeMapKey(indexedValue) // Check if this value already exists if (this.valueMap.has(normalizedValue)) { @@ -97,7 +82,7 @@ export class BTreeIndex< // Create new set for this value const keySet = new Set([key]) this.valueMap.set(normalizedValue, keySet) - this.orderedEntries.set(indexedValue, undefined) + this.orderedEntries.set(normalizedValue, undefined) } this.indexedKeys.add(key) @@ -120,7 +105,7 @@ export class BTreeIndex< } // Normalize the value for Map key usage - const normalizedValue = normalizeMapKey(indexedValue) + const normalizedValue = this.normalizeMapKey(indexedValue) if (this.valueMap.has(normalizedValue)) { const keySet = this.valueMap.get(normalizedValue)! @@ -131,7 +116,7 @@ export class BTreeIndex< this.valueMap.delete(normalizedValue) // Remove from ordered entries - this.orderedEntries.delete(indexedValue) + this.orderedEntries.delete(normalizedValue) } } @@ -216,7 +201,7 @@ export class BTreeIndex< * Performs an equality lookup */ equalityLookup(value: any): Set { - const normalizedValue = normalizeMapKey(value) + const normalizedValue = this.normalizeMapKey(value) return new Set(this.valueMap.get(normalizedValue) ?? []) } @@ -228,8 +213,10 @@ export class BTreeIndex< const { from, to, fromInclusive = true, toInclusive = true } = options const result = new Set() - const fromKey = from ?? this.orderedEntries.minKey() - const toKey = to ?? this.orderedEntries.maxKey() + const normalizedFrom = this.normalizeMapKey(from) + const normalizedTo = this.normalizeMapKey(to) + const fromKey = normalizedFrom ?? this.orderedEntries.minKey() + const toKey = normalizedTo ?? this.orderedEntries.maxKey() this.orderedEntries.forRange( fromKey, @@ -262,7 +249,7 @@ export class BTreeIndex< const keysInResult: Set = new Set() const result: Array = [] const nextKey = (k?: any) => this.orderedEntries.nextHigherKey(k) - let key = from + let key = this.normalizeMapKey(from) while ((key = nextKey(key)) && result.length < n) { const keys = this.valueMap.get(key) @@ -288,7 +275,7 @@ export class BTreeIndex< const result = new Set() for (const value of values) { - const normalizedValue = normalizeMapKey(value) + const normalizedValue = this.normalizeMapKey(value) const keys = this.valueMap.get(normalizedValue) if (keys) { keys.forEach((key) => result.add(key)) @@ -312,4 +299,15 @@ export class BTreeIndex< get valueMapData(): Map> { return this.valueMap } + + /** + * Normalizes values for use as Map keys to ensure proper equality and range queries + * For Date objects, uses timestamp. + */ + private normalizeMapKey(value: any): any { + if (value instanceof Date) { + return value.valueOf() + } + return value + } } diff --git a/packages/db/src/query/builder/functions.ts b/packages/db/src/query/builder/functions.ts index f1281ae2b..402c6de97 100644 --- a/packages/db/src/query/builder/functions.ts +++ b/packages/db/src/query/builder/functions.ts @@ -53,10 +53,10 @@ type ExtractType = // Helper type to determine aggregate return type based on input nullability type AggregateReturnType = ExtractType extends infer U - ? U extends number | undefined | null + ? U extends number | undefined | null | Date | bigint ? Aggregate - : Aggregate - : Aggregate + : Aggregate + : Aggregate // Helper type to determine string function return type based on input nullability type StringFunctionReturnType = diff --git a/packages/db/src/query/compiler/evaluators.ts b/packages/db/src/query/compiler/evaluators.ts index 56c56013c..d41881eb9 100644 --- a/packages/db/src/query/compiler/evaluators.ts +++ b/packages/db/src/query/compiler/evaluators.ts @@ -142,13 +142,10 @@ function compileFunction(func: Func, isSingleRow: boolean): (data: any) => any { const argA = compiledArgs[0]! const argB = compiledArgs[1]! return (data) => { - let a = argA(data) - let b = argB(data) - if (a instanceof Date) { - a = a.getTime() - } - if (b instanceof Date) { - b = b.getTime() + const a = argA(data) + const b = argB(data) + if (a instanceof Date && b instanceof Date) { + return a.valueOf() == b.valueOf() } return a === b } diff --git a/packages/db/src/query/compiler/group-by.ts b/packages/db/src/query/compiler/group-by.ts index 41681eb96..e2bd10109 100644 --- a/packages/db/src/query/compiler/group-by.ts +++ b/packages/db/src/query/compiler/group-by.ts @@ -361,6 +361,8 @@ function getAggregateFunction(aggExpr: Aggregate) { : value != null ? Number(value) : 0 + } + // Create a raw value extractor function for the expression to aggregate const rawValueExtractor = ([, namespacedRow]: [string, NamespacedRow]) => { return compiledExpr(namespacedRow) diff --git a/packages/db/src/utils/comparison.ts b/packages/db/src/utils/comparison.ts index d8826f104..fa8750610 100644 --- a/packages/db/src/utils/comparison.ts +++ b/packages/db/src/utils/comparison.ts @@ -51,7 +51,7 @@ export const ascComparator = (a: any, b: any, opts: CompareOptions): number => { // If both are dates, compare them if (a instanceof Date && b instanceof Date) { - return a.getTime() - b.getTime() + return a.valueOf() - b.valueOf() } // If at least one of the values is an object, use stable IDs for comparison diff --git a/packages/db/tests/query/group-by.test.ts b/packages/db/tests/query/group-by.test.ts index 24a035865..aec23a9f8 100644 --- a/packages/db/tests/query/group-by.test.ts +++ b/packages/db/tests/query/group-by.test.ts @@ -24,8 +24,7 @@ type Order = { customer_id: number amount: number status: string - date: string - date_instance: Date + date: Date product_category: string quantity: number discount: number @@ -65,8 +64,7 @@ const sampleOrders: Array = [ customer_id: 1, amount: 100, status: `completed`, - date: `2023-01-01`, - date_instance: new Date(`2023-01-01`), + date: new Date(`2023-01-01`), product_category: `electronics`, quantity: 2, discount: 0, @@ -103,8 +101,7 @@ const sampleOrders: Array = [ customer_id: 1, amount: 200, status: `completed`, - date: `2023-01-15`, - date_instance: new Date(`2023-01-15`), + date: new Date(`2023-01-15`), product_category: `electronics`, quantity: 1, discount: 10, @@ -137,8 +134,7 @@ const sampleOrders: Array = [ customer_id: 2, amount: 150, status: `pending`, - date: `2023-01-20`, - date_instance: new Date(`2023-01-20`), + date: new Date(`2023-01-20`), product_category: `books`, quantity: 3, discount: 5, @@ -171,8 +167,7 @@ const sampleOrders: Array = [ customer_id: 2, amount: 300, status: `completed`, - date: `2023-02-01`, - date_instance: new Date(`2023-02-01`), + date: new Date(`2023-02-01`), product_category: `electronics`, quantity: 1, discount: 0, @@ -183,8 +178,7 @@ const sampleOrders: Array = [ customer_id: 3, amount: 250, status: `pending`, - date: `2023-02-10`, - date_instance: new Date(`2023-02-10`), + date: new Date(`2023-02-10`), product_category: `books`, quantity: 5, discount: 15, @@ -195,8 +189,7 @@ const sampleOrders: Array = [ customer_id: 3, amount: 75, status: `cancelled`, - date: `2023-02-15`, - date_instance: new Date(`2023-02-15`), + date: new Date(`2023-02-15`), product_category: `electronics`, quantity: 1, discount: 0, @@ -207,8 +200,7 @@ const sampleOrders: Array = [ customer_id: 1, amount: 400, status: `completed`, - date: `2023-03-01`, - date_instance: new Date(`2023-03-01`), + date: new Date(`2023-03-01`), product_category: `books`, quantity: 2, discount: 20, @@ -250,8 +242,8 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { avg_amount: avg(orders.amount), min_amount: min(orders.amount), max_amount: max(orders.amount), - min_date: min(orders.date_instance), - max_date: max(orders.date_instance), + min_date: min(orders.date), + max_date: max(orders.date), })), }) @@ -763,13 +755,13 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { min_amount: min(orders.amount), max_amount: max(orders.amount), spending_range: max(orders.amount), // We'll calculate range in the filter - last_date: max(orders.date_instance), + last_date: max(orders.date), })) .having(({ orders }) => and( gte(min(orders.amount), 75), gte(max(orders.amount), 300), - gte(max(orders.date_instance), new Date(`2020-09-17`)) + gte(max(orders.date), new Date(`2020-09-17`)) ) ), }) @@ -862,8 +854,7 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { customer_id: 1, amount: 500, status: `completed`, - date: `2023-03-15`, - date_instance: new Date(`2023-03-15`), + date: new Date(`2023-03-15`), product_category: `electronics`, quantity: 2, discount: 0, @@ -884,8 +875,7 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { customer_id: 4, amount: 350, status: `pending`, - date: `2023-03-20`, - date_instance: new Date(`2023-03-20`), + date: new Date(`2023-03-20`), product_category: `books`, quantity: 1, discount: 5, @@ -1066,8 +1056,7 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { customer_id: 1, amount: 100, status: `completed`, - date: `2023-01-01`, - date_instance: new Date(`2023-01-01`), + date: new Date(`2023-01-01`), product_category: `electronics`, quantity: 1, discount: 0, @@ -1352,7 +1341,7 @@ function createGroupByTests(autoIndex: `off` | `eager`): void { customer_id: 999, amount: 500, status: `completed`, - date: `2023-03-01`, + date: new Date(`2023-03-01`), product_category: `luxury`, quantity: 1, discount: 0, diff --git a/packages/db/tests/query/where.test.ts b/packages/db/tests/query/where.test.ts index afbc470f2..2eb78a38b 100644 --- a/packages/db/tests/query/where.test.ts +++ b/packages/db/tests/query/where.test.ts @@ -30,8 +30,7 @@ type Employee = { department_id: number | null salary: number active: boolean - hire_date: string - hire_date_instance: Date + hire_date: Date email: string | null first_name: string last_name: string @@ -76,8 +75,7 @@ const sampleEmployees: Array = [ department_id: 1, salary: 75000, active: true, - hire_date: `2020-01-15`, - hire_date_instance: new Date(`2020-01-15`), + hire_date: new Date(`2020-01-15`), email: `alice@company.com`, first_name: `Alice`, last_name: `Johnson`, @@ -117,8 +115,7 @@ const sampleEmployees: Array = [ department_id: 2, salary: 65000, active: true, - hire_date: `2020-01-15`, - hire_date_instance: new Date(`2020-01-15`), + hire_date: new Date(`2020-01-15`), email: `bob@company.com`, first_name: `Bob`, last_name: `Smith`, @@ -152,8 +149,7 @@ const sampleEmployees: Array = [ department_id: 1, salary: 85000, active: false, - hire_date: `2018-07-10`, - hire_date_instance: new Date(`2018-07-10`), + hire_date: new Date(`2018-07-10`), email: null, first_name: `Charlie`, last_name: `Brown`, @@ -179,8 +175,7 @@ const sampleEmployees: Array = [ department_id: 3, salary: 95000, active: true, - hire_date: `2021-11-05`, - hire_date_instance: new Date(`2021-11-05`), + hire_date: new Date(`2021-11-05`), email: `diana@company.com`, first_name: `Diana`, last_name: `Miller`, @@ -206,8 +201,7 @@ const sampleEmployees: Array = [ department_id: 2, salary: 55000, active: true, - hire_date: `2022-02-14`, - hire_date_instance: new Date(`2022-02-14`), + hire_date: new Date(`2022-02-14`), email: `eve@company.com`, first_name: `Eve`, last_name: `Wilson`, @@ -219,8 +213,7 @@ const sampleEmployees: Array = [ department_id: null, salary: 45000, active: false, - hire_date: `2017-09-30`, - hire_date_instance: new Date(`2017-09-30`), + hire_date: new Date(`2017-09-30`), email: `frank@company.com`, first_name: `Frank`, last_name: `Davis`, @@ -254,15 +247,17 @@ function createWhereTests(autoIndex: `off` | `eager`): void { query: (q) => q .from({ emp: employeesCollection }) - .where(({ emp }) => - eq(emp.hire_date_instance, new Date(`2020-01-15`)) - ) + .where(({ emp }) => eq(emp.hire_date, new Date(`2020-01-15`))) .select(({ emp }) => ({ id: emp.id, name: emp.name, active: emp.active, + hire_date: emp.hire_date, })), }) + for (const emp of hireDateEmployee.toArray) { + expect(emp.hire_date).toStrictEqual(new Date(`2020-01-15`)) + } expect(hireDateEmployee.size).toBe(2) const activeEmployees = createLiveQueryCollection({ @@ -301,8 +296,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { department_id: 1, salary: 70000, active: true, - hire_date: `2023-01-10`, - hire_date_instance: new Date(`2023-01-10`), + hire_date: new Date(`2023-01-10`), email: `grace@company.com`, first_name: `Grace`, last_name: `Lee`, @@ -364,6 +358,25 @@ function createWhereTests(autoIndex: `off` | `eager`): void { expect(seniors.size).toBe(3) // Bob (32), Charlie (35), Frank (40) + // Test with hire date + const recentHires = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ emp: employeesCollection }) + .where(({ emp }) => gt(emp.hire_date, new Date(`2020-01-01`))) + .select(({ emp }) => ({ + id: emp.id, + name: emp.name, + hire_date: emp.hire_date, + })), + }) + + for (const emp of recentHires.toArray) { + expect(emp.hire_date > new Date(`2020-01-01`)).toBe(true) + } + expect(recentHires.size).toBe(4) + // Test live updates const youngerEmployee: Employee = { id: 8, @@ -371,8 +384,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { department_id: 1, salary: 80000, // Above 70k threshold active: true, - hire_date: `2023-01-15`, - hire_date_instance: new Date(`2023-01-15`), + hire_date: new Date(`2023-01-15`), email: `henry@company.com`, first_name: `Henry`, last_name: `Young`, @@ -456,6 +468,25 @@ function createWhereTests(autoIndex: `off` | `eager`): void { }) expect(youngEmployees.size).toBe(3) // Alice (28), Diana (29), Eve (25) + + // Test with hire date + const recentHires = createLiveQueryCollection({ + startSync: true, + query: (q) => + q + .from({ emp: employeesCollection }) + .where(({ emp }) => lt(emp.hire_date, new Date(`2020-01-01`))) + .select(({ emp }) => ({ + id: emp.id, + name: emp.name, + hire_date: emp.hire_date, + })), + }) + + for (const emp of recentHires.toArray) { + expect(emp.hire_date < new Date(`2020-01-01`)).toBe(true) + } + expect(recentHires.size).toBe(2) }) test(`lte operator - less than or equal comparison`, () => { @@ -1024,8 +1055,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { department_id: 1, salary: 80000, // >= 70k active: true, - hire_date: `2023-01-20`, - hire_date_instance: new Date(`2023-01-20`), + hire_date: new Date(`2023-01-20`), email: `ian@company.com`, first_name: `Ian`, last_name: `Clark`, @@ -1089,8 +1119,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { department_id: 1, salary: 60000, active: true, - hire_date: `2023-01-25`, - hire_date_instance: new Date(`2023-01-25`), + hire_date: new Date(`2023-01-25`), email: `amy@company.com`, first_name: `amy`, // lowercase 'a' last_name: `stone`, @@ -1159,8 +1188,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { department_id: 1, salary: 60000, active: true, - hire_date: `2023-02-01`, - hire_date_instance: new Date(`2023-02-01`), + hire_date: new Date(`2023-02-01`), email: null, // null email first_name: `Jack`, last_name: `Null`, @@ -1210,8 +1238,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { department_id: 1, salary: 60000, active: true, - hire_date: `2023-02-05`, - hire_date_instance: new Date(`2023-02-05`), + hire_date: new Date(`2023-02-05`), email: `first@company.com`, first_name: `First`, last_name: `Employee`, @@ -1372,8 +1399,7 @@ function createWhereTests(autoIndex: `off` | `eager`): void { department_id: 1, salary: 80000, // >= 70k active: true, // active - hire_date: `2023-01-01`, - hire_date_instance: new Date(`2023-01-01`), + hire_date: new Date(`2023-01-01`), email: `john@company.com`, first_name: `John`, last_name: `Doe`, From 41b1076acdc5baf5d296388a10ed928dc08e0f3b Mon Sep 17 00:00:00 2001 From: Sam Willis Date: Wed, 24 Sep 2025 20:45:17 +0100 Subject: [PATCH 5/5] changeset --- .changeset/brave-rocks-lie.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/brave-rocks-lie.md diff --git a/.changeset/brave-rocks-lie.md b/.changeset/brave-rocks-lie.md new file mode 100644 index 000000000..f16aea92a --- /dev/null +++ b/.changeset/brave-rocks-lie.md @@ -0,0 +1,6 @@ +--- +"@tanstack/db-ivm": patch +"@tanstack/db": patch +--- + +Add support for Date objects to min/max aggregates and range queries when using an index.