From 239dccb94707a2c8eaaa4b6cfe63836bc10d2694 Mon Sep 17 00:00:00 2001 From: Erez Rokah Date: Thu, 17 Aug 2023 11:07:55 +0200 Subject: [PATCH 1/2] fix: Remove date in favor of timestamp, handle interval, pass underlying value to arrow (#62) --- src/scalar/date.ts | 83 ------------------------------------------ src/scalar/scalar.ts | 11 ++++-- src/schema/resource.ts | 7 +--- 3 files changed, 9 insertions(+), 92 deletions(-) delete mode 100644 src/scalar/date.ts diff --git a/src/scalar/date.ts b/src/scalar/date.ts deleted file mode 100644 index cd0310c..0000000 --- a/src/scalar/date.ts +++ /dev/null @@ -1,83 +0,0 @@ -import type { DataType, DateUnit } from '@apache-arrow/esnext-esm'; -import { Date_ as ArrowDate } from '@apache-arrow/esnext-esm'; -import { DateTime } from 'luxon'; - -import { FormatError } from '../errors/errors.js'; -import type { Nullable } from '../schema/types.js'; - -import type { Scalar } from './scalar.js'; -import { isInvalid, NULL_VALUE } from './util.js'; - -export class Date implements Scalar> { - private _valid = false; - private _value: Nullable = null; - private _unit: DateUnit; - - public constructor(unit: DateUnit, v?: unknown) { - this._unit = unit; - this.value = v; - return this; - } - - public get dataType(): DataType { - return new ArrowDate(this._unit); - } - - public get valid(): boolean { - return this._valid; - } - - public get value(): Nullable { - if (!this._valid) { - return null; - } - return this._value; - } - - public set value(value: unknown) { - if (isInvalid(value)) { - this._valid = false; - return; - } - - if (value instanceof Date && value.dataType === this.dataType) { - this._valid = value.valid; - this._value = value.value; - return; - } - - let dateValue: DateTime | null = null; - - if (typeof value === 'string') { - dateValue = DateTime.fromISO(value, { setZone: true }); - - if (!dateValue.isValid) { - dateValue = DateTime.fromFormat(value, 'yyyy-MM-dd', { zone: 'utc' }); - } - } - - if (value instanceof DateTime) { - dateValue = value; - } - - if (value instanceof globalThis.Date) { - dateValue = DateTime.fromJSDate(value, { zone: 'utc' }); - } - - if (dateValue && dateValue.isValid) { - this._value = dateValue.toJSDate(); - this._valid = true; - return; - } - - throw new FormatError(`Unable to set Date from value`, { props: { value } }); - } - - public toString(): string { - if (this._valid) { - return this._value!.toISOString(); - } - - return NULL_VALUE; - } -} diff --git a/src/scalar/scalar.ts b/src/scalar/scalar.ts index ec88821..c85d98f 100644 --- a/src/scalar/scalar.ts +++ b/src/scalar/scalar.ts @@ -1,10 +1,9 @@ -import { DataType, Precision } from '@apache-arrow/esnext-esm'; +import { DataType, Precision, TimeUnit } from '@apache-arrow/esnext-esm'; import { JSONType } from '../types/json.js'; import { UUIDType } from '../types/uuid.js'; import { Bool } from './bool.js'; -import { Date } from './date.js'; import { Float32 } from './float32.js'; import { Float64 } from './float64.js'; import { Int16 } from './int16.js'; @@ -90,7 +89,13 @@ export const newScalar = (dataType: DataType): Scalar => { } if (DataType.isDate(dataType)) { - return new Date(dataType.unit); + // TODO: Add Date support + return new Timestamp(TimeUnit.SECOND); + } + + if (DataType.isInterval(dataType)) { + // TODO: Add Interval support + return new JSONScalar(); } if (dataType instanceof UUIDType) { diff --git a/src/schema/resource.ts b/src/schema/resource.ts index e80304a..a22cede 100644 --- a/src/schema/resource.ts +++ b/src/schema/resource.ts @@ -3,7 +3,6 @@ import { tableToIPC, Table as ArrowTable, RecordBatch, vectorFromArray } from '@ import { ResourceError } from '../errors/errors.js'; import type { Scalar, Vector } from '../scalar/scalar.js'; import { newScalar } from '../scalar/scalar.js'; -import { isExtensionType } from '../types/extensions.js'; import { cqIDColumn } from './meta.js'; import type { Table } from './table.js'; @@ -64,11 +63,7 @@ export const encodeResource = (resource: Resource): Uint8Array => { let batch = new RecordBatch(schema, undefined); for (let index = 0; index < table.columns.length; index++) { const column = table.columns[index]; - // For extension types, we need to get the underlying value - const data = isExtensionType(column.type) - ? resource.getColumnData(column.name).value - : resource.getColumnData(column.name); - + const data = resource.getColumnData(column.name).value; const vector = vectorFromArray([data], column.type); batch = batch.setChildAt(index, vector); } From 7a4c4a54d7fcfb266d7cd75c371b86a378ac7969 Mon Sep 17 00:00:00 2001 From: CloudQuery Bot <102256036+cq-bot@users.noreply.github.com> Date: Thu, 17 Aug 2023 12:09:09 +0300 Subject: [PATCH 2/2] chore(main): Release v0.0.4 (#63) :robot: I have created a release *beep* *boop* --- ## [0.0.4](https://github.com/cloudquery/plugin-sdk-javascript/compare/v0.0.3...v0.0.4) (2023-08-17) ### Bug Fixes * Remove date in favor of timestamp, handle interval, pass underlying value to arrow ([#62](https://github.com/cloudquery/plugin-sdk-javascript/issues/62)) ([239dccb](https://github.com/cloudquery/plugin-sdk-javascript/commit/239dccb94707a2c8eaaa4b6cfe63836bc10d2694)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 124d46c..c2afc5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.0.4](https://github.com/cloudquery/plugin-sdk-javascript/compare/v0.0.3...v0.0.4) (2023-08-17) + + +### Bug Fixes + +* Remove date in favor of timestamp, handle interval, pass underlying value to arrow ([#62](https://github.com/cloudquery/plugin-sdk-javascript/issues/62)) ([239dccb](https://github.com/cloudquery/plugin-sdk-javascript/commit/239dccb94707a2c8eaaa4b6cfe63836bc10d2694)) + ## [0.0.3](https://github.com/cloudquery/plugin-sdk-javascript/compare/v0.0.2...v0.0.3) (2023-08-16) diff --git a/package-lock.json b/package-lock.json index 8b30974..4e647a8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@cloudquery/plugin-sdk-javascript", - "version": "0.0.3", + "version": "0.0.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@cloudquery/plugin-sdk-javascript", - "version": "0.0.3", + "version": "0.0.4", "license": "MPL-2.0", "dependencies": { "@apache-arrow/esnext-esm": "^12.0.1", diff --git a/package.json b/package.json index 3ce2735..9e587b8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@cloudquery/plugin-sdk-javascript", - "version": "0.0.3", + "version": "0.0.4", "files": [ "dist", "!dist/**/*.test.*",