Skip to content

fix(nested avro): fixing key / value schemas after adding version #3093

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shirshanka merged 1 commit into from
Aug 13, 2021
Merged

fix(nested avro): fixing key / value schemas after adding version #3093

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...ile/__tests__/translateFieldPath.test.tsx → ...ests__/schema/translateFieldPath.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import translateFieldPath from '../schema/utils/translateFieldPath';
import translateFieldPath from '../../schema/utils/translateFieldPath';

describe('translateFieldPath', () => {
it('translates qualified unions', () => {
Expand Down
2 changes: 1 addition & 1 deletion ...ests__/translateFieldPathSegment.test.tsx → ...schema/translateFieldPathSegment.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import translateFieldPathSegment from '../schema/utils/translateFieldPathSegment';
import translateFieldPathSegment from '../../schema/utils/translateFieldPathSegment';

describe('translateFieldPathSegment', () => {
it('translates unions', () => {
Expand Down
49 changes: 49 additions & 0 deletions datahub-web-react/src/app/entity/dataset/profile/__tests__/schema/utils.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import { SchemaFieldDataType } from '../../../../../../types.generated';
import { filterKeyFieldPath } from '../../schema/utils/utils';

describe('utils', () => {
describe('filterKeyFieldPath', () => {
it('allows keys when looking for keys', () => {
expect(
filterKeyFieldPath(true, {
fieldPath: '[version=2.0].[key=True].[type=long].field',
nullable: false,
type: SchemaFieldDataType.Number,
recursive: false,
}),
).toEqual(true);
});
it('blocks non-keys when looking for keys', () => {
expect(
filterKeyFieldPath(true, {
fieldPath: '[version=2.0].[type=long].field',
nullable: false,
type: SchemaFieldDataType.Number,
recursive: false,
}),
).toEqual(false);
});

it('allows non-keys when looking for non-keys', () => {
expect(
filterKeyFieldPath(false, {
fieldPath: '[version=2.0].[type=long].field',
nullable: false,
type: SchemaFieldDataType.Number,
recursive: false,
}),
).toEqual(true);
});

it('blocks keys when looking for non-keys', () => {
expect(
filterKeyFieldPath(false, {
fieldPath: '[version=2.0].[key=True].[type=long].field',
nullable: false,
type: SchemaFieldDataType.Number,
recursive: false,
}),
).toEqual(false);
});
});
});
2 changes: 1 addition & 1 deletion datahub-web-react/src/app/entity/dataset/profile/schema/Schema.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ export default function SchemaView({
});

const hasKeySchema = useMemo(
() => (schema?.fields?.findIndex((field) => field.fieldPath.startsWith(KEY_SCHEMA_PREFIX)) || -1) !== -1,
() => (schema?.fields?.findIndex((field) => field.fieldPath.indexOf(KEY_SCHEMA_PREFIX) > -1) || -1) !== -1,
[schema],
);

Expand Down
2 changes: 1 addition & 1 deletion datahub-web-react/src/app/entity/dataset/profile/schema/utils/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ export function convertEditableSchemaMetadataForUpdate(
}

export function filterKeyFieldPath(showKeySchema: boolean, field: SchemaField) {
return field.fieldPath.startsWith(KEY_SCHEMA_PREFIX) ? showKeySchema : !showKeySchema;
return field.fieldPath.indexOf(KEY_SCHEMA_PREFIX) > -1 ? showKeySchema : !showKeySchema;
}

export function downgradeV2FieldPath(fieldPath?: string | null) {
Expand Down
19 changes: 17 additions & 2 deletions metadata-ingestion/examples/mce_files/bootstrap_mce.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@
},
"fields": [
{
"fieldPath": "field_foo_2",
"fieldPath": "[version=2.0].[type=boolean].field_foo_2",
"jsonPath": null,
"nullable": false,
"description": {
Expand All @@ -224,7 +224,7 @@
"recursive": false
},
{
"fieldPath": "field_bar",
"fieldPath": "[version=2.0].[type=boolean].field_bar",
"jsonPath": null,
"nullable": false,
"description": {
Expand All @@ -237,6 +237,21 @@
},
"nativeDataType": "boolean",
"recursive": false
},
{
"fieldPath": "[version=2.0].[key=True].[type=int].id",
"jsonPath": null,
"nullable": false,
"description": {
"string": "Id specifying which partition the message should go to"
},
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.BooleanType": {}
}
},
"nativeDataType": "boolean",
"recursive": false
}
],
"primaryKeys": null,
Expand Down