Skip to content

[SPARK-48699][SQL] Refine collation API #47014

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
uros-db wants to merge 9 commits into from
Closed

[SPARK-48699][SQL] Refine collation API #47014

Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
f0f6e01
Initial commit
uros-db Jun 18, 2024
9ef928a
Fix tests
uros-db Jun 19, 2024
8ba2106
Fast path for toLowerCase/toUpperCase
uros-db Jun 20, 2024
3329450
Update comparison
uros-db Jun 20, 2024
ecb0a65
Min/Max code point check
uros-db Jun 20, 2024
24e7e4c
CollationSpecUTF8
uros-db Jun 21, 2024
a905890
Merge branch 'apache:master' into fix-collation-api
uros-db Jun 25, 2024
dcedb9a
Fixes
uros-db Jun 25, 2024
b56c032
Fix comments
uros-db Jun 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
CollationSpecUTF8
  • Loading branch information
@uros-db
uros-db committed Jun 21, 2024
commit 24e7e4c0944c8d89bb90094f49d4112fb54ba31c
26 changes: 13 additions & 13 deletions common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,7 @@ private static Collation fetchCollation(int collationId) {
== DefinitionOrigin.PREDEFINED);
if (collationId == UTF8_BINARY_COLLATION_ID) {
// Skip cache.
return CollationSpecUTF8Binary.UTF8_BINARY_COLLATION;
return CollationSpecUTF8.UTF8_BINARY_COLLATION;
} else if (collationMap.containsKey(collationId)) {
// Already in cache.
return collationMap.get(collationId);
Expand All @@ -308,7 +308,7 @@ private static Collation fetchCollation(int collationId) {
CollationSpec spec;
ImplementationProvider implementationProvider = getImplementationProvider(collationId);
if (implementationProvider == ImplementationProvider.UTF8_BINARY) {
spec = CollationSpecUTF8Binary.fromCollationId(collationId);
spec = CollationSpecUTF8.fromCollationId(collationId);
} else {
spec = CollationSpecICU.fromCollationId(collationId);
}
Expand All @@ -327,7 +327,7 @@ private static int collationNameToId(String collationName) throws SparkException
// Collation names provided by user are treated as case-insensitive.
String collationNameUpper = collationName.toUpperCase();
if (collationNameUpper.startsWith("UTF8_")) {
return CollationSpecUTF8Binary.collationNameToId(collationName, collationNameUpper);
return CollationSpecUTF8.collationNameToId(collationName, collationNameUpper);
} else {
return CollationSpecICU.collationNameToId(collationName, collationNameUpper);
}
Expand All @@ -336,7 +336,7 @@ private static int collationNameToId(String collationName) throws SparkException
protected abstract Collation buildCollation();
}

private static class CollationSpecUTF8Binary extends CollationSpec {
private static class CollationSpecUTF8 extends CollationSpec {

/**
* Bit 0 in collation ID having value 0 for plain UTF8_BINARY and 1 for UTF8_LCASE
Expand All @@ -357,17 +357,17 @@ private enum CaseSensitivity {
private static final int CASE_SENSITIVITY_MASK = 0b1;

private static final int UTF8_BINARY_COLLATION_ID =
new CollationSpecUTF8Binary(CaseSensitivity.UNSPECIFIED).collationId;
new CollationSpecUTF8(CaseSensitivity.UNSPECIFIED).collationId;
private static final int UTF8_LCASE_COLLATION_ID =
new CollationSpecUTF8Binary(CaseSensitivity.LCASE).collationId;
new CollationSpecUTF8(CaseSensitivity.LCASE).collationId;
protected static Collation UTF8_BINARY_COLLATION =
new CollationSpecUTF8Binary(CaseSensitivity.UNSPECIFIED).buildCollation();
new CollationSpecUTF8(CaseSensitivity.UNSPECIFIED).buildCollation();
protected static Collation UTF8_LCASE_COLLATION =
new CollationSpecUTF8Binary(CaseSensitivity.LCASE).buildCollation();
new CollationSpecUTF8(CaseSensitivity.LCASE).buildCollation();

private final int collationId;

private CollationSpecUTF8Binary(CaseSensitivity caseSensitivity) {
private CollationSpecUTF8(CaseSensitivity caseSensitivity) {
this.collationId =
SpecifierUtils.setSpecValue(0, CASE_SENSITIVITY_OFFSET, caseSensitivity);
}
Expand All @@ -384,14 +384,14 @@ private static int collationNameToId(String originalName, String collationName)
}
}

private static CollationSpecUTF8Binary fromCollationId(int collationId) {
private static CollationSpecUTF8 fromCollationId(int collationId) {
// Extract case sensitivity from collation ID.
int caseConversionOrdinal = SpecifierUtils.getSpecValue(collationId,
CASE_SENSITIVITY_OFFSET, CASE_SENSITIVITY_MASK);
// Verify only case sensitivity bits were set settable in UTF8_BINARY family of collations.
assert (SpecifierUtils.removeSpec(collationId,
CASE_SENSITIVITY_OFFSET, CASE_SENSITIVITY_MASK) == 0);
return new CollationSpecUTF8Binary(CaseSensitivity.values()[caseConversionOrdinal]);
return new CollationSpecUTF8(CaseSensitivity.values()[caseConversionOrdinal]);
}

@Override
Expand Down Expand Up @@ -727,9 +727,9 @@ public CollationIdentifier identifier() {
public static final List<String> SUPPORTED_PROVIDERS = List.of(PROVIDER_SPARK, PROVIDER_ICU);

public static final int UTF8_BINARY_COLLATION_ID =
Collation.CollationSpecUTF8Binary.UTF8_BINARY_COLLATION_ID;
Collation.CollationSpecUTF8.UTF8_BINARY_COLLATION_ID;
public static final int UTF8_LCASE_COLLATION_ID =
Collation.CollationSpecUTF8Binary.UTF8_LCASE_COLLATION_ID;
Collation.CollationSpecUTF8.UTF8_LCASE_COLLATION_ID;
public static final int UNICODE_COLLATION_ID =
Collation.CollationSpecICU.UNICODE_COLLATION_ID;
public static final int UNICODE_CI_COLLATION_ID =
Expand Down