Skip to content
Merged
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
3433217
Regenerate thrift headers
benibus Jun 14, 2023
0909cd1
Implement LogicalType class
benibus Jun 14, 2023
3113573
Implement column statistics
benibus Jun 14, 2023
046e967
Apply suggestion from code review
benibus Jun 15, 2023
66efa36
Add Float16 utils to Arrow
benibus Jun 16, 2023
e51d0d1
Replace float_internal.h
benibus Jun 16, 2023
a2f72ac
Minor test tweaks
benibus Jun 16, 2023
1163b4e
Add tests for Float16 operators
benibus Jun 18, 2023
bc640ff
Support multiple endians in Float16 class
benibus Jun 18, 2023
2d7e65f
Small refactor
benibus Jun 18, 2023
5e925ac
Address more review points
benibus Jun 21, 2023
87d121c
Support reading/writing `arrow::HalfFloat`
benibus Jul 9, 2023
a064bec
Fix MSVC truncation warning
benibus Jul 10, 2023
6b3d61c
Fix test input generation
benibus Jul 18, 2023
1758105
Support conversions to/from float32
benibus Aug 21, 2023
d41a0c5
Remove `Float16Base` class
benibus Aug 21, 2023
aaef4b4
Update/restructure comparison tests
benibus Aug 22, 2023
9e5cf14
Fix comment
benibus Aug 26, 2023
e124986
Minor changes to Float16 class/tests
benibus Aug 26, 2023
a12176f
Update statistics and tests
benibus Aug 31, 2023
6496aef
Update Arrow reader
benibus Aug 31, 2023
102dfb4
Remove big-endian handling in column writer
benibus Aug 31, 2023
2a45f29
Tweak Arrow/Parquet schema tests
benibus Aug 31, 2023
d340a82
Support `util::Float16` in `random_real`
benibus Aug 31, 2023
6285af1
Update Arrow reader/writer tests
benibus Aug 31, 2023
554de9d
Add To/FromDouble methods to `Float16`
benibus Aug 31, 2023
40e58f5
Add tests for `double` conversions
benibus Sep 5, 2023
7407ce4
Change misleading types
benibus Sep 5, 2023
bb4ca6a
Some `Float16` API changes
benibus Sep 5, 2023
354f6f6
Refactor typed comparators
benibus Sep 5, 2023
ea5f5dc
Add logical type to docs
benibus Sep 5, 2023
ab84630
Replace public `Float16(uint16_t)` constructor
benibus Oct 19, 2023
c8404bb
`Float16` tweaks, add constexpr tests
benibus Oct 19, 2023
157e0d7
Add test for `ColumnIndex`/`BoundaryOrder`
benibus Oct 20, 2023
5eb90d7
Tweak ToEndian methods
benibus Nov 14, 2023
cb17f56
Relocate random Float16 function
benibus Nov 14, 2023
36b8a3b
Add missing schema tests
benibus Nov 14, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Refactor typed comparators
  • Loading branch information
benibus committed Nov 13, 2023
commit 354f6f6e6eafa930579f296769b7e834978690de
39 changes: 24 additions & 15 deletions cpp/src/parquet/statistics.cc
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,8 @@ template <bool is_signed>
struct CompareHelper<FLBAType, is_signed>
: public BinaryLikeCompareHelperBase<FLBAType, is_signed> {};

struct Float16CompareHelper {
template <>
struct CompareHelper<Float16LogicalType, /*is_signed=*/true> {
using T = FLBA;

static T DefaultMin() { return T{Float16Constants::max()}; }
Expand Down Expand Up @@ -412,12 +413,24 @@ optional<std::pair<ByteArray, ByteArray>> CleanStatistic(
return min_max;
}

template <bool is_signed, typename DType,
typename HelperType = CompareHelper<DType, is_signed>>
class TypedComparatorImpl : virtual public TypedComparator<DType> {
template <typename T>
struct RebindLogical {
using DType = T;
using c_type = typename DType::c_type;
};

template <>
struct RebindLogical<Float16LogicalType> {
using DType = FLBAType;
using c_type = DType::c_type;
};

template <bool is_signed, typename DType>
class TypedComparatorImpl
: virtual public TypedComparator<typename RebindLogical<DType>::DType> {
public:
using T = typename DType::c_type;
using Helper = HelperType;
using T = typename RebindLogical<DType>::c_type;
using Helper = CompareHelper<DType, is_signed>;

explicit TypedComparatorImpl(int type_length = -1) : type_length_(type_length) {}

Expand Down Expand Up @@ -464,7 +477,9 @@ class TypedComparatorImpl : virtual public TypedComparator<DType> {
return {min, max};
}

std::pair<T, T> GetMinMax(const ::arrow::Array& values) override;
std::pair<T, T> GetMinMax(const ::arrow::Array& values) override {
ParquetException::NYI(values.type()->ToString());
}

private:
int type_length_;
Expand Down Expand Up @@ -492,12 +507,6 @@ TypedComparatorImpl</*is_signed=*/false, Int32Type>::GetMinMax(const int32_t* va
return {SafeCopy<int32_t>(min), SafeCopy<int32_t>(max)};
}

template <bool is_signed, typename DType, typename Helper>
std::pair<typename DType::c_type, typename DType::c_type>
TypedComparatorImpl<is_signed, DType, Helper>::GetMinMax(const ::arrow::Array& values) {
ParquetException::NYI(values.type()->ToString());
}

template <bool is_signed>
std::pair<ByteArray, ByteArray> GetMinMaxBinaryHelper(
const TypedComparatorImpl<is_signed, ByteArrayType>& comparator,
Expand Down Expand Up @@ -926,8 +935,8 @@ std::shared_ptr<Comparator> DoMakeComparator(Type::type physical_type,
return std::make_shared<TypedComparatorImpl<true, ByteArrayType>>();
case Type::FIXED_LEN_BYTE_ARRAY:
if (logical_type == LogicalType::Type::FLOAT16) {
return std::make_shared<
TypedComparatorImpl<true, FLBAType, Float16CompareHelper>>(type_length);
return std::make_shared<TypedComparatorImpl<true, Float16LogicalType>>(
type_length);
}
return std::make_shared<TypedComparatorImpl<true, FLBAType>>(type_length);
default:
Expand Down