Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
3433217
Regenerate thrift headers
benibus Jun 14, 2023
0909cd1
Implement LogicalType class
benibus Jun 14, 2023
3113573
Implement column statistics
benibus Jun 14, 2023
046e967
Apply suggestion from code review
benibus Jun 15, 2023
66efa36
Add Float16 utils to Arrow
benibus Jun 16, 2023
e51d0d1
Replace float_internal.h
benibus Jun 16, 2023
a2f72ac
Minor test tweaks
benibus Jun 16, 2023
1163b4e
Add tests for Float16 operators
benibus Jun 18, 2023
bc640ff
Support multiple endians in Float16 class
benibus Jun 18, 2023
2d7e65f
Small refactor
benibus Jun 18, 2023
5e925ac
Address more review points
benibus Jun 21, 2023
87d121c
Support reading/writing `arrow::HalfFloat`
benibus Jul 9, 2023
a064bec
Fix MSVC truncation warning
benibus Jul 10, 2023
6b3d61c
Fix test input generation
benibus Jul 18, 2023
1758105
Support conversions to/from float32
benibus Aug 21, 2023
d41a0c5
Remove `Float16Base` class
benibus Aug 21, 2023
aaef4b4
Update/restructure comparison tests
benibus Aug 22, 2023
9e5cf14
Fix comment
benibus Aug 26, 2023
e124986
Minor changes to Float16 class/tests
benibus Aug 26, 2023
a12176f
Update statistics and tests
benibus Aug 31, 2023
6496aef
Update Arrow reader
benibus Aug 31, 2023
102dfb4
Remove big-endian handling in column writer
benibus Aug 31, 2023
2a45f29
Tweak Arrow/Parquet schema tests
benibus Aug 31, 2023
d340a82
Support `util::Float16` in `random_real`
benibus Aug 31, 2023
6285af1
Update Arrow reader/writer tests
benibus Aug 31, 2023
554de9d
Add To/FromDouble methods to `Float16`
benibus Aug 31, 2023
40e58f5
Add tests for `double` conversions
benibus Sep 5, 2023
7407ce4
Change misleading types
benibus Sep 5, 2023
bb4ca6a
Some `Float16` API changes
benibus Sep 5, 2023
354f6f6
Refactor typed comparators
benibus Sep 5, 2023
ea5f5dc
Add logical type to docs
benibus Sep 5, 2023
ab84630
Replace public `Float16(uint16_t)` constructor
benibus Oct 19, 2023
c8404bb
`Float16` tweaks, add constexpr tests
benibus Oct 19, 2023
157e0d7
Add test for `ColumnIndex`/`BoundaryOrder`
benibus Oct 20, 2023
5eb90d7
Tweak ToEndian methods
benibus Nov 14, 2023
cb17f56
Relocate random Float16 function
benibus Nov 14, 2023
36b8a3b
Add missing schema tests
benibus Nov 14, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Remove Float16Base class
  • Loading branch information
benibus committed Nov 13, 2023
commit d41a0c579d7b4deacb210805fb2b3fe0e2d84e7e
6 changes: 2 additions & 4 deletions cpp/src/arrow/util/float16.cc
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ uint32_t Binary16BitsToBinary32Bits(uint16_t h_bits) {

} // namespace

float Float16Base::ToFloat() const {
float Float16::ToFloat() const {
const uint32_t f_bits = Binary16BitsToBinary32Bits(value_);
return SafeCopy<float>(f_bits);
}
Expand All @@ -181,9 +181,7 @@ Float16 Float16::FromFloat(float f) {
return Float16{Binary32BitsToBinary16Bits(f_bits)};
}

std::ostream& operator<<(std::ostream& os, Float16Base arg) {
return (os << arg.ToFloat());
}
std::ostream& operator<<(std::ostream& os, Float16 arg) { return (os << arg.ToFloat()); }

} // namespace util
} // namespace arrow
100 changes: 45 additions & 55 deletions cpp/src/arrow/util/float16.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,19 +31,35 @@
namespace arrow {
namespace util {

/// \brief Base class for an IEEE half-precision float, encoded as a `uint16_t`
/// \brief Class representing an IEEE half-precision float, encoded as a `uint16_t`
///
/// The exact format is as follows (from MSB to LSB):
/// - bit 0: sign
/// - bits 1-5: exponent
/// - bits 6-15: mantissa
/// The exact format is as follows (from LSB to MSB):
/// - bits 0-10: mantissa
/// - bits 10-15: exponent
/// - bit 15: sign
///
/// NOTE: Methods in the class should not mutate the unerlying value or produce copies.
/// Such functionality is delegated to subclasses.
class ARROW_EXPORT Float16Base {
class ARROW_EXPORT Float16 {
public:
Float16Base() = default;
constexpr explicit Float16Base(uint16_t value) : value_(value) {}
Float16() = default;
constexpr explicit Float16(uint16_t value) : value_(value) {}

/// \brief Create a `Float16` from a 32-bit float (may lose precision)
static Float16 FromFloat(float f);

/// \brief Read a `Float16` from memory in native-endian byte order
static Float16 FromBytes(const uint8_t* src) {
return Float16(SafeLoadAs<uint16_t>(src));
}

/// \brief Read a `Float16` from memory in little-endian byte order
static Float16 FromLittleEndian(const uint8_t* src) {
return Float16(bit_util::FromLittleEndian(SafeLoadAs<uint16_t>(src)));
}

/// \brief Read a `Float16` from memory in big-endian byte order
static Float16 FromBigEndian(const uint8_t* src) {
return Float16(bit_util::FromBigEndian(SafeLoadAs<uint16_t>(src)));
}

/// \brief Return the value's integer representation
constexpr uint16_t bits() const { return value_; }
Expand All @@ -61,6 +77,9 @@ class ARROW_EXPORT Float16Base {
/// \brief Return true if the value is positive/negative zero
constexpr bool is_zero() const { return (value_ & 0x7fff) == 0; }

/// \brief Convert to a 32-bit float
float ToFloat() const;

/// \brief Copy the value's bytes in native-endian byte order
void ToBytes(uint8_t* dest) const { std::memcpy(dest, &value_, sizeof(value_)); }
/// \brief Return the value's bytes in native-endian byte order
Expand All @@ -74,7 +93,7 @@ class ARROW_EXPORT Float16Base {

/// \brief Copy the value's bytes in little-endian byte order
void ToLittleEndian(uint8_t* dest) const {
Float16Base{bit_util::ToLittleEndian(value_)}.ToBytes(dest);
Float16{bit_util::ToLittleEndian(value_)}.ToBytes(dest);
}
/// \brief Return the value's bytes in little-endian byte order
constexpr std::array<uint8_t, 2> ToLittleEndian() const {
Expand All @@ -87,7 +106,7 @@ class ARROW_EXPORT Float16Base {

/// \brief Copy the value's bytes in big-endian byte order
void ToBigEndian(uint8_t* dest) const {
Float16Base{bit_util::ToBigEndian(value_)}.ToBytes(dest);
Float16{bit_util::ToBigEndian(value_)}.ToBytes(dest);
}
/// \brief Return the value's bytes in big-endian byte order
constexpr std::array<uint8_t, 2> ToBigEndian() const {
Expand All @@ -98,41 +117,38 @@ class ARROW_EXPORT Float16Base {
#endif
}

float ToFloat() const;
constexpr Float16 operator-() const { return Float16(value_ ^ 0x8000); }
constexpr Float16 operator+() const { return Float16(value_); }

friend constexpr bool operator==(Float16Base lhs, Float16Base rhs) {
friend constexpr bool operator==(Float16 lhs, Float16 rhs) {
if (lhs.is_nan() || rhs.is_nan()) return false;
return Float16Base::CompareEq(lhs, rhs);
}
friend constexpr bool operator!=(Float16Base lhs, Float16Base rhs) {
return !(lhs == rhs);
return Float16::CompareEq(lhs, rhs);
}
friend constexpr bool operator!=(Float16 lhs, Float16 rhs) { return !(lhs == rhs); }

friend constexpr bool operator<(Float16Base lhs, Float16Base rhs) {
friend constexpr bool operator<(Float16 lhs, Float16 rhs) {
if (lhs.is_nan() || rhs.is_nan()) return false;
return Float16Base::CompareLt(lhs, rhs);
return Float16::CompareLt(lhs, rhs);
}
friend constexpr bool operator>(Float16Base lhs, Float16Base rhs) { return rhs < lhs; }
friend constexpr bool operator>(Float16 lhs, Float16 rhs) { return rhs < lhs; }

friend constexpr bool operator<=(Float16Base lhs, Float16Base rhs) {
friend constexpr bool operator<=(Float16 lhs, Float16 rhs) {
if (lhs.is_nan() || rhs.is_nan()) return false;
return !Float16Base::CompareLt(rhs, lhs);
}
friend constexpr bool operator>=(Float16Base lhs, Float16Base rhs) {
return rhs <= lhs;
return !Float16::CompareLt(rhs, lhs);
}
friend constexpr bool operator>=(Float16 lhs, Float16 rhs) { return rhs <= lhs; }

ARROW_FRIEND_EXPORT friend std::ostream& operator<<(std::ostream& os, Float16Base arg);
ARROW_FRIEND_EXPORT friend std::ostream& operator<<(std::ostream& os, Float16 arg);

protected:
uint16_t value_;

private:
// Comparison helpers that assume neither operand is NaN
static constexpr bool CompareEq(Float16Base lhs, Float16Base rhs) {
static constexpr bool CompareEq(Float16 lhs, Float16 rhs) {
return (lhs.bits() == rhs.bits()) || (lhs.is_zero() && rhs.is_zero());
}
static constexpr bool CompareLt(Float16Base lhs, Float16Base rhs) {
static constexpr bool CompareLt(Float16 lhs, Float16 rhs) {
if (lhs.signbit()) {
if (rhs.signbit()) {
// Both are negative
Expand All @@ -150,32 +166,6 @@ class ARROW_EXPORT Float16Base {
}
};

/// \brief Wrapper class for an IEEE half-precision float, encoded as a `uint16_t`
class ARROW_EXPORT Float16 : public Float16Base {
public:
using Float16Base::Float16Base;

constexpr Float16 operator-() const { return Float16(value_ ^ 0x8000); }
constexpr Float16 operator+() const { return Float16(value_); }

static Float16 FromFloat(float f);

/// \brief Read a `Float16` from memory in native-endian byte order
static Float16 FromBytes(const uint8_t* src) {
return Float16(SafeLoadAs<uint16_t>(src));
}

/// \brief Read a `Float16` from memory in little-endian byte order
static Float16 FromLittleEndian(const uint8_t* src) {
return Float16(bit_util::FromLittleEndian(SafeLoadAs<uint16_t>(src)));
}

/// \brief Read a `Float16` from memory in big-endian byte order
static Float16 FromBigEndian(const uint8_t* src) {
return Float16(bit_util::FromBigEndian(SafeLoadAs<uint16_t>(src)));
}
};

static_assert(std::is_trivial_v<Float16>);

} // namespace util
Expand Down
22 changes: 8 additions & 14 deletions cpp/src/parquet/statistics_test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -63,22 +63,16 @@ using schema::PrimitiveNode;

namespace test {

class BufferedFloat16 : public ::arrow::util::Float16Base {
public:
explicit BufferedFloat16(Float16 f16) : Float16Base(f16) {
buffer_ = *::arrow::AllocateBuffer(sizeof(value_));
ToLittleEndian(buffer_->mutable_data());
struct BufferedFloat16 {
explicit BufferedFloat16(Float16 f16)
: f16(f16), buffer(*::arrow::AllocateBuffer(sizeof(uint16_t))) {
this->f16.ToLittleEndian(buffer->mutable_data());
}
explicit BufferedFloat16(uint16_t value) : BufferedFloat16(Float16(value)) {}

const uint8_t* bytes() const { return buffer_->data(); }
const std::shared_ptr<::arrow::Buffer>& buffer() { return buffer_; }

BufferedFloat16 operator+() const { return *this; }
BufferedFloat16 operator-() const { return BufferedFloat16(value_ ^ 0x8000); }
explicit BufferedFloat16(uint16_t bits) : BufferedFloat16(Float16(bits)) {}
const uint8_t* bytes() const { return buffer->data(); }

private:
std::shared_ptr<::arrow::Buffer> buffer_;
Float16 f16;
std::shared_ptr<::arrow::Buffer> buffer;
};

// ----------------------------------------------------------------------
Expand Down