Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
3433217
Regenerate thrift headers
benibus Jun 14, 2023
0909cd1
Implement LogicalType class
benibus Jun 14, 2023
3113573
Implement column statistics
benibus Jun 14, 2023
046e967
Apply suggestion from code review
benibus Jun 15, 2023
66efa36
Add Float16 utils to Arrow
benibus Jun 16, 2023
e51d0d1
Replace float_internal.h
benibus Jun 16, 2023
a2f72ac
Minor test tweaks
benibus Jun 16, 2023
1163b4e
Add tests for Float16 operators
benibus Jun 18, 2023
bc640ff
Support multiple endians in Float16 class
benibus Jun 18, 2023
2d7e65f
Small refactor
benibus Jun 18, 2023
5e925ac
Address more review points
benibus Jun 21, 2023
87d121c
Support reading/writing `arrow::HalfFloat`
benibus Jul 9, 2023
a064bec
Fix MSVC truncation warning
benibus Jul 10, 2023
6b3d61c
Fix test input generation
benibus Jul 18, 2023
1758105
Support conversions to/from float32
benibus Aug 21, 2023
d41a0c5
Remove `Float16Base` class
benibus Aug 21, 2023
aaef4b4
Update/restructure comparison tests
benibus Aug 22, 2023
9e5cf14
Fix comment
benibus Aug 26, 2023
e124986
Minor changes to Float16 class/tests
benibus Aug 26, 2023
a12176f
Update statistics and tests
benibus Aug 31, 2023
6496aef
Update Arrow reader
benibus Aug 31, 2023
102dfb4
Remove big-endian handling in column writer
benibus Aug 31, 2023
2a45f29
Tweak Arrow/Parquet schema tests
benibus Aug 31, 2023
d340a82
Support `util::Float16` in `random_real`
benibus Aug 31, 2023
6285af1
Update Arrow reader/writer tests
benibus Aug 31, 2023
554de9d
Add To/FromDouble methods to `Float16`
benibus Aug 31, 2023
40e58f5
Add tests for `double` conversions
benibus Sep 5, 2023
7407ce4
Change misleading types
benibus Sep 5, 2023
bb4ca6a
Some `Float16` API changes
benibus Sep 5, 2023
354f6f6
Refactor typed comparators
benibus Sep 5, 2023
ea5f5dc
Add logical type to docs
benibus Sep 5, 2023
ab84630
Replace public `Float16(uint16_t)` constructor
benibus Oct 19, 2023
c8404bb
`Float16` tweaks, add constexpr tests
benibus Oct 19, 2023
157e0d7
Add test for `ColumnIndex`/`BoundaryOrder`
benibus Oct 20, 2023
5eb90d7
Tweak ToEndian methods
benibus Nov 14, 2023
cb17f56
Relocate random Float16 function
benibus Nov 14, 2023
36b8a3b
Add missing schema tests
benibus Nov 14, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Relocate random Float16 function
  • Loading branch information
benibus committed Nov 14, 2023
commit cb17f5675e6dd225afb7c036c2738ad655aa3773
23 changes: 7 additions & 16 deletions cpp/src/parquet/arrow/test_util.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
#include "arrow/util/decimal.h"
#include "arrow/util/float16.h"
#include "parquet/column_reader.h"
#include "parquet/test_util.h"

namespace parquet {

Expand Down Expand Up @@ -66,26 +67,15 @@ struct Decimal256WithPrecisionAndScale {
static constexpr int32_t scale = PRECISION - 1;
};

inline void RandomHalfFloatValues(int64_t n, uint32_t seed,
::arrow::util::Float16 min_value,
::arrow::util::Float16 max_value,
std::vector<uint16_t>* out) {
std::vector<float> values;
::arrow::random_real(n, seed, static_cast<float>(min_value),
static_cast<float>(max_value), &values);
out->resize(values.size());
std::transform(values.begin(), values.end(), out->begin(),
[](float f) { return ::arrow::util::Float16(f).bits(); });
}

template <class ArrowType>
::arrow::enable_if_floating_point<ArrowType, Status> NonNullArray(
size_t size, std::shared_ptr<Array>* out) {
using c_type = typename ArrowType::c_type;
std::vector<c_type> values;
if constexpr (::arrow::is_half_float_type<ArrowType>::value) {
RandomHalfFloatValues(size, 0, ::arrow::util::Float16(0.0f),
::arrow::util::Float16(1.0f), &values);
values.resize(size);
test::random_float16_numbers(static_cast<int>(size), 0, ::arrow::util::Float16(0.0f),
::arrow::util::Float16(1.0f), values.data());
} else {
::arrow::random_real(size, 0, static_cast<c_type>(0), static_cast<c_type>(1),
&values);
Expand Down Expand Up @@ -221,8 +211,9 @@ ::arrow::enable_if_floating_point<ArrowType, Status> NullableArray(
using c_type = typename ArrowType::c_type;
std::vector<c_type> values;
if constexpr (::arrow::is_half_float_type<ArrowType>::value) {
RandomHalfFloatValues(size, seed, ::arrow::util::Float16(-1e4f),
::arrow::util::Float16(1e4f), &values);
values.resize(size);
test::random_float16_numbers(static_cast<int>(size), 0, ::arrow::util::Float16(-1e4f),
::arrow::util::Float16(1e4f), values.data());
} else {
::arrow::random_real(size, seed, static_cast<c_type>(-1e10),
static_cast<c_type>(1e10), &values);
Expand Down
10 changes: 10 additions & 0 deletions cpp/src/parquet/test_util.cc
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,16 @@ void random_Int96_numbers(int n, uint32_t seed, int32_t min_value, int32_t max_v
}
}

void random_float16_numbers(int n, uint32_t seed, ::arrow::util::Float16 min_value,
::arrow::util::Float16 max_value, uint16_t* out) {
std::vector<float> values(n);
random_numbers(n, seed, static_cast<float>(min_value), static_cast<float>(max_value),
values.data());
for (int i = 0; i < n; ++i) {
out[i] = ::arrow::util::Float16(values[i]).bits();
}
}

void random_fixed_byte_array(int n, uint32_t seed, uint8_t* buf, int len, FLBA* out) {
std::default_random_engine gen(seed);
std::uniform_int_distribution<int> d(0, 255);
Expand Down
4 changes: 4 additions & 0 deletions cpp/src/parquet/test_util.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@

#include "arrow/io/memory.h"
#include "arrow/testing/util.h"
#include "arrow/util/float16.h"

#include "parquet/column_page.h"
#include "parquet/column_reader.h"
Expand Down Expand Up @@ -148,6 +149,9 @@ inline void random_numbers(int n, uint32_t seed, double min_value, double max_va
void random_Int96_numbers(int n, uint32_t seed, int32_t min_value, int32_t max_value,
Int96* out);

void random_float16_numbers(int n, uint32_t seed, ::arrow::util::Float16 min_value,
::arrow::util::Float16 max_value, uint16_t* out);

void random_fixed_byte_array(int n, uint32_t seed, uint8_t* buf, int len, FLBA* out);

void random_byte_array(int n, uint32_t seed, uint8_t* buf, ByteArray* out, int min_size,
Expand Down