|
26 | 26 | #include <fstream> |
27 | 27 | #include <iostream> |
28 | 28 | #include <limits> |
| 29 | +#include <memory> |
29 | 30 | #include <sstream> |
30 | 31 | #include <string> |
31 | 32 |
|
| 33 | +#ifdef MPS_PARSER_WITH_BZIP2 |
| 34 | +#include <bzlib.h> |
| 35 | +#endif // MPS_PARSER_WITH_BZIP2 |
| 36 | + |
| 37 | +#ifdef MPS_PARSER_WITH_ZLIB |
| 38 | +#include <zlib.h> |
| 39 | +#endif // MPS_PARSER_WITH_ZLIB |
| 40 | + |
| 41 | +#if defined(MPS_PARSER_WITH_BZIP2) || defined(MPS_PARSER_WITH_ZLIB) |
| 42 | +#include <dlfcn.h> |
| 43 | +#endif // MPS_PARSER_WITH_BZIP2 || MPS_PARSER_WITH_ZLIB |
| 44 | + |
| 45 | +namespace { |
| 46 | +using cuopt::mps_parser::error_type_t; |
| 47 | +using cuopt::mps_parser::mps_parser_expects; |
| 48 | +using cuopt::mps_parser::mps_parser_expects_fatal; |
| 49 | + |
| 50 | +struct FcloseDeleter { |
| 51 | + void operator()(FILE* fp) |
| 52 | + { |
| 53 | + mps_parser_expects_fatal( |
| 54 | + fclose(fp) == 0, error_type_t::ValidationError, "Error closing MPS file!"); |
| 55 | + } |
| 56 | +}; |
| 57 | +} // end namespace |
| 58 | + |
| 59 | +#ifdef MPS_PARSER_WITH_BZIP2 |
| 60 | +namespace { |
| 61 | +using BZ2_bzReadOpen_t = decltype(&BZ2_bzReadOpen); |
| 62 | +using BZ2_bzReadClose_t = decltype(&BZ2_bzReadClose); |
| 63 | +using BZ2_bzRead_t = decltype(&BZ2_bzRead); |
| 64 | + |
| 65 | +std::vector<char> bz2_file_to_string(const std::string& file) |
| 66 | +{ |
| 67 | + struct DlCloseDeleter { |
| 68 | + void operator()(void* fp) |
| 69 | + { |
| 70 | + mps_parser_expects_fatal( |
| 71 | + dlclose(fp) == 0, error_type_t::ValidationError, "Error closing libbz2.so!"); |
| 72 | + } |
| 73 | + }; |
| 74 | + struct BzReadCloseDeleter { |
| 75 | + void operator()(void* f) |
| 76 | + { |
| 77 | + int bzerror; |
| 78 | + if (f != nullptr) fptr(&bzerror, f); |
| 79 | + mps_parser_expects_fatal( |
| 80 | + bzerror == BZ_OK, error_type_t::ValidationError, "Error closing bzip2 file!"); |
| 81 | + } |
| 82 | + BZ2_bzReadClose_t fptr = nullptr; |
| 83 | + }; |
| 84 | + |
| 85 | + std::unique_ptr<void, DlCloseDeleter> lbz2handle{dlopen("libbz2.so", RTLD_LAZY)}; |
| 86 | + mps_parser_expects( |
| 87 | + lbz2handle != nullptr, |
| 88 | + error_type_t::ValidationError, |
| 89 | + "Could not open .mps.bz2 file since libbz2.so was not found. In order to open .mps.bz2 files " |
| 90 | + "directly, please ensure libbzip2 is installed. Alternatively, decompress the .mps.bz2 file " |
| 91 | + "manually and open the uncompressed .mps file. Given path: %s", |
| 92 | + file.c_str()); |
| 93 | + |
| 94 | + BZ2_bzReadOpen_t BZ2_bzReadOpen = |
| 95 | + reinterpret_cast<BZ2_bzReadOpen_t>(dlsym(lbz2handle.get(), "BZ2_bzReadOpen")); |
| 96 | + BZ2_bzReadClose_t BZ2_bzReadClose = |
| 97 | + reinterpret_cast<BZ2_bzReadClose_t>(dlsym(lbz2handle.get(), "BZ2_bzReadClose")); |
| 98 | + BZ2_bzRead_t BZ2_bzRead = reinterpret_cast<BZ2_bzRead_t>(dlsym(lbz2handle.get(), "BZ2_bzRead")); |
| 99 | + mps_parser_expects( |
| 100 | + BZ2_bzReadOpen != nullptr && BZ2_bzReadClose != nullptr && BZ2_bzRead != nullptr, |
| 101 | + error_type_t::ValidationError, |
| 102 | + "Error loading libbzip2! Library version might be incompatible. Please decompress the .mps.bz2 " |
| 103 | + "file manually and open the uncompressed .mps file. Given path: %s", |
| 104 | + file.c_str()); |
| 105 | + |
| 106 | + std::unique_ptr<FILE, FcloseDeleter> fp{fopen(file.c_str(), "rb")}; |
| 107 | + mps_parser_expects(fp != nullptr, |
| 108 | + error_type_t::ValidationError, |
| 109 | + "Error opening MPS file! Given path: %s", |
| 110 | + file.c_str()); |
| 111 | + int bzerror = BZ_OK; |
| 112 | + std::unique_ptr<void, BzReadCloseDeleter> bzfile{ |
| 113 | + BZ2_bzReadOpen(&bzerror, fp.get(), 0, 0, nullptr, 0), {BZ2_bzReadClose}}; |
| 114 | + mps_parser_expects(bzerror == BZ_OK, |
| 115 | + error_type_t::ValidationError, |
| 116 | + "Could not open bzip2 compressed file! Given path: %s", |
| 117 | + file.c_str()); |
| 118 | + |
| 119 | + std::vector<char> buf; |
| 120 | + const size_t readbufsize = 1ull << 24; // 16MiB - just a guess. |
| 121 | + std::vector<char> readbuf(readbufsize); |
| 122 | + while (bzerror == BZ_OK) { |
| 123 | + const size_t bytes_read = BZ2_bzRead(&bzerror, bzfile.get(), readbuf.data(), readbuf.size()); |
| 124 | + if (bzerror == BZ_OK || bzerror == BZ_STREAM_END) { |
| 125 | + buf.insert(buf.end(), begin(readbuf), begin(readbuf) + bytes_read); |
| 126 | + } |
| 127 | + } |
| 128 | + buf.push_back('\0'); |
| 129 | + mps_parser_expects(bzerror == BZ_STREAM_END, |
| 130 | + error_type_t::ValidationError, |
| 131 | + "Error in bzip2 decompression of MPS file! Given path: %s", |
| 132 | + file.c_str()); |
| 133 | + return buf; |
| 134 | +} |
| 135 | +} // end namespace |
| 136 | +#endif // MPS_PARSER_WITH_BZIP2 |
| 137 | + |
| 138 | +#ifdef MPS_PARSER_WITH_ZLIB |
| 139 | +namespace { |
| 140 | +using gzopen_t = decltype(&gzopen); |
| 141 | +using gzclose_r_t = decltype(&gzclose_r); |
| 142 | +using gzbuffer_t = decltype(&gzbuffer); |
| 143 | +using gzread_t = decltype(&gzread); |
| 144 | +using gzerror_t = decltype(&gzerror); |
| 145 | +std::vector<char> zlib_file_to_string(const std::string& file) |
| 146 | +{ |
| 147 | + struct DlCloseDeleter { |
| 148 | + void operator()(void* fp) |
| 149 | + { |
| 150 | + mps_parser_expects_fatal( |
| 151 | + dlclose(fp) == 0, error_type_t::ValidationError, "Error closing libbz2.so!"); |
| 152 | + } |
| 153 | + }; |
| 154 | + struct GzCloseDeleter { |
| 155 | + void operator()(gzFile_s* f) |
| 156 | + { |
| 157 | + int err = fptr(f); |
| 158 | + mps_parser_expects_fatal( |
| 159 | + err == Z_OK, error_type_t::ValidationError, "Error closing gz file!"); |
| 160 | + } |
| 161 | + gzclose_r_t fptr = nullptr; |
| 162 | + }; |
| 163 | + |
| 164 | + std::unique_ptr<void, DlCloseDeleter> lzhandle{dlopen("libz.so.1", RTLD_LAZY)}; |
| 165 | + mps_parser_expects( |
| 166 | + lzhandle != nullptr, |
| 167 | + error_type_t::ValidationError, |
| 168 | + "Could not open .mps.gz file since libz.so was not found. In order to open .mps.gz files " |
| 169 | + "directly, please ensure zlib is installed. Alternatively, decompress the .mps.gz file " |
| 170 | + "manually and open the uncompressed .mps file. Given path: %s", |
| 171 | + file.c_str()); |
| 172 | + gzopen_t gzopen = reinterpret_cast<gzopen_t>(dlsym(lzhandle.get(), "gzopen")); |
| 173 | + gzclose_r_t gzclose_r = reinterpret_cast<gzclose_r_t>(dlsym(lzhandle.get(), "gzclose_r")); |
| 174 | + gzbuffer_t gzbuffer = reinterpret_cast<gzbuffer_t>(dlsym(lzhandle.get(), "gzbuffer")); |
| 175 | + gzread_t gzread = reinterpret_cast<gzread_t>(dlsym(lzhandle.get(), "gzread")); |
| 176 | + gzerror_t gzerror = reinterpret_cast<gzerror_t>(dlsym(lzhandle.get(), "gzerror")); |
| 177 | + mps_parser_expects( |
| 178 | + gzopen != nullptr && gzclose_r != nullptr && gzbuffer != nullptr && gzread != nullptr && |
| 179 | + gzerror != nullptr, |
| 180 | + error_type_t::ValidationError, |
| 181 | + "Error loading zlib! Library version might be incompatible. Please decompress the .mps.gz file " |
| 182 | + "manually and open the uncompressed .mps file. Given path: %s", |
| 183 | + file.c_str()); |
| 184 | + std::unique_ptr<gzFile_s, GzCloseDeleter> gzfp{gzopen(file.c_str(), "rb"), {gzclose_r}}; |
| 185 | + mps_parser_expects(gzfp != nullptr, |
| 186 | + error_type_t::ValidationError, |
| 187 | + "Error opening compressed MPS file! Given path: %s", |
| 188 | + file.c_str()); |
| 189 | + int zlib_status = gzbuffer(gzfp.get(), 1 << 20); // 1 MiB |
| 190 | + mps_parser_expects(zlib_status == Z_OK, |
| 191 | + error_type_t::ValidationError, |
| 192 | + "Could not set zlib internal buffer size for decompression! Given path: %s", |
| 193 | + file.c_str()); |
| 194 | + std::vector<char> buf; |
| 195 | + const size_t readbufsize = 1ull << 24; // 16MiB |
| 196 | + std::vector<char> readbuf(readbufsize); |
| 197 | + int bytes_read = -1; |
| 198 | + while (bytes_read != 0) { |
| 199 | + bytes_read = gzread(gzfp.get(), readbuf.data(), readbuf.size()); |
| 200 | + if (bytes_read > 0) { buf.insert(buf.end(), begin(readbuf), begin(readbuf) + bytes_read); } |
| 201 | + if (bytes_read < 0) { |
| 202 | + gzerror(gzfp.get(), &zlib_status); |
| 203 | + break; |
| 204 | + } |
| 205 | + } |
| 206 | + buf.push_back('\0'); |
| 207 | + mps_parser_expects(zlib_status == Z_OK, |
| 208 | + error_type_t::ValidationError, |
| 209 | + "Error in zlib decompression of MPS file! Given path: %s", |
| 210 | + file.c_str()); |
| 211 | + return buf; |
| 212 | +} |
| 213 | +} // end namespace |
| 214 | +#endif // MPS_PARSER_WITH_ZLIB |
| 215 | + |
32 | 216 | namespace cuopt::mps_parser { |
33 | 217 |
|
34 | 218 | template <typename i_t> |
@@ -342,6 +526,18 @@ std::vector<char> mps_parser_t<i_t, f_t>::file_to_string(const std::string& file |
342 | 526 | { |
343 | 527 | // raft::common::nvtx::range fun_scope("file to string"); |
344 | 528 |
|
| 529 | +#ifdef MPS_PARSER_WITH_BZIP2 |
| 530 | + if (file.size() > 4 && file.substr(file.size() - 4, 4) == ".bz2") { |
| 531 | + return bz2_file_to_string(file); |
| 532 | + } |
| 533 | +#endif // MPS_PARSER_WITH_BZIP2 |
| 534 | + |
| 535 | +#ifdef MPS_PARSER_WITH_ZLIB |
| 536 | + if (file.size() > 3 && file.substr(file.size() - 3, 3) == ".gz") { |
| 537 | + return zlib_file_to_string(file); |
| 538 | + } |
| 539 | +#endif // MPS_PARSER_WITH_ZLIB |
| 540 | + |
345 | 541 | // Faster than using C++ I/O |
346 | 542 | FILE* fp = fopen(file.c_str(), "r"); |
347 | 543 | mps_parser_expects(fp != nullptr, |
|
0 commit comments