From f80e0d9801bac6376d5a99464d02e5206b6ec5be Mon Sep 17 00:00:00 2001 From: Veratil Date: Sat, 11 Jun 2022 11:07:47 -0500 Subject: [PATCH 1/3] Add non-Qt base64 encode/decode methods and tests --- include/base64.h | 57 ++++++++++++++- src/core/base64.cpp | 131 ++++++++++++++++++++++++++++++++++ tests/CMakeLists.txt | 1 + tests/src/core/base64Test.cpp | 55 ++++++++++++++ 4 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 tests/src/core/base64Test.cpp diff --git a/include/base64.h b/include/base64.h index f52d90d136e..c85dc75b45e 100644 --- a/include/base64.h +++ b/include/base64.h @@ -30,6 +30,10 @@ #include #include +#include +#include +#include + namespace lmms::base64 { @@ -53,4 +57,55 @@ namespace lmms::base64 } // namespace lmms::base64 -#endif +namespace lmms::base64 { + constexpr std::array map = + { + 'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z', + 'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z', + '0','1','2','3','4','5','6','7','8','9', + '-','_' + }; + const std::map reverse_map { + {'A', 0}, {'B', 1}, {'C', 2}, {'D', 3}, {'E', 4}, + {'F', 5}, {'G', 6}, {'H', 7}, {'I', 8}, {'J', 9}, + {'K', 10}, {'L', 11}, {'M', 12}, {'N', 13}, {'O', 14}, + {'P', 15}, {'Q', 16}, {'R', 17}, {'S', 18}, {'T', 19}, + {'U', 20}, {'V', 21}, {'W', 22}, {'X', 23}, {'Y', 24}, + {'Z', 25}, {'a', 26}, {'b', 27}, {'c', 28}, {'d', 28}, + {'e', 30}, {'f', 31}, {'g', 32}, {'h', 33}, {'i', 34}, + {'j', 35}, {'k', 36}, {'l', 37}, {'m', 38}, {'n', 39}, + {'o', 40}, {'p', 41}, {'q', 42}, {'r', 43}, {'s', 44}, + {'t', 45}, {'u', 46}, {'v', 47}, {'w', 48}, {'x', 49}, + {'y', 50}, {'z', 51}, {'0', 52}, {'1', 53}, {'2', 54}, + {'3', 55}, {'4', 56}, {'5', 57}, {'6', 58}, {'7', 59}, + {'8', 60}, {'9', 61}, {'-', 62}, {'_', 63} //, {'=', 64} + }; + constexpr char pad = '='; + + /* + This section of math ensures that base64 encode/decode will work + as intended. Some rare architectures don't use 8-bit char's, and + it's possible this won't work as intended if a char isn't 8-bits. + + In the rare case this is ported to an architecture where this + happens, feel free to comment out the static_assert's and test. + */ + constexpr int char_bits = std::numeric_limits::digits; + constexpr int sign_bit = std::numeric_limits::is_signed ? 1 : 0; + // check that the string_view character type is 8 (7 signed + 1 sign) bits wide + static_assert(char_bits + sign_bit == 8); + constexpr int numBitsPerChar = char_bits + sign_bit; + constexpr int numBitsPerBase64Char = 6; + constexpr int lcm = std::lcm(numBitsPerChar, numBitsPerBase64Char); + // make sure math works, 24 bits + static_assert(lcm == 24); + constexpr int numBytesPerChunk = lcm / numBitsPerChar; + constexpr int numBase64CharPerChunk = lcm / numBitsPerBase64Char; + // double check math works and bit width matches + static_assert(numBytesPerChunk * numBitsPerChar == numBase64CharPerChunk * numBitsPerBase64Char); + + std::string encode(std::string_view data); + std::string decode(std::string_view data); +} // namespace lmms::base64 + +#endif // _BASE64_H diff --git a/src/core/base64.cpp b/src/core/base64.cpp index b1c26b610ba..36c41311c1b 100644 --- a/src/core/base64.cpp +++ b/src/core/base64.cpp @@ -54,4 +54,135 @@ QVariant decode( const QString & _b64, QVariant::Type _force_type ) } +//TODO C++20: It *may* be possible to make the following functions constepxr given C++20's constexpr std::string. + +/** + * @brief Base64 encodes data. + * @param data + * @return std::string containing the Base64 encoded data. + */ +std::string encode(std::string_view data) +{ + if (data.empty()) { return ""; } + + // base64 encoded string + std::string result; + + // number of chunks to process + padding + auto [numChunks, numTrailingBytes] = std::div(data.length(), numBytesPerChunk); + + // add 1 chunk to handle last padded chunk + if (numTrailingBytes) { ++numChunks; } + + // allocate one time to prevent in-loop reallocations + result.reserve(numChunks * numBase64CharPerChunk); + + /* + char1 char2 char3 + 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 + 5 4 3 2 1 0 | 5 4 3 2 1 0 | 5 4 3 2 1 0 | 5 4 3 2 1 0 + b64c1 b64c2 b64c3 b64c4 + */ + // Get the first base64 character offset from the char chunk + auto b64char1 = [](std::string_view chunk) { + return static_cast(chunk[0]) >> 2; + }; + // Get the second base64 character offset from the char chunk + auto b64char2 = [](std::string_view chunk) { + return ( + ((static_cast(chunk[0]) & 0x03) << 4) + | + ((static_cast(chunk.size() > 0 ? chunk[1] : '\0') & 0xF0) >> 4) + ); + }; + // Get the third base64 character offset from the char chunk + auto b64char3 = [](std::string_view chunk) { + return ( + ((static_cast(chunk.size() > 0 ? chunk[1] : '\0') & 0x0F) << 2) + | + ((static_cast(chunk.size() > 1 ? chunk[2] : '\0') & 0xC0) >> 6) + ); + }; + // Get the fourth base64 character offset from the char chunk + auto b64char4 = [](std::string_view chunk) { + return static_cast(chunk.size() > 1 ? chunk[2] : '\0') & 0x3F; + }; + for (int currentChunk = 0; currentChunk < numChunks; ++currentChunk) + { + std::string_view chunk = data.substr(currentChunk * numBytesPerChunk, numBytesPerChunk); + std::string output{pad, pad, pad, pad}; + output[0] = map[b64char1(chunk)]; + output[1] = map[b64char2(chunk)]; + switch (chunk.length()) { + case 3: + output[3] = map[b64char4(chunk)]; + case 2: + output[2] = map[b64char3(chunk)]; + default: /* no-op */; + }; + result += output; + } + return result; +} + +/** + * @brief Decodes data in Base64. + * @param data + * @return std::string containing the original data. + */ +std::string decode(std::string_view data) +{ + if (data.empty()) { return ""; } + if (data.length() % numBase64CharPerChunk != 0) { + throw std::length_error("base64::decode : data length not a multiple of 4"); + } + + std::string result; + + // number of chunks to process + auto numChunks = std::div(data.length(), numBase64CharPerChunk); + + // allocate one time to prevent in-loop reallocations + result.reserve(numChunks.quot * numBytesPerChunk); + /* + char1 char2 char3 + 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 + 5 4 3 2 1 0 | 5 4 3 2 1 0 | 5 4 3 2 1 0 | 5 4 3 2 1 0 + b64c1 b64c2 b64c3 b64c4 + */ + // Get the first character from the base64 chunk + auto char1 = [](std::string_view chunk) { + return static_cast( + (static_cast(reverse_map.at(chunk[0])) << 2) + | + (static_cast(reverse_map.at(chunk[1])) >> 4) + ); + }; + // Get the second character from the base64 chunk + auto char2 = [](std::string_view chunk) { + return chunk[2] == pad ? '\0' : static_cast( + (static_cast(reverse_map.at(chunk[1]) & 0x0F) << 4) + | + (static_cast(reverse_map.at(chunk[2])) >> 2) + ); + }; + // Get the third character from the base64 chunk + auto char3 = [](std::string_view chunk) { + return chunk[3] == pad ? '\0' : static_cast( + (static_cast(reverse_map.at(chunk[2]) & 0x03) << 6) + | + static_cast(reverse_map.at(chunk[3])) + ); + }; + for (int currentChunk = 0; currentChunk < numChunks.quot; ++currentChunk) { + std::string_view chunk = data.substr(currentChunk * numBase64CharPerChunk, numBase64CharPerChunk); + std::string output{0, 0, 0}; + output[0] = char1(chunk); + output[1] = char2(chunk); + output[2] = char3(chunk); + result += output; + } + return result; +} + } // namespace lmms::base64 diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6ff9c41e967..d614619d8a9 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -20,6 +20,7 @@ ADD_EXECUTABLE(tests $ src/core/AutomatableModelTest.cpp + src/core/base64Test.cpp src/core/ProjectVersionTest.cpp src/core/RelativePathsTest.cpp diff --git a/tests/src/core/base64Test.cpp b/tests/src/core/base64Test.cpp new file mode 100644 index 00000000000..e55ee97c6f0 --- /dev/null +++ b/tests/src/core/base64Test.cpp @@ -0,0 +1,55 @@ +/* + * base64Test.cpp + * + * Copyright (c) 2022 Kevin Zander + * + * This file is part of LMMS - https://lmms.io + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program (see COPYING); if not, write to the + * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301 USA. + * + */ + +#include "QTestSuite.h" + +#include "base64.h" + +class Base64Test : QTestSuite +{ + Q_OBJECT +private slots: + void Base64Tests() + { + using namespace lmms::base64; + + // Test Vectors from RFC 4648 Section 10 + std::vector> test_vectors{ + {"", ""}, + {"f", "Zg=="}, + {"fo", "Zm8="}, + {"foo", "Zm9v"}, + {"foob", "Zm9vYg=="}, + {"fooba", "Zm9vYmE="}, + {"foobar", "Zm9vYmFy"}, + }; + for (auto vector : test_vectors) + { + QCOMPARE(QString(encode(vector.first).c_str()), QString(vector.second.c_str())); + QCOMPARE(QString(vector.first.c_str()), QString(decode(vector.second).c_str())); + } + } +} Base64Tests; + +#include "base64Test.moc" From 29ec4bfbc8f2209d18a2550ce918ba16692f793a Mon Sep 17 00:00:00 2001 From: Veratil Date: Wed, 6 Jul 2022 11:34:43 -0500 Subject: [PATCH 2/3] Small updates --- include/base64.h | 2 +- src/core/base64.cpp | 55 ++++++++++++++++++++++----------------------- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/include/base64.h b/include/base64.h index c85dc75b45e..84f120adb9f 100644 --- a/include/base64.h +++ b/include/base64.h @@ -65,7 +65,7 @@ namespace lmms::base64 { '0','1','2','3','4','5','6','7','8','9', '-','_' }; - const std::map reverse_map { + const std::map rmap { {'A', 0}, {'B', 1}, {'C', 2}, {'D', 3}, {'E', 4}, {'F', 5}, {'G', 6}, {'H', 7}, {'I', 8}, {'J', 9}, {'K', 10}, {'L', 11}, {'M', 12}, {'N', 13}, {'O', 14}, diff --git a/src/core/base64.cpp b/src/core/base64.cpp index 36c41311c1b..8f08a0682d4 100644 --- a/src/core/base64.cpp +++ b/src/core/base64.cpp @@ -85,34 +85,37 @@ std::string encode(std::string_view data) */ // Get the first base64 character offset from the char chunk auto b64char1 = [](std::string_view chunk) { - return static_cast(chunk[0]) >> 2; + return static_cast(chunk[0] >> 2); }; // Get the second base64 character offset from the char chunk auto b64char2 = [](std::string_view chunk) { - return ( - ((static_cast(chunk[0]) & 0x03) << 4) + return static_cast( + ((chunk[0] & 0x03) << 4) | - ((static_cast(chunk.size() > 0 ? chunk[1] : '\0') & 0xF0) >> 4) + (chunk.size() > 0 ? ((chunk[1] & 0xF0) >> 4) : 0) ); }; // Get the third base64 character offset from the char chunk auto b64char3 = [](std::string_view chunk) { - return ( - ((static_cast(chunk.size() > 0 ? chunk[1] : '\0') & 0x0F) << 2) + return static_cast( + (chunk.size() > 0 ? ((chunk[1] & 0x0F) << 2) : 0) | - ((static_cast(chunk.size() > 1 ? chunk[2] : '\0') & 0xC0) >> 6) + (chunk.size() > 1 ? ((chunk[2] & 0xC0) >> 6) : 0) ); }; // Get the fourth base64 character offset from the char chunk auto b64char4 = [](std::string_view chunk) { - return static_cast(chunk.size() > 1 ? chunk[2] : '\0') & 0x3F; + return static_cast(chunk.size() > 1 ? (chunk[2] & 0x3F) : 0); }; for (int currentChunk = 0; currentChunk < numChunks; ++currentChunk) { std::string_view chunk = data.substr(currentChunk * numBytesPerChunk, numBytesPerChunk); - std::string output{pad, pad, pad, pad}; - output[0] = map[b64char1(chunk)]; - output[1] = map[b64char2(chunk)]; + std::string output{ + map[b64char1(chunk)], + map[b64char2(chunk)], + pad, + pad + }; switch (chunk.length()) { case 3: output[3] = map[b64char4(chunk)]; @@ -152,35 +155,31 @@ std::string decode(std::string_view data) */ // Get the first character from the base64 chunk auto char1 = [](std::string_view chunk) { - return static_cast( - (static_cast(reverse_map.at(chunk[0])) << 2) - | - (static_cast(reverse_map.at(chunk[1])) >> 4) + return static_cast( + (rmap.at(chunk[0]) << 2) | (rmap.at(chunk[1]) >> 4) ); }; // Get the second character from the base64 chunk auto char2 = [](std::string_view chunk) { - return chunk[2] == pad ? '\0' : static_cast( - (static_cast(reverse_map.at(chunk[1]) & 0x0F) << 4) - | - (static_cast(reverse_map.at(chunk[2])) >> 2) + return static_cast( + chunk[2] == pad + ? 0 + : ((rmap.at(chunk[1]) & 0x0F) << 4) | (rmap.at(chunk[2]) >> 2) ); }; // Get the third character from the base64 chunk auto char3 = [](std::string_view chunk) { - return chunk[3] == pad ? '\0' : static_cast( - (static_cast(reverse_map.at(chunk[2]) & 0x03) << 6) - | - static_cast(reverse_map.at(chunk[3])) + return static_cast( + chunk[3] == pad + ? 0 + : ((rmap.at(chunk[2]) & 0x03) << 6) | rmap.at(chunk[3]) ); }; for (int currentChunk = 0; currentChunk < numChunks.quot; ++currentChunk) { std::string_view chunk = data.substr(currentChunk * numBase64CharPerChunk, numBase64CharPerChunk); - std::string output{0, 0, 0}; - output[0] = char1(chunk); - output[1] = char2(chunk); - output[2] = char3(chunk); - result += output; + result.push_back(char1(chunk)); + result.push_back(char2(chunk)); + result.push_back(char3(chunk)); } return result; } From 7844205b4392af227616b20a0662d3985a888554 Mon Sep 17 00:00:00 2001 From: Veratil Date: Fri, 13 Jan 2023 22:54:13 -0600 Subject: [PATCH 3/3] Address review comments --- include/base64.h | 12 +++++---- src/core/base64.cpp | 12 +++++---- tests/src/core/base64Test.cpp | 50 ++++++++++++++++++++++++----------- 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/include/base64.h b/include/base64.h index 84f120adb9f..580dc7dda40 100644 --- a/include/base64.h +++ b/include/base64.h @@ -30,9 +30,11 @@ #include #include +#include +#include +#include #include #include -#include namespace lmms::base64 { @@ -58,14 +60,14 @@ namespace lmms::base64 } // namespace lmms::base64 namespace lmms::base64 { - constexpr std::array map = + constexpr inline std::array map = { 'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z', 'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z', '0','1','2','3','4','5','6','7','8','9', '-','_' }; - const std::map rmap { + const inline std::map rmap { {'A', 0}, {'B', 1}, {'C', 2}, {'D', 3}, {'E', 4}, {'F', 5}, {'G', 6}, {'H', 7}, {'I', 8}, {'J', 9}, {'K', 10}, {'L', 11}, {'M', 12}, {'N', 13}, {'O', 14}, @@ -104,8 +106,8 @@ namespace lmms::base64 { // double check math works and bit width matches static_assert(numBytesPerChunk * numBitsPerChar == numBase64CharPerChunk * numBitsPerBase64Char); - std::string encode(std::string_view data); - std::string decode(std::string_view data); + auto encode(std::string_view data) -> std::string; + auto decode(std::string_view data) -> std::string; } // namespace lmms::base64 #endif // _BASE64_H diff --git a/src/core/base64.cpp b/src/core/base64.cpp index 8f08a0682d4..b2f3de1249b 100644 --- a/src/core/base64.cpp +++ b/src/core/base64.cpp @@ -61,7 +61,7 @@ QVariant decode( const QString & _b64, QVariant::Type _force_type ) * @param data * @return std::string containing the Base64 encoded data. */ -std::string encode(std::string_view data) +auto encode(std::string_view data) -> std::string { if (data.empty()) { return ""; } @@ -69,7 +69,9 @@ std::string encode(std::string_view data) std::string result; // number of chunks to process + padding - auto [numChunks, numTrailingBytes] = std::div(data.length(), numBytesPerChunk); + auto div_result = std::div(data.length(), numBytesPerChunk); + auto numChunks = div_result.quot; + auto numTrailingBytes = div_result.rem; // add 1 chunk to handle last padded chunk if (numTrailingBytes) { ++numChunks; } @@ -110,7 +112,7 @@ std::string encode(std::string_view data) for (int currentChunk = 0; currentChunk < numChunks; ++currentChunk) { std::string_view chunk = data.substr(currentChunk * numBytesPerChunk, numBytesPerChunk); - std::string output{ + std::array output{ map[b64char1(chunk)], map[b64char2(chunk)], pad, @@ -123,7 +125,7 @@ std::string encode(std::string_view data) output[2] = map[b64char3(chunk)]; default: /* no-op */; }; - result += output; + result.append(output.data(), output.size()); } return result; } @@ -133,7 +135,7 @@ std::string encode(std::string_view data) * @param data * @return std::string containing the original data. */ -std::string decode(std::string_view data) +auto decode(std::string_view data) -> std::string { if (data.empty()) { return ""; } if (data.length() % numBase64CharPerChunk != 0) { diff --git a/tests/src/core/base64Test.cpp b/tests/src/core/base64Test.cpp index e55ee97c6f0..fe4c81e5675 100644 --- a/tests/src/core/base64Test.cpp +++ b/tests/src/core/base64Test.cpp @@ -30,25 +30,43 @@ class Base64Test : QTestSuite { Q_OBJECT private slots: - void Base64Tests() + void create_test_data() { - using namespace lmms::base64; + QTest::addColumn("original"); + QTest::addColumn("encoded"); // Test Vectors from RFC 4648 Section 10 - std::vector> test_vectors{ - {"", ""}, - {"f", "Zg=="}, - {"fo", "Zm8="}, - {"foo", "Zm9v"}, - {"foob", "Zm9vYg=="}, - {"fooba", "Zm9vYmE="}, - {"foobar", "Zm9vYmFy"}, - }; - for (auto vector : test_vectors) - { - QCOMPARE(QString(encode(vector.first).c_str()), QString(vector.second.c_str())); - QCOMPARE(QString(vector.first.c_str()), QString(decode(vector.second).c_str())); - } + QTest::newRow("empty string") << "" << ""; + QTest::newRow("1 chunk 2 pad") << "f" << "Zg=="; + QTest::newRow("1 chunk 1 pad") << "fo" << "Zm8="; + QTest::newRow("1 chunk 0 pad") << "foo" << "Zm9v"; + QTest::newRow("2 chunk 2 pad") << "foob" << "Zm9vYg=="; + QTest::newRow("2 chunk 1 pad") << "fooba" << "Zm9vYmE="; + QTest::newRow("2 chunk 0 pad") << "foobar" << "Zm9vYmFy"; + } + void b64_encode_data() + { + create_test_data(); + } + void b64_encode() + { + using namespace lmms::base64; + + QFETCH(QString, original); + QFETCH(QString, encoded); + QCOMPARE(QString(encode(original.toStdString()).c_str()), encoded); + } + void b64_decode_data() + { + create_test_data(); + } + void b64_decode() + { + using namespace lmms::base64; + + QFETCH(QString, original); + QFETCH(QString, encoded); + QCOMPARE(original, QString(decode(encoded.toStdString()).c_str())); } } Base64Tests;