Allow whitespace for decoding

.azure/pipelines/ci-fuzz.yml

@@ -31,3 +31,17 @@ stages:
       - template: jobs/fuzzing-libFuzzer.yml
         parameters:
           method: Base64_Url_Decode
+
+  - stage: Base64_Whitespace
+    dependsOn: [] # break sequential dependency
+    jobs:
+      - template: jobs/fuzzing-libFuzzer-no-matrix.yml
+        parameters:
+          method: Base64_Default_Decode_Whitespace
+
+  - stage: Base64Url_Whitespace
+    dependsOn: [] # break sequential dependency
+    jobs:
+      - template: jobs/fuzzing-libFuzzer-no-matrix.yml
+        parameters:
+          method: Base64_Url_Decode_Whitespace

.azure/pipelines/jobs/fuzzing-libFuzzer-no-matrix.yml

@@ -0,0 +1,12 @@
+parameters:
+  method: ''
+
+jobs:
+  - job: ${{ parameters.method }}
+    pool:
+      vmImage: 'ubuntu-20.04'
+    timeoutInMinutes: 0     # 360 minutes
+    steps:
+      - template: steps/fuzz.yml
+        parameters:
+          method: ${{ parameters.method }}

.azure/pipelines/jobs/fuzzing-libFuzzer.yml

@@ -16,66 +16,6 @@ jobs:
         SSE_disabled:
           DOTNET_EnableSSE: 0
     steps:
-      # ~SDKs already installed~
-      - template: steps/dotnet-install.yml
-      # Needed for sharpfuzz
-      - task: UseDotNet@2
-        displayName: 'Use .NET SDK 6.0'
-        inputs:
-          version: 6.x
-          includePreviewVersions: true
-          installationPath: $(Agent.ToolsDirectory)/dotnet
-
-      - bash: |
-          cd fuzz
-
-          # -L to follow the redirect
-          curl -Lo libfuzzer-dotnet https://github.com/gfoidl/libfuzzer-dotnet/releases/download/v1.1.0-preview-1/libfuzzer-dotnet
-
-          chmod +x *.sh
-          chmod +x libfuzzer-dotnet
-        displayName: pre-requisites
-
-      - bash: |
-          # Install SharpFuzz.CommandLine global .NET tool
-          dotnet tool install --global SharpFuzz.CommandLine
-        displayName: setup
-
-      - bash: |
-          # Workaround
-          # https://github.com/Microsoft/azure-pipelines-tasks/issues/8291#issuecomment-441707116
-          export PATH="$PATH:$HOME/.dotnet/tools"
-
-          cd fuzz
-          ./init.sh
-        displayName: init
-
-      - bash: |
-          echo "------------------------------------------------"
-          echo "writing random data (encoded) to testcases/4.dat"
-          echo "------------------------------------------------"
-
-          cd fuzz
-          head /dev/urandom | base64 | tee testcases/4.dat
-        displayName: 'more entropy for fuzz'
-
-      - bash: |
-          if [[ "$DOTNET_ENABLEAVX" == "0" ]]; then
-              export DOTNET_EnableAVX=0
-          fi
-
-          if [[ "$DOTNET_ENABLESSE" == "0" ]]; then
-              export DOTNET_EnableSSE=0
-          fi
-
-          cd fuzz
-
-          # TIMEOUT variable defined in pipeline-UI (e.g. 350m = 21000s)
-          ./run.sh libFuzzer $(TIMEOUT) ${{ parameters.method }}
-        displayName: run
-
-      - task: PublishBuildArtifacts@1
-        condition: failed()
-        inputs:
-          pathtoPublish: 'fuzz/findings/crashes'
-          artifactName: '${{ parameters.method }}'
+      - template: steps/fuzz.yml
+        parameters:
+          method: ${{ parameters.method }}

.azure/pipelines/jobs/steps/fuzz.yml

@@ -0,0 +1,67 @@
+parameters:
+  method: ''
+
+steps:
+  # ~SDKs already installed~
+  - template: ./dotnet-install.yml
+  # Needed for sharpfuzz
+  - task: UseDotNet@2
+    displayName: 'Use .NET SDK 6.0'
+    inputs:
+      version: 6.x
+      includePreviewVersions: true
+      installationPath: $(Agent.ToolsDirectory)/dotnet
+
+  - bash: |
+      cd fuzz
+
+      # -L to follow the redirect
+      curl -Lo libfuzzer-dotnet https://github.com/gfoidl/libfuzzer-dotnet/releases/download/v1.1.0-preview-1/libfuzzer-dotnet
+
+      chmod +x *.sh
+      chmod +x libfuzzer-dotnet
+    displayName: pre-requisites
+
+  - bash: |
+      # Install SharpFuzz.CommandLine global .NET tool
+      dotnet tool install --global SharpFuzz.CommandLine
+    displayName: setup
+
+  - bash: |
+      # Workaround
+      # https://github.com/Microsoft/azure-pipelines-tasks/issues/8291#issuecomment-441707116
+      export PATH="$PATH:$HOME/.dotnet/tools"
+
+      cd fuzz
+      ./init.sh
+    displayName: init
+
+  - bash: |
+      echo "------------------------------------------------"
+      echo "writing random data (encoded) to testcases/4.dat"
+      echo "------------------------------------------------"
+
+      cd fuzz
+      head /dev/urandom | base64 | tee testcases/4.dat
+    displayName: 'more entropy for fuzz'
+
+  - bash: |
+      if [[ "$DOTNET_ENABLEAVX" == "0" ]]; then
+          export DOTNET_EnableAVX=0
+      fi
+
+      if [[ "$DOTNET_ENABLESSE" == "0" ]]; then
+          export DOTNET_EnableSSE=0
+      fi
+
+      cd fuzz
+
+      # TIMEOUT variable defined in pipeline-UI (e.g. 350m = 21000s)
+      ./run.sh libFuzzer $(TIMEOUT) ${{ parameters.method }}
+    displayName: run
+
+  - task: PublishBuildArtifacts@1
+    condition: failed()
+    inputs:
+      pathtoPublish: 'fuzz/findings/crashes'
+      artifactName: '${{ parameters.method }}'

fuzz/gfoidl.Base64.FuzzTests/AflFuzzImpl.cs

@@ -6,9 +6,14 @@ namespace gfoidl.Base64.FuzzTests;
 
 internal sealed class AflFuzzImpl : FuzzImpl
 {
-    public override void Base64_Default_Decode() => Fuzzer.Run(Base64_Default_Decode);
-    public override void Base64_Url_Decode    () => Fuzzer.Run(Base64_Url_Decode);
+    public override void Base64_Default_Decode(WhitespaceMode whitespaceMode)
+        => Fuzzer.Run(base64Text => Base64_Default_Decode(base64Text, whitespaceMode));
+    public override void Base64_Url_Decode(WhitespaceMode whitespaceMode)
+        => Fuzzer.Run(base64Text => Base64_Url_Decode(base64Text, whitespaceMode));
     //-------------------------------------------------------------------------
-    private static void Base64_Default_Decode(string base64Text) => Base64_Decode(base64Text, Base64.Default);
-    private static void Base64_Url_Decode    (string base64Text) => Base64_Decode(base64Text, Base64.Url);
+    private static void Base64_Default_Decode(string base64Text, WhitespaceMode whitespaceMode)
+        => Base64_Decode(base64Text, Base64.Default, whitespaceMode);
+
+    private static void Base64_Url_Decode(string base64Text, WhitespaceMode whitespaceMode)
+        => Base64_Decode(base64Text, Base64.Url, whitespaceMode);
 }

fuzz/gfoidl.Base64.FuzzTests/FuzzImpl.cs

@@ -8,11 +8,11 @@ namespace gfoidl.Base64.FuzzTests;
 
 internal abstract class FuzzImpl
 {
-    public abstract void Base64_Default_Decode();
-    public abstract void Base64_Url_Decode    ();
+    public abstract void Base64_Default_Decode(WhitespaceMode whitespaceMode);
+    public abstract void Base64_Url_Decode    (WhitespaceMode whitespaceMode);
     //-------------------------------------------------------------------------
     [SkipLocalsInit]
-    protected static void Base64_Decode(ReadOnlySpan<char> base64Text, Base64 encoder)
+    protected static void Base64_Decode(ReadOnlySpan<char> base64Text, Base64 encoder, WhitespaceMode whitespaceMode)
     {
         byte[]? dataArrayFromPool = null;
         try
@@ -22,16 +22,22 @@ protected static void Base64_Decode(ReadOnlySpan<char> base64Text, Base64 encode
                 ? stackalloc byte[256]
                 : dataArrayFromPool = ArrayPool<byte>.Shared.Rent(maxDecodedLength);
 
-            OperationStatus status = encoder.Decode(base64Text, data, out int consumed, out int written);
+            OperationStatus status = encoder.Decode(base64Text, data, out int consumed, out int written, whitespaceMode);
 
-            if (ContainsInvalidData(base64Text, encoder) && status != OperationStatus.InvalidData)
+            int idxOfInvalidData = IndexOfInvalidData(base64Text, encoder, whitespaceMode);
+            if (idxOfInvalidData >= 0 && status != OperationStatus.InvalidData)
             {
-                throw new Exception("contains invalid data -- not detected");
+                throw new Exception($"""
+                    contains invalid data -- not detected
+                    status: {status}
+                    idx:    {idxOfInvalidData}
+                    value:  {base64Text[idxOfInvalidData]}
+                    """);
             }
 
             if (status == OperationStatus.Done && consumed != base64Text.Length)
             {
-                throw new Exception("consumed != encoded.Length");
+                throw new Exception($"consumed ({consumed}) != encoded.Length ({base64Text.Length})");
             }
         }
         catch (ArgumentOutOfRangeException)
@@ -51,34 +57,80 @@ protected static void Base64_Decode(ReadOnlySpan<char> base64Text, Base64 encode
         }
     }
     //-------------------------------------------------------------------------
-    private static bool ContainsInvalidData(ReadOnlySpan<char> base64Text, Base64 encoder)
+    private static int IndexOfInvalidData(ReadOnlySpan<char> base64Text, Base64 encoder, WhitespaceMode whitespaceMode)
     {
-        ReadOnlySpan<sbyte> decodingMap = default;
+        ReadOnlySpan<sbyte> decodingMap;
 
         if (encoder is Base64Impl<Base64Encoding>)
         {
             decodingMap = Base64Encoding.DecodingMap;
 
-            // Check for padding at the end
-            int paddingCount = 0;
-
-            if (base64Text.Length > 1 && base64Text[^1] == Constants.EncodingPadChar) paddingCount++;
-            if (base64Text.Length > 2 && base64Text[^2] == Constants.EncodingPadChar) paddingCount++;
-
-            base64Text = base64Text[0..^paddingCount];
+            // Trim padding at the end
+            int paddingCountInclFinalWhitespace = GetPaddingCountInclFinalWhitespace(base64Text, whitespaceMode);
+            base64Text                          = base64Text[0..^paddingCountInclFinalWhitespace];
         }
         else if (encoder is Base64Impl<Base64UrlEncoding>)
         {
             decodingMap = Base64UrlEncoding.DecodingMap;
         }
+        else
+        {
+            throw new NotSupportedException();
+        }
 
         for (int i = 0; i < base64Text.Length; ++i)
         {
             char ch = base64Text[i];
 
-            if (ch > 0x7F || decodingMap[ch] == -1) return true;
+            if (whitespaceMode == WhitespaceMode.Allow && IsAllowedWhitespace(ch))
+            {
+                continue;
+            }
+
+            if (ch > 0x7F || decodingMap[ch] == -1) return i;
         }
 
-        return false;
+        return -1;
+    }
+    //-------------------------------------------------------------------------
+    private static int GetPaddingCountInclFinalWhitespace(ReadOnlySpan<char> base64Text, WhitespaceMode whitespaceMode)
+    {
+        // Check for padding at the end
+        int paddingCount = 0;
+
+        if (whitespaceMode == WhitespaceMode.None)
+        {
+            if (base64Text.Length > 1 && base64Text[^1] == Constants.EncodingPadChar) paddingCount++;
+            if (base64Text.Length > 2 && base64Text[^2] == Constants.EncodingPadChar) paddingCount++;
+        }
+        else if (whitespaceMode == WhitespaceMode.Allow)
+        {
+            for (int i = base64Text.Length - 1; i >= 0; --i)
+            {
+                char ch = base64Text[i];
+
+                if (IsAllowedWhitespace(ch) || ch == Constants.EncodingPadChar)
+                {
+                    paddingCount++;
+                }
+                else
+                {
+                    break;
+                }
+            }
+        }
+        else
+        {
+            throw new NotSupportedException($"Unknown mode: {whitespaceMode}");
+        }
+
+        return paddingCount;
+    }
+    //-------------------------------------------------------------------------
+    private static readonly char[] s_allowedWhitespace = { '\t', '\n', '\r', ' ' };
+
+    private static bool IsAllowedWhitespace(char ch)
+    {
+        return s_allowedWhitespace.AsSpan().IndexOf(ch) >= 0;
     }
 }

fuzz/gfoidl.Base64.FuzzTests/LibFuzzerFuzzImpl.cs

@@ -8,13 +8,19 @@ namespace gfoidl.Base64.FuzzTests;
 
 internal sealed class LibFuzzerFuzzImpl : FuzzImpl
 {
-    public override void Base64_Default_Decode() => Fuzzer.LibFuzzer.Run(Base64_Default_Decode);
-    public override void Base64_Url_Decode()     => Fuzzer.LibFuzzer.Run(Base64_Url_Decode);
+    public override void Base64_Default_Decode(WhitespaceMode whitespaceMode)
+        => Fuzzer.LibFuzzer.Run(base64 => Base64_Default_Decode(base64, whitespaceMode));
+
+    public override void Base64_Url_Decode(WhitespaceMode whitespaceMode)
+        => Fuzzer.LibFuzzer.Run(base64 => Base64_Url_Decode(base64, whitespaceMode));
     //-------------------------------------------------------------------------
-    private static void Base64_Default_Decode(ReadOnlySpan<byte> base64) => Base64_DecodeCore(base64, Base64.Default);
-    private static void Base64_Url_Decode    (ReadOnlySpan<byte> base64) => Base64_DecodeCore(base64, Base64.Url);
+    private static void Base64_Default_Decode(ReadOnlySpan<byte> base64, WhitespaceMode whitespaceMode)
+        => Base64_DecodeCore(base64, Base64.Default, whitespaceMode);
+
+    private static void Base64_Url_Decode(ReadOnlySpan<byte> base64, WhitespaceMode whitespaceMode)
+        => Base64_DecodeCore(base64, Base64.Url, whitespaceMode);
     //-------------------------------------------------------------------------
-    private static void Base64_DecodeCore(ReadOnlySpan<byte> base64, Base64 encoder)
+    private static void Base64_DecodeCore(ReadOnlySpan<byte> base64, Base64 encoder, WhitespaceMode whitespaceMode)
     {
         char[]? arrayFromPool = null;
 
@@ -26,9 +32,9 @@ private static void Base64_DecodeCore(ReadOnlySpan<byte> base64, Base64 encoder)
                 : arrayFromPool = ArrayPool<char>.Shared.Rent(maxChars);
 
             int written = Encoding.UTF8.GetChars(base64, chars);
-            chars = chars[..written];
+            chars       = chars[..written];
 
-            Base64_Decode(chars, encoder);
+            Base64_Decode(chars, encoder, whitespaceMode);
         }
         finally
         {

fuzz/gfoidl.Base64.FuzzTests/Program.cs

@@ -18,7 +18,9 @@
         As first arg the fuzzing method (sut) must be given.
         List of supported suts:
             - Base64_Default_Decode
+            - Base64_Default_Decode_Whitespace
             - Base64_Url_Decode
+            - Base64_Url_Decode_Whitespace
 
         Or 'PrintMachineInfo' to display useful info about the machine, like
         enabled vectorization, etc.
@@ -38,8 +40,10 @@ As first arg the fuzzing method (sut) must be given.
 
 switch (args[0])
 {
-    case "Base64_Default_Decode": fuzzImpl.Base64_Default_Decode(); break;
-    case "Base64_Url_Decode"    : fuzzImpl.Base64_Url_Decode()    ; break;
+    case "Base64_Default_Decode"           : fuzzImpl.Base64_Default_Decode(WhitespaceMode.None) ; break;
+    case "Base64_Url_Decode_Whitespace"    : fuzzImpl.Base64_Url_Decode    (WhitespaceMode.Allow); break;
+    case "Base64_Url_Decode"               : fuzzImpl.Base64_Url_Decode    (WhitespaceMode.None) ; break;
+    case "Base64_Default_Decode_Whitespace": fuzzImpl.Base64_Default_Decode(WhitespaceMode.Allow); break;
     default:
         Console.WriteLine($"Unknown fuzzing function: {args[0]}");
         Environment.Exit(2);

fuzz/run.sh

@@ -9,11 +9,13 @@ path="gfoidl.Base64.FuzzTests/bin/Release/$tfm"
 "$path"/gfoidl.Base64.FuzzTests "PrintMachineInfo"
 
 if [[ $# -lt 3 ]]; then
-    echo "first  arg must be the fuzz type, either 'afl' or 'libfuzzer'"
+    echo "first  arg must be the fuzz type, either 'afl' or 'libFuzzer'"
     echo "second arg must be duration for timeout (in seconds)"
     echo "third  arg must be the fuzz-function"
     echo "  - Base64_Default_Decode"
+    echo "  - Base64_Default_Decode_Whitespace"
     echo "  - Base64_Url_Decode"
+    echo "  - Base64_Url_Decode_Whitespace"
     exit 1
 fi