[release/6.0-rc2] Fix buffering bugs in TextFieldParser

src/libraries/Microsoft.VisualBasic.Core/src/Microsoft/VisualBasic/FileIO/TextFieldParser.vb

@@ -690,16 +690,14 @@ Namespace Microsoft.VisualBasic.FileIO
 
             ' No need to slide if we're already at the beginning
             If m_Position > 0 Then
-                Dim BufferLength As Integer = m_Buffer.Length
-                Dim TempArray(BufferLength - 1) As Char
-                Array.Copy(m_Buffer, m_Position, TempArray, 0, BufferLength - m_Position)
+                Dim ContentLength As Integer = m_CharsRead - m_Position
 
-                ' Fill the rest of the buffer
-                Dim CharsRead As Integer = m_Reader.Read(TempArray, BufferLength - m_Position, m_Position)
-                m_CharsRead = m_CharsRead - m_Position + CharsRead
+                Array.Copy(m_Buffer, m_Position, m_Buffer, 0, ContentLength)
 
+                ' Try to fill the rest of the buffer
+                Dim CharsRead As Integer = m_Reader.Read(m_Buffer, ContentLength, m_Buffer.Length - ContentLength)
+                m_CharsRead = ContentLength + CharsRead
                 m_Position = 0
-                m_Buffer = TempArray
 
                 Return CharsRead
             End If
@@ -717,27 +715,30 @@ Namespace Microsoft.VisualBasic.FileIO
 
             Debug.Assert(m_Buffer IsNot Nothing, "There's no buffer")
             Debug.Assert(m_Reader IsNot Nothing, "There's no StreamReader")
+            Debug.Assert(m_Position = 0, "Non-zero position")
 
             ' Set cursor
             m_PeekPosition = m_CharsRead
 
-            ' Create a larger buffer and copy our data into it
-            Dim BufferSize As Integer = m_Buffer.Length + DEFAULT_BUFFER_LENGTH
+            If m_CharsRead = m_Buffer.Length Then
+                ' Create a larger buffer and copy our data into it
+                Dim BufferSize As Integer = m_Buffer.Length + DEFAULT_BUFFER_LENGTH
+
+                ' Make sure the buffer hasn't grown too large
+                If BufferSize > m_MaxBufferSize Then
+                    Throw GetInvalidOperationException(SR.TextFieldParser_BufferExceededMaxSize)
+                End If
 
-            ' Make sure the buffer hasn't grown too large
-            If BufferSize > m_MaxBufferSize Then
-                Throw GetInvalidOperationException(SR.TextFieldParser_BufferExceededMaxSize)
+                Dim TempArray(BufferSize - 1) As Char
+                Array.Copy(m_Buffer, TempArray, m_Buffer.Length)
+                m_Buffer = TempArray
             End If
 
-            Dim TempArray(BufferSize - 1) As Char
+            Dim CharsRead As Integer = m_Reader.Read(m_Buffer, m_CharsRead, m_Buffer.Length - m_CharsRead)
+            Debug.Assert(CharsRead <= m_Buffer.Length - m_CharsRead, "We've read more chars than we have space for")
 
-            Array.Copy(m_Buffer, TempArray, m_Buffer.Length)
-            Dim CharsRead As Integer = m_Reader.Read(TempArray, m_Buffer.Length, DEFAULT_BUFFER_LENGTH)
-            m_Buffer = TempArray
             m_CharsRead += CharsRead
 
-            Debug.Assert(m_CharsRead <= BufferSize, "We've read more chars than we have space for")
-
             Return CharsRead
         End Function
 

src/libraries/Microsoft.VisualBasic.Core/tests/Microsoft/VisualBasic/FileIO/TextFieldParserTests.cs

@@ -2,7 +2,10 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
+using System.Text;
 using Xunit;
 
 namespace Microsoft.VisualBasic.FileIO.Tests
@@ -372,5 +375,73 @@ public void UnmatchedQuote_MalformedLineException()
                 Assert.Throws<MalformedLineException>(() => parser.ReadFields());
             }
         }
+
+        [Theory]
+        [InlineData(false)]
+        [InlineData(true)]
+        public void ReadFields_PartialReadsFromStream_Large(bool fieldsInQuotes) =>
+            ReadFields_PartialReadsFromStream(fieldsInQuotes, 1023);
+
+        [SkipOnTargetFramework(TargetFrameworkMonikers.NetFramework, ".NET Framework doesn't properly handle streams frequently returning much less than requested")]
+        [Theory]
+        [InlineData(false)]
+        [InlineData(true)]
+        public void ReadFields_PartialReadsFromStream_Small(bool fieldsInQuotes) =>
+            ReadFields_PartialReadsFromStream(fieldsInQuotes, 10);
+
+        private void ReadFields_PartialReadsFromStream(bool fieldsInQuotes, int maxCount)
+        {
+            // Generate some data
+            var rand = new Random(42);
+            string[][] expected = Enumerable
+                .Range(0, 10_000)
+                .Select(_ => Enumerable.Range(0, 4).Select(_ => new string('s', rand.Next(0, 10))).ToArray())
+                .ToArray();
+
+            // Write it out
+            Stream s = new DribbleStream() { MaxCount = maxCount };
+            using (var writer = new StreamWriter(s, Encoding.UTF8, 1024, leaveOpen: true))
+            {
+                foreach (string[] line in expected)
+                {
+                    string separator = "";
+                    foreach (string part in line)
+                    {
+                        writer.Write(separator);
+                        separator = ",";
+                        if (fieldsInQuotes) writer.Write('"');
+                        writer.Write(part);
+                        if (fieldsInQuotes) writer.Write('"');
+                    }
+                    writer.WriteLine();
+                }
+            }
+
+            // Read/parse it back in
+            s.Position = 0;
+            using (var parser = new TextFieldParser(s))
+            {
+                parser.TextFieldType = FieldType.Delimited;
+                parser.SetDelimiters(new[] { "," });
+                parser.HasFieldsEnclosedInQuotes = fieldsInQuotes;
+
+                int i = 0;
+                while (!parser.EndOfData)
+                {
+                    string[]? actual = parser.ReadFields();
+                    Assert.Equal(expected[i], actual);
+                    i++;
+                }
+                Assert.Equal(expected.Length, i);
+            }
+        }
+
+        private sealed class DribbleStream : MemoryStream
+        {
+            public int MaxCount { get; set; } = 1;
+
+            public override int Read(byte[] buffer, int offset, int count) =>
+                base.Read(buffer, offset, Math.Min(count, MaxCount));
+        }
     }
 }