diff --git a/src/FastSerialization/SegmentedDictionary/SegmentedDictionary.cs b/src/FastSerialization/SegmentedDictionary/SegmentedDictionary.cs index 4e2991fcf..c1254f543 100644 --- a/src/FastSerialization/SegmentedDictionary/SegmentedDictionary.cs +++ b/src/FastSerialization/SegmentedDictionary/SegmentedDictionary.cs @@ -781,7 +781,8 @@ public int EnsureCapacity(int capacity) throw new ArgumentOutOfRangeException(nameof(capacity)); } - var currentCapacity = _entries.Capacity; + // Normal usage of a dictionary should never ask for a capacity that exceeds int32.MaxValue. + var currentCapacity = (int)_entries.Capacity; if (currentCapacity >= capacity) { return currentCapacity; diff --git a/src/FastSerialization/SegmentedList.cs b/src/FastSerialization/SegmentedList.cs index 9fb1594c2..4326f3766 100644 --- a/src/FastSerialization/SegmentedList.cs +++ b/src/FastSerialization/SegmentedList.cs @@ -100,7 +100,7 @@ public long Count } } - public int Capacity => this.capacity; + internal long Capacity => this.capacity; /// /// Copy to Array @@ -177,7 +177,7 @@ public T this[long index] } } - public ref T GetElementByReference(int index) => + internal ref T GetElementByReference(int index) => ref this.items[index >> this.segmentShift][index & this.offsetMask]; /// @@ -190,6 +190,18 @@ public bool IsValidIndex(long index) return this.items[index >> this.segmentShift] != null; } + /// + /// Get slot of an element + /// + /// + /// + /// + public T[] GetSlot(int index, out int slot) + { + slot = index & this.offsetMask; + return this.items[index >> this.segmentShift]; + } + /// /// Adds new element at the end of the list. /// @@ -443,12 +455,12 @@ public void CopyTo(T[] array, int arrayIndex) "Destination array is not long enough to copy all the items in the collection. Check array index and length."); } - int remain = (int)this.count; + long remain = this.count; - for (int i = 0; (remain > 0) && (i < this.items.Length); i++) + for (long i = 0; (remain > 0) && (i < this.items.Length); i++) { // We can safely cast to int, since that is the max value that items[i].Length can have. - int len = Math.Min(remain, this.items[i].Length); + int len = (int)Math.Min(remain, this.items[i].Length); Array.Copy(this.items[i], 0, array, arrayIndex, len); diff --git a/src/FastSerialization/SegmentedMemoryStreamReader.cs b/src/FastSerialization/SegmentedMemoryStreamReader.cs index 5239d2aeb..1cb94d244 100644 --- a/src/FastSerialization/SegmentedMemoryStreamReader.cs +++ b/src/FastSerialization/SegmentedMemoryStreamReader.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Text; namespace FastSerialization @@ -13,12 +12,31 @@ public class SegmentedMemoryStreamReader /// /// Create a IStreamReader (reads binary data) from a given byte buffer /// - public SegmentedMemoryStreamReader(SegmentedList data) : this(data, 0, (int)data.Count) { } + public SegmentedMemoryStreamReader(SegmentedList data, SerializationConfiguration config = null) : this(data, 0, data.Count, config) { } /// /// Create a IStreamReader (reads binary data) from a given subregion of a byte buffer /// - public SegmentedMemoryStreamReader(SegmentedList data, int start, int length) + public SegmentedMemoryStreamReader(SegmentedList data, long start, long length, SerializationConfiguration config = null) { + SerializationConfiguration = config ?? new SerializationConfiguration(); + + if (SerializationConfiguration.StreamLabelWidth == StreamLabelWidth.FourBytes) + { + readLabel = () => + { + return (StreamLabel)(uint)ReadInt32(); + }; + sizeOfSerializedStreamLabel = 4; + } + else + { + readLabel = () => + { + return (StreamLabel)(ulong)ReadInt64(); + }; + sizeOfSerializedStreamLabel = 8; + } + bytes = new SegmentedList(65_536, length); bytes.AppendFrom(data, start, length); position = start; @@ -130,18 +148,24 @@ public string ReadString() /// /// Implementation of IStreamReader /// - public StreamLabel ReadLabel() - { - return (StreamLabel)(uint)ReadInt32(); - } + public StreamLabel ReadLabel() => readLabel(); + /// /// Implementation of IStreamReader /// public virtual void Goto(StreamLabel label) { Debug.Assert(label != StreamLabel.Invalid); - Debug.Assert((long)label <= int.MaxValue); - position = (int)label; + + if (SerializationConfiguration.StreamLabelWidth == StreamLabelWidth.FourBytes) + { + Debug.Assert((long)label <= int.MaxValue); + position = (uint)label; + } + else + { + position = (long)label; + } } /// /// Implementation of IStreamReader @@ -150,7 +174,14 @@ public virtual StreamLabel Current { get { - return (StreamLabel)(uint)position; + if (SerializationConfiguration.StreamLabelWidth == StreamLabelWidth.FourBytes) + { + return (StreamLabel)(uint)position; + } + else + { + return (StreamLabel)position; + } } } /// @@ -158,8 +189,7 @@ public virtual StreamLabel Current /// public virtual void GotoSuffixLabel() { - const int serializedStreamLabelSize = 4; - Goto((StreamLabel)(Length - serializedStreamLabelSize)); + Goto((StreamLabel)(Length - sizeOfSerializedStreamLabel)); Goto(ReadLabel()); } /// @@ -174,6 +204,11 @@ public void Dispose() /// Dispose pattern /// protected virtual void Dispose(bool disposing) { } + + /// + /// Returns the SerializationConfiguration for this stream reader. + /// + internal SerializationConfiguration SerializationConfiguration { get; private set; } #endregion #region private @@ -182,9 +217,11 @@ protected virtual void Dispose(bool disposing) { } throw new Exception("Streamreader read past end of buffer"); } internal /*protected*/ SegmentedList bytes; - internal /*protected*/ int position; - internal /*protected*/ int endPosition; + internal /*protected*/ long position; + internal /*protected*/ long endPosition; private StringBuilder sb; + private Func readLabel; + private readonly int sizeOfSerializedStreamLabel; #endregion } } diff --git a/src/FastSerialization/SegmentedMemoryStreamWriter.cs b/src/FastSerialization/SegmentedMemoryStreamWriter.cs index 7094be52a..e97783bad 100644 --- a/src/FastSerialization/SegmentedMemoryStreamWriter.cs +++ b/src/FastSerialization/SegmentedMemoryStreamWriter.cs @@ -2,15 +2,32 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Text; namespace FastSerialization { public class SegmentedMemoryStreamWriter { - public SegmentedMemoryStreamWriter() : this(64) { } - public SegmentedMemoryStreamWriter(int initialSize) + public SegmentedMemoryStreamWriter(SerializationConfiguration config = null) : this(64, config) { } + public SegmentedMemoryStreamWriter(long initialSize, SerializationConfiguration config = null) { + SerializationConfiguration = config ?? new SerializationConfiguration(); + + if (SerializationConfiguration.StreamLabelWidth == StreamLabelWidth.FourBytes) + { + writeLabel = (value) => + { + Debug.Assert((long)value <= int.MaxValue, "The StreamLabel overflowed, it should not be treated as a 32bit value."); + Write((int)value); + }; + } + else + { + writeLabel = (value) => + { + Write((long)value); + }; + } + bytes = new SegmentedList(65_536, initialSize); } @@ -45,10 +62,8 @@ public void Write(long value) bytes.Add((byte)value); value = value >> 8; bytes.Add((byte)value); value = value >> 8; } - public void Write(StreamLabel value) - { - Write((int)value); - } + public void Write(StreamLabel value) => writeLabel(value); + public void Write(string value) { if (value == null) @@ -85,10 +100,15 @@ public void WriteToStream(Stream outputStream) public SegmentedMemoryStreamReader GetReader() { var readerBytes = bytes; - return new SegmentedMemoryStreamReader(readerBytes, 0, (int)readerBytes.Count); + return new SegmentedMemoryStreamReader(readerBytes, 0, readerBytes.Count, SerializationConfiguration); } public void Dispose() { } + /// + /// Returns the SerializationConfiguration for this stream writer. + /// + internal SerializationConfiguration SerializationConfiguration { get; private set; } + #region private protected virtual void MakeSpace() { @@ -101,6 +121,7 @@ public byte[] GetBytes() } protected SegmentedList bytes; + private Action writeLabel; #endregion } } diff --git a/src/HeapDump/GCHeapDump.cs b/src/HeapDump/GCHeapDump.cs index f89c1e5b4..3790fa368 100644 --- a/src/HeapDump/GCHeapDump.cs +++ b/src/HeapDump/GCHeapDump.cs @@ -709,7 +709,7 @@ void IFastSerializable.FromStream(Deserializer deserializer) /// /// /// -public class XmlGcHeapDump +internal class XmlGcHeapDump { public static GCHeapDump ReadGCHeapDumpFromXml(string fileName) { @@ -840,7 +840,7 @@ public static MemoryGraph ReadMemoryGraphFromXml(XmlReader reader) return graph; } - public static void WriteGCDumpToXml(GCHeapDump gcDump, StreamWriter writer) + internal static void WriteGCDumpToXml(GCHeapDump gcDump, StreamWriter writer) { writer.WriteLine(""); diff --git a/src/MemoryGraph/MemoryGraph.cs b/src/MemoryGraph/MemoryGraph.cs index bf2b1e2ff..0ea680875 100644 --- a/src/MemoryGraph/MemoryGraph.cs +++ b/src/MemoryGraph/MemoryGraph.cs @@ -12,7 +12,7 @@ public MemoryGraph(int expectedSize, bool isVeryLargeGraph = false) { // If we have too many addresses we will reach the Dictionary's internal array's size limit and throw. // Therefore use a new implementation of it that is similar in performance but that can handle the extra load. - if (expectedSize > 200_000) + if (isVeryLargeGraph) { m_addressToNodeIndex = new SegmentedDictionary(expectedSize); } diff --git a/src/MemoryGraph/graph.cs b/src/MemoryGraph/graph.cs index 9b08d46b9..44ad63bda 100644 --- a/src/MemoryGraph/graph.cs +++ b/src/MemoryGraph/graph.cs @@ -5,7 +5,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Text; using System.Text.RegularExpressions; using Address = System.UInt64; @@ -120,9 +119,9 @@ public virtual NodeType AllocTypeNodeStorage() /// public NodeIndex NodeIndexLimit { get { return (NodeIndex)m_nodes.Count; } } /// - /// Same as NodeIndexLimit, just cast to an integer. + /// Same as NodeIndexLimit. /// - public int NodeCount { get { return (int)m_nodes.Count; } } + public long NodeCount { get { return m_nodes.Count; } } /// /// It is expected that users will want additional information associated with TYPES of the nodes of the graph. They can /// do this by allocating an array of code:NodeTypeIndexLimit and then indexing this by code:NodeTypeIndex @@ -161,8 +160,11 @@ public virtual NodeType AllocTypeNodeStorage() /// /// TODO I can eliminate the need for AllowReading. /// + /// if isVeryLargeGraph argument is true, then StreamLabels will be serialized as longs + /// too acommodate for the extra size of the graph's stream representation. public Graph(int expectedNodeCount, bool isVeryLargeGraph = false) { + m_isVeryLargeGraph = isVeryLargeGraph; m_expectedNodeCount = expectedNodeCount; m_types = new GrowableArray(Math.Max(expectedNodeCount / 100, 2000)); m_nodes = new SegmentedList(SegmentSize, m_expectedNodeCount); @@ -462,7 +464,8 @@ private void ClearWorker() RootIndex = NodeIndex.Invalid; if (m_writer == null) { - m_writer = new SegmentedMemoryStreamWriter(m_expectedNodeCount * 8); + m_writer = new SegmentedMemoryStreamWriter(m_expectedNodeCount * 8, + m_isVeryLargeGraph ? new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.EightBytes } : null); } m_totalSize = 0; @@ -590,7 +593,9 @@ public void FromStream(Deserializer deserializer) // Read in the Blob stream. // TODO be lazy about reading in the blobs. int blobCount = deserializer.ReadInt(); - SegmentedMemoryStreamWriter writer = new SegmentedMemoryStreamWriter(blobCount); + SegmentedMemoryStreamWriter writer = new SegmentedMemoryStreamWriter(blobCount, + m_isVeryLargeGraph ? new SerializationConfiguration() { StreamLabelWidth = StreamLabelWidth.EightBytes } : null); + while (8 <= blobCount) { writer.Write(deserializer.ReadInt64()); @@ -649,7 +654,7 @@ public void FromStream(Deserializer deserializer) } } - private int m_expectedNodeCount; // Initial guess at graph Size. + private long m_expectedNodeCount; // Initial guess at graph Size. private long m_totalSize; // Total Size of all the nodes in the graph. internal int m_totalRefs; // Total Number of references in the graph internal GrowableArray m_types; // We expect only thousands of these