Fix #41618 Correct marshalling of SortKey objects on Linux

src/libraries/System.DirectoryServices.Protocols/src/System.DirectoryServices.Protocols.csproj

@@ -42,6 +42,7 @@
     <Compile Include="System\DirectoryServices\Protocols\ldap\LdapPartialResultsProcessor.cs" />
     <Compile Include="System\DirectoryServices\Protocols\ldap\LdapSessionOptions.cs" />
     <Compile Include="System\DirectoryServices\Protocols\Interop\SafeHandles.cs" />
+    <Compile Include="System\DirectoryServices\Protocols\Interop\SortKeyInterop.cs" />
     <Compile Include="$(CommonPath)DisableRuntimeMarshalling.cs"
       Condition="$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net7.0'))">
       <Link>Common\DisableRuntimeMarshalling.cs</Link>
@@ -58,6 +59,7 @@
     <Compile Include="System\DirectoryServices\Protocols\common\QuotaControl.Windows.cs" />
     <Compile Include="System\DirectoryServices\Protocols\Interop\LdapPal.Windows.cs" />
     <Compile Include="System\DirectoryServices\Protocols\Interop\BerPal.Windows.cs" />
+    <Compile Include="System\DirectoryServices\Protocols\Interop\SortKeyInterop.Windows.cs" />
     <Compile Include="System\DirectoryServices\Protocols\ldap\LdapConnection.Windows.cs" />
     <Compile Include="System\DirectoryServices\Protocols\ldap\LdapSessionOptions.Windows.cs" />
     <Compile Include="System\DirectoryServices\Protocols\Interop\SafeHandles.Windows.cs" />
@@ -76,6 +78,7 @@
     <Compile Include="System\DirectoryServices\Protocols\common\QuotaControl.Linux.cs" />
     <Compile Include="System\DirectoryServices\Protocols\Interop\LdapPal.Linux.cs" />
     <Compile Include="System\DirectoryServices\Protocols\Interop\BerPal.Linux.cs" />
+    <Compile Include="System\DirectoryServices\Protocols\Interop\SortKeyInterop.Linux.cs" />
     <Compile Include="System\DirectoryServices\Protocols\ldap\LdapConnection.Linux.cs" />
     <Compile Include="System\DirectoryServices\Protocols\ldap\LdapSessionOptions.Linux.cs" />
     <Compile Include="System\DirectoryServices\Protocols\ldap\LocalAppContextSwitches.cs" />

src/libraries/System.DirectoryServices.Protocols/src/System/DirectoryServices/Protocols/Interop/SortKeyInterop.Linux.cs

@@ -0,0 +1,18 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections;
+using System.ComponentModel;
+using System.Diagnostics;
+using System.Text;
+using System.Runtime.InteropServices;
+using System.Security.Principal;
+using System.Runtime.Versioning;
+
+namespace System.DirectoryServices.Protocols
+{
+    [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)]
+    internal partial struct SortKeyInterop
+    {
+    }
+}

src/libraries/System.DirectoryServices.Protocols/src/System/DirectoryServices/Protocols/Interop/SortKeyInterop.Windows.cs

@@ -0,0 +1,18 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections;
+using System.ComponentModel;
+using System.Diagnostics;
+using System.Text;
+using System.Runtime.InteropServices;
+using System.Security.Principal;
+using System.Runtime.Versioning;
+
+namespace System.DirectoryServices.Protocols
+{
+    [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
+    internal partial struct SortKeyInterop
+    {
+    }
+}

src/libraries/System.DirectoryServices.Protocols/src/System/DirectoryServices/Protocols/Interop/SortKeyInterop.cs

@@ -0,0 +1,34 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections;
+using System.ComponentModel;
+using System.Diagnostics;
+using System.Text;
+using System.Runtime.InteropServices;
+using System.Security.Principal;
+using System.Runtime.Versioning;
+
+namespace System.DirectoryServices.Protocols
+{
+    // Declared as partial in order to be able to set the different StructLayout
+    // attributes in the Windows and Linux specific files.
+    internal partial struct SortKeyInterop
+    {
+        public SortKeyInterop(SortKey sortKey)
+        {
+            if (sortKey == null)
+                throw new ArgumentNullException(nameof(sortKey));
+
+            AttributeName = sortKey.AttributeName;
+            MatchingRule = sortKey.MatchingRule;
+            ReverseOrder = sortKey.ReverseOrder;
+        }
+
+        public string AttributeName { get; set; }
+
+        public string MatchingRule { get; set; }
+
+        public bool ReverseOrder { get; set; }
+    }
+}

src/libraries/System.DirectoryServices.Protocols/src/System/DirectoryServices/Protocols/common/DirectoryControl.cs

@@ -50,7 +50,6 @@ internal static class UtilityHandle
         public static ConnectionHandle GetHandle() => s_handle;
     }
 
-    [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
     public class SortKey
     {
         private string _name;
@@ -703,9 +702,15 @@ public SortKey[] SortKeys
 
         public override byte[] GetValue()
         {
+            SortKeyInterop[] nativeSortKeys = new SortKeyInterop[_keys.Length];
+            for (int i = 0; i < _keys.Length; ++i)
+            {
+                nativeSortKeys[i] = new SortKeyInterop(_keys[i]);
+            }
+
             IntPtr control = IntPtr.Zero;
-            int structSize = Marshal.SizeOf(typeof(SortKey));
-            int keyCount = _keys.Length;
+            int structSize = Marshal.SizeOf(typeof(SortKeyInterop));
+            int keyCount = nativeSortKeys.Length;
             IntPtr memHandle = Utility.AllocHGlobalIntPtrArray(keyCount + 1);
 
             try
@@ -716,7 +721,7 @@ public override byte[] GetValue()
                 for (i = 0; i < keyCount; i++)
                 {
                     sortPtr = Marshal.AllocHGlobal(structSize);
-                    Marshal.StructureToPtr(_keys[i], sortPtr, false);
+                    Marshal.StructureToPtr(nativeSortKeys[i], sortPtr, false);
                     tempPtr = (IntPtr)((long)memHandle + IntPtr.Size * i);
                     Marshal.WriteIntPtr(tempPtr, sortPtr);
                 }

src/libraries/System.DirectoryServices.Protocols/tests/DirectoryServicesProtocolsTests.cs

@@ -533,6 +533,58 @@ public void TestPageRequests()
             }
         }
 
+        [ConditionalFact(nameof(IsLdapConfigurationExist))]
+        public void TestSortedSearch()
+        {
+            using (LdapConnection connection = GetConnection())
+            {
+                string ouName = "ProtocolsGroup10";
+                string dn = "ou=" + ouName;
+
+                try
+                {
+                    for (int i=0; i<10; i++)
+                    {
+                        DeleteEntry(connection, "ou=ProtocolsSubGroup10." + i + "," + dn);
+                    }
+                    DeleteEntry(connection, dn);
+
+                    AddOrganizationalUnit(connection, dn);
+                    SearchResultEntry sre = SearchOrganizationalUnit(connection, LdapConfiguration.Configuration.SearchDn, ouName);
+                    Assert.NotNull(sre);
+
+                    for (int i=0; i<10; i++)
+                    {
+                        AddOrganizationalUnit(connection, "ou=ProtocolsSubGroup10." + i + "," + dn);
+                    }
+
+                    string filter = "(objectClass=*)";
+                    SearchRequest searchRequest = new SearchRequest(
+                                                        dn + "," + LdapConfiguration.Configuration.SearchDn,
+                                                        filter,
+                                                        SearchScope.Subtree,
+                                                        null);
+
+                    var sortRequestControl = new SortRequestControl("ou", true);
+                    searchRequest.Controls.Add(sortRequestControl);
+
+                    SearchResponse searchResponse = (SearchResponse) connection.SendRequest(searchRequest);
+                    Assert.Equal(1, searchResponse.Controls.Length);
+                    Assert.True(searchResponse.Controls[0] is SortResponseControl);
+                    Assert.True(searchResponse.Entries.Count > 0);
+                    Assert.Equal("ou=ProtocolsSubGroup10.9," + dn + "," + LdapConfiguration.Configuration.SearchDn, searchResponse.Entries[0].DistinguishedName);
+                }
+                finally
+                {
+                    for (int i=0; i<20; i++)
+                    {
+                        DeleteEntry(connection, "ou=ProtocolsSubGroup10." + i + "," + dn);
+                    }
+                    DeleteEntry(connection, dn);
+                }
+            }
+        }
+
         private void DeleteAttribute(LdapConnection connection, string entryDn, string attributeName)
         {
             string dn = entryDn + "," + LdapConfiguration.Configuration.SearchDn;

src/libraries/System.DirectoryServices.Protocols/tests/SortRequestControlTests.cs

@@ -32,11 +32,15 @@ public void Ctor_SortKeys(bool critical)
 
             control.IsCritical = critical;
             var expected = (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) ?
+                // WLDAP formatted ASN.1
                 new byte[] { 48, 132, 0, 0, 0, 43, 48, 132, 0, 0, 0, 17, 4, 5,110,
                 97, 109, 101, 49, 128, 5, 114, 117, 108, 101, 49, 129,
                 1, 255, 48, 132, 0, 0, 0, 14, 4, 5, 110, 97, 109, 101,
-                50, 128, 5, 114, 117, 108, 101, 50} :
-                new byte[] { 48, 19, 48, 9, 4, 1, 110, 128, 1, 114, 129, 1, 255, 48, 6, 4, 1, 110, 128, 1, 114 };
+                50, 128, 5, 114, 117, 108, 101, 50 } :
+                // OpenLdap formatted ASN.1
+                new byte[] { 48, 35, 48, 17, 4, 5, 110, 97, 109, 101, 49, 128, 5, 
+                114, 117, 108, 101, 49, 129, 1, 255, 48, 14, 4, 5, 110, 97, 109, 
+                101, 50, 128, 5, 114, 117, 108, 101, 50 };
             Assert.Equal(expected, control.GetValue());
         }