diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index f6c30fd2ff808e..d38d5d68f88dab 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -15,7 +15,7 @@ ] }, "microsoft.dotnet.xharness.cli": { - "version": "8.0.0-prerelease.25255.2", + "version": "8.0.0-prerelease.25270.1", "commands": [ "xharness" ] diff --git a/NuGet.config b/NuGet.config index bf140690b22d67..5a09f7eff9d983 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,16 @@ + + + + + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 862e386a6a756c..419ff2f1f7f411 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -90,18 +90,18 @@ 45dd3a73dd5b64b010c4251303b3664bb30df029 - + https://github.com/dotnet/emsdk - 976b101e5539557c20e2ac39885ac879531bcf82 + 6fd14a46f7bc405f4ba53b1e691c76fdc6324b0b - + https://github.com/dotnet/emsdk - 976b101e5539557c20e2ac39885ac879531bcf82 + 6fd14a46f7bc405f4ba53b1e691c76fdc6324b0b - + https://github.com/dotnet/source-build-reference-packages - 6ae07097c0f03eb59c8a581faaedcc3f2e4cc42c + cee2d057118e3ad3c58a72a23271ad88b87d4d4d @@ -111,9 +111,9 @@ - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 @@ -121,121 +121,121 @@ 73f0850939d96131c28cf6ea6ee5aacb4da0083a - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 - + https://github.com/dotnet/runtime-assets - f223eaeaa143e11a248ee81611819e3803883e6f + 1ef844c35b4db6c120722d0abcc64dba0d3d0685 - + https://github.com/dotnet/runtime-assets - f223eaeaa143e11a248ee81611819e3803883e6f + 1ef844c35b4db6c120722d0abcc64dba0d3d0685 - + https://github.com/dotnet/runtime-assets - f223eaeaa143e11a248ee81611819e3803883e6f + 1ef844c35b4db6c120722d0abcc64dba0d3d0685 - + https://github.com/dotnet/runtime-assets - f223eaeaa143e11a248ee81611819e3803883e6f + 1ef844c35b4db6c120722d0abcc64dba0d3d0685 - + https://github.com/dotnet/runtime-assets - f223eaeaa143e11a248ee81611819e3803883e6f + 1ef844c35b4db6c120722d0abcc64dba0d3d0685 - + https://github.com/dotnet/runtime-assets - f223eaeaa143e11a248ee81611819e3803883e6f + 1ef844c35b4db6c120722d0abcc64dba0d3d0685 - + https://github.com/dotnet/runtime-assets - f223eaeaa143e11a248ee81611819e3803883e6f + 1ef844c35b4db6c120722d0abcc64dba0d3d0685 - + https://github.com/dotnet/runtime-assets - f223eaeaa143e11a248ee81611819e3803883e6f + 1ef844c35b4db6c120722d0abcc64dba0d3d0685 - + https://github.com/dotnet/runtime-assets - f223eaeaa143e11a248ee81611819e3803883e6f + 1ef844c35b4db6c120722d0abcc64dba0d3d0685 - + https://github.com/dotnet/runtime-assets - f223eaeaa143e11a248ee81611819e3803883e6f + 1ef844c35b4db6c120722d0abcc64dba0d3d0685 - + https://github.com/dotnet/runtime-assets - f223eaeaa143e11a248ee81611819e3803883e6f + 1ef844c35b4db6c120722d0abcc64dba0d3d0685 - + https://github.com/dotnet/runtime-assets - f223eaeaa143e11a248ee81611819e3803883e6f + 1ef844c35b4db6c120722d0abcc64dba0d3d0685 - + https://github.com/dotnet/runtime-assets - f223eaeaa143e11a248ee81611819e3803883e6f + 1ef844c35b4db6c120722d0abcc64dba0d3d0685 https://github.com/dotnet/llvm-project @@ -322,21 +322,21 @@ https://github.com/dotnet/runtime edbd5c769a19798b6955050baccf99e6797d3208 - + https://github.com/dotnet/xharness - 5429dfdddde318dd8355c661173da7f2f285ebce + c3f3d30ca6a7630b38e061043d4489559c06d4cc - + https://github.com/dotnet/xharness - 5429dfdddde318dd8355c661173da7f2f285ebce + c3f3d30ca6a7630b38e061043d4489559c06d4cc - + https://github.com/dotnet/xharness - 5429dfdddde318dd8355c661173da7f2f285ebce + c3f3d30ca6a7630b38e061043d4489559c06d4cc - + https://github.com/dotnet/arcade - 20ab70a74d52b68f4271bd946884e24049b14f83 + 2ce3f8c7b2614c2b19985b669ffcd934bc39ffd1 https://dev.azure.com/dnceng/internal/_git/dotnet-optimization @@ -354,13 +354,13 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-optimization 67613417f5e1af250e6ddfba79f8f2885d8e90fb - + https://github.com/dotnet/hotreload-utils - 6907eb448c5b24e402a45086171ab6c7e2269056 + 733b3be8cce2b6eb42a151bf95fbb05500fa40ee - + https://github.com/dotnet/runtime-assets - f223eaeaa143e11a248ee81611819e3803883e6f + 1ef844c35b4db6c120722d0abcc64dba0d3d0685 https://github.com/dotnet/roslyn diff --git a/eng/Versions.props b/eng/Versions.props index 454b6acef5fe32..01d2fa496566c5 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,11 +1,11 @@ - 8.0.18 + 8.0.19 8 0 - 18 + 19 8.0.100 7.0.20 6.0.36 @@ -87,21 +87,21 @@ 8.0.100 - 8.0.0-beta.25263.4 - 8.0.0-beta.25263.4 - 8.0.0-beta.25263.4 - 8.0.0-beta.25263.4 - 8.0.0-beta.25263.4 - 2.5.1-beta.25263.4 - 8.0.0-beta.25263.4 - 8.0.0-beta.25263.4 - 8.0.0-beta.25263.4 - 8.0.0-beta.25263.4 - 8.0.0-beta.25263.4 - 8.0.0-beta.25263.4 - 8.0.0-beta.25263.4 - 8.0.0-beta.25263.4 - 8.0.0-beta.25263.4 + 8.0.0-beta.25310.3 + 8.0.0-beta.25310.3 + 8.0.0-beta.25310.3 + 8.0.0-beta.25310.3 + 8.0.0-beta.25310.3 + 2.5.1-beta.25310.3 + 8.0.0-beta.25310.3 + 8.0.0-beta.25310.3 + 8.0.0-beta.25310.3 + 8.0.0-beta.25310.3 + 8.0.0-beta.25310.3 + 8.0.0-beta.25310.3 + 8.0.0-beta.25310.3 + 8.0.0-beta.25310.3 + 8.0.0-beta.25310.3 6.0.0-preview.1.102 @@ -153,20 +153,20 @@ 4.5.0 8.0.0-rc.1.23406.6 - 8.0.0-beta.25266.1 - 8.0.0-beta.25266.1 - 8.0.0-beta.25266.1 - 8.0.0-beta.25266.1 - 8.0.0-beta.25266.1 - 8.0.0-beta.25266.1 - 8.0.0-beta.25266.1 - 8.0.0-beta.25266.1 - 8.0.0-beta.25266.1 - 8.0.0-beta.25266.1 - 8.0.0-beta.25266.1 - 8.0.0-beta.25266.1 - 8.0.0-beta.25266.1 - 8.0.0-beta.25266.1 + 8.0.0-beta.25311.1 + 8.0.0-beta.25311.1 + 8.0.0-beta.25311.1 + 8.0.0-beta.25311.1 + 8.0.0-beta.25311.1 + 8.0.0-beta.25311.1 + 8.0.0-beta.25311.1 + 8.0.0-beta.25311.1 + 8.0.0-beta.25311.1 + 8.0.0-beta.25311.1 + 8.0.0-beta.25311.1 + 8.0.0-beta.25311.1 + 8.0.0-beta.25311.1 + 8.0.0-beta.25311.1 1.0.0-prerelease.23566.3 1.0.0-prerelease.23566.3 @@ -195,10 +195,10 @@ 1.1.0 17.4.0-preview-20220707-01 - 8.0.0-prerelease.25255.2 - 8.0.0-prerelease.25255.2 - 8.0.0-prerelease.25255.2 - 8.0.0-alpha.0.25265.1 + 8.0.0-prerelease.25270.1 + 8.0.0-prerelease.25270.1 + 8.0.0-prerelease.25270.1 + 8.0.0-alpha.0.25330.2 2.4.2 1.0.0 2.4.5 @@ -252,7 +252,7 @@ Note: when the name is updated, make sure to update dependency name in eng/pipelines/common/xplat-setup.yml like - DarcDependenciesChanged.Microsoft_NET_Workload_Emscripten_Current_Manifest-8_0_100_Transport --> - 8.0.18 + 8.0.19 $(MicrosoftNETWorkloadEmscriptenCurrentManifest80100Version) 1.1.87-gba258badda @@ -269,7 +269,7 @@ 3.1.7 1.0.406601 - 8.0.116 + 8.0.117 $(MicrosoftDotnetSdkInternalVersion) diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml index 9a8b0cfb9ee96b..f034aafd8b9db2 100644 --- a/eng/pipelines/libraries/helix-queues-setup.yml +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -81,14 +81,6 @@ jobs: # Limiting interp runs as we don't need as much coverage. - (Debian.12.Amd64.Open)Ubuntu.2204.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-helix-amd64 - # Linux s390x - - ${{ if eq(parameters.platform, 'linux_s390x') }}: - - Ubuntu.2004.S390X.Experimental.Open - - # Linux PPC64le - - ${{ if eq(parameters.platform, 'linux_ppc64le') }}: - - Ubuntu.2204.PPC64le.Experimental.Open - # OSX arm64 - ${{ if eq(parameters.platform, 'osx_arm64') }}: - OSX.1200.ARM64.Open diff --git a/eng/pipelines/runtime-community.yml b/eng/pipelines/runtime-community.yml index a91388e244b0b8..2ef33cfd409b2c 100644 --- a/eng/pipelines/runtime-community.yml +++ b/eng/pipelines/runtime-community.yml @@ -70,17 +70,6 @@ extends: eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_mono_excluding_wasm.containsChange'], true), eq(variables['isRollingBuild'], true)) - # extra steps, run tests - postBuildSteps: - - template: /eng/pipelines/libraries/helix.yml - parameters: - creator: dotnet-bot - testRunNamePrefixSuffix: Mono_$(_BuildConfig) - condition: >- - or( - eq(variables['librariesContainsChange'], true), - eq(variables['monoContainsChange'], true), - eq(variables['isRollingBuild'], true)) # # Build the whole product using Mono diff --git a/global.json b/global.json index a55368f8197457..fef3993456c853 100644 --- a/global.json +++ b/global.json @@ -1,16 +1,16 @@ { "sdk": { - "version": "8.0.116", + "version": "8.0.117", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "8.0.116" + "dotnet": "8.0.117" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25263.4", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.25263.4", - "Microsoft.DotNet.SharedFramework.Sdk": "8.0.0-beta.25263.4", + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25310.3", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.25310.3", + "Microsoft.DotNet.SharedFramework.Sdk": "8.0.0-beta.25310.3", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.Build.Traversal": "3.4.0", "Microsoft.NET.Sdk.IL": "8.0.0-rc.1.23406.6" diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index b96b5917358ba5..c0086bed2f10f0 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -2487,8 +2487,6 @@ uint32_t gc_heap::m_high_memory_load_th; uint32_t gc_heap::v_high_memory_load_th; -uint32_t gc_heap::almost_high_memory_load_th; - bool gc_heap::is_restricted_physical_mem; uint64_t gc_heap::total_physical_mem = 0; @@ -12732,29 +12730,35 @@ void gc_heap::rearrange_heap_segments(BOOL compacting) #endif //!USE_REGIONS #if defined(USE_REGIONS) -// trim down the list of regions pointed at by src down to target_count, moving the extra ones to dest -static void trim_region_list (region_free_list* dest, region_free_list* src, size_t target_count) +// trim down the list of free regions pointed at by free_list down to target_count, moving the extra ones to surplus_list +static void remove_surplus_regions (region_free_list* free_list, region_free_list* surplus_list, size_t target_count) { - while (src->get_num_free_regions() > target_count) + while (free_list->get_num_free_regions() > target_count) { - heap_segment* region = src->unlink_region_front(); - dest->add_region_front (region); + // remove one region from the heap's free list + heap_segment* region = free_list->unlink_region_front(); + + // and put it on the surplus list + surplus_list->add_region_front (region); } } -// add regions from src to dest, trying to grow the size of dest to target_count -static int64_t grow_region_list (region_free_list* dest, region_free_list* src, size_t target_count) +// add regions from surplus_list to free_list, trying to reach target_count +static int64_t add_regions (region_free_list* free_list, region_free_list* surplus_list, size_t target_count) { int64_t added_count = 0; - while (dest->get_num_free_regions() < target_count) + while (free_list->get_num_free_regions() < target_count) { - if (src->get_num_free_regions() == 0) + if (surplus_list->get_num_free_regions() == 0) break; added_count++; - heap_segment* region = src->unlink_region_front(); - dest->add_region_front (region); + // remove one region from the surplus list + heap_segment* region = surplus_list->unlink_region_front(); + + // and put it on the heap's free list + free_list->add_region_front (region); } return added_count; } @@ -13236,102 +13240,12 @@ void region_free_list::sort_by_committed_and_age() } tail_free_region = prev; } +#endif //USE_REGIONS -void gc_heap::age_free_regions (const char* msg) -{ - // If we are doing an ephemeral GC as a precursor to a BGC, then we will age all of the region - // kinds during the ephemeral GC and skip the call to age_free_regions during the BGC itself. - bool age_all_region_kinds = (settings.condemned_generation == max_generation); - - if (!age_all_region_kinds) - { -#ifdef MULTIPLE_HEAPS - gc_heap* hp = g_heaps[0]; -#else //MULTIPLE_HEAPS - gc_heap* hp = pGenGCHeap; -#endif //MULTIPLE_HEAPS - age_all_region_kinds = (hp->current_bgc_state == bgc_initialized); - } - - if (age_all_region_kinds) - { - global_free_huge_regions.age_free_regions(); - } - -#ifdef MULTIPLE_HEAPS - for (int i = 0; i < n_heaps; i++) - { - gc_heap* hp = g_heaps[i]; -#else //MULTIPLE_HEAPS - { - gc_heap* hp = pGenGCHeap; - const int i = 0; -#endif //MULTIPLE_HEAPS - - if (age_all_region_kinds) - { - // age and print all kinds of free regions - region_free_list::age_free_regions (hp->free_regions); - region_free_list::print (hp->free_regions, i, msg); - } - else - { - // age and print only basic free regions - hp->free_regions[basic_free_region].age_free_regions(); - hp->free_regions[basic_free_region].print (i, msg); - } - } -} - -// distribute_free_regions is called during all blocking GCs and in the start of the BGC mark phase -// unless we already called it during an ephemeral GC right before the BGC. -// -// Free regions are stored on the following permanent lists: -// - global_regions_to_decommit -// - global_free_huge_regions -// - (per-heap) free_regions -// and the following lists that are local to distribute_free_regions: -// - aged_regions -// - surplus_regions -// -// For reason_induced_aggressive GCs, we decommit all regions. Therefore, the below description is -// for other GC types. -// -// distribute_free_regions steps: -// -// 1. Process region ages -// a. Move all huge regions from free_regions to global_free_huge_regions. -// (The intention is that free_regions shouldn't contain any huge regions outside of the period -// where a GC reclaims them and distribute_free_regions moves them to global_free_huge_regions, -// though perhaps BGC can leave them there. Future work could verify and assert this.) -// b. Move any basic region in global_regions_to_decommit (which means we intended to decommit them -// but haven't done so yet) to surplus_regions -// c. Move all huge regions that are past the age threshold from global_free_huge_regions to aged_regions -// d. Move all basic/large regions that are past the age threshold from free_regions to aged_regions -// 2. Move all regions from aged_regions to global_regions_to_decommit. Note that the intention is to -// combine this with move_highest_free_regions in a future change, which is why we don't just do this -// in steps 1c/1d. -// 3. Compute the required per-heap budgets for SOH (basic regions) and the balance. The budget for LOH -// (large) is zero as we are using an entirely age-based approach. -// balance = (number of free regions) - budget -// 4. Decide if we are going to distribute or decommit a nonzero balance. To distribute, we adjust the -// per-heap budgets, so after this step the LOH (large) budgets can be positive. -// a. A negative balance (deficit) for SOH (basic) will be distributed it means we expect to use -// more memory than we have on the free lists. A negative balance for LOH (large) isn't possible -// for LOH since the budgets start at zero. -// b. For SOH (basic), we will decommit surplus regions unless we are in a foreground GC during BGC. -// c. For LOH (large), we will distribute surplus regions since we are using an entirely age-based -// approach. However, if we are in a high-memory-usage scenario, we will decommit. In this case, -// we will also decommit the huge regions in global_free_huge_regions. Note that they were not -// originally included in the balance because they are kept in a global list. Only basic/large -// regions are kept in per-heap lists where they can be distributed. -// 5. Implement the distribute-or-decommit strategy. To distribute, we simply move regions across heaps, -// using surplus_regions as a holding space. To decommit, for server GC we generally leave them on the -// global_regions_to_decommit list and decommit them over time. However, in high-memory-usage scenarios, -// we will immediately decommit some or all of these regions. For workstation GC, we decommit a limited -// amount and move the rest back to the (one) heap's free_list. void gc_heap::distribute_free_regions() { +#ifdef USE_REGIONS + const int kind_count = large_free_region + 1; #ifdef MULTIPLE_HEAPS BOOL joined_last_gc_before_oom = FALSE; @@ -13348,8 +13262,6 @@ void gc_heap::distribute_free_regions() #endif //MULTIPLE_HEAPS if (settings.reason == reason_induced_aggressive) { - global_regions_to_decommit[huge_free_region].transfer_regions (&global_free_huge_regions); - #ifdef MULTIPLE_HEAPS for (int i = 0; i < n_heaps; i++) { @@ -13400,22 +13312,22 @@ void gc_heap::distribute_free_regions() } // first step: accumulate the number of free regions and the budget over all heaps - // - // The initial budget will only be calculated for basic free regions. For large regions, the initial budget - // is zero, and distribute-vs-decommit will be determined entirely by region ages and whether we are in a - // high memory usage scenario. Distributing a surplus/deficit of regions can change the budgets that are used. - size_t total_num_free_regions[count_distributed_free_region_kinds] = { 0, 0 }; - size_t total_budget_in_region_units[count_distributed_free_region_kinds] = { 0, 0 }; - - size_t heap_budget_in_region_units[count_distributed_free_region_kinds][MAX_SUPPORTED_CPUS] = {}; - size_t min_heap_budget_in_region_units[count_distributed_free_region_kinds][MAX_SUPPORTED_CPUS] = {}; - region_free_list aged_regions[count_free_region_kinds]; - region_free_list surplus_regions[count_distributed_free_region_kinds]; - - // we may still have regions left on the regions_to_decommit list - - // use these to fill the budget as well - surplus_regions[basic_free_region].transfer_regions (&global_regions_to_decommit[basic_free_region]); + // and move huge regions to global free list + size_t total_num_free_regions[kind_count] = { 0, 0 }; + size_t total_budget_in_region_units[kind_count] = { 0, 0 }; + size_t num_decommit_regions_by_time = 0; + size_t size_decommit_regions_by_time = 0; + size_t heap_budget_in_region_units[MAX_SUPPORTED_CPUS][kind_count]; + size_t min_heap_budget_in_region_units[MAX_SUPPORTED_CPUS]; + size_t region_size[kind_count] = { global_region_allocator.get_region_alignment(), global_region_allocator.get_large_region_alignment() }; + region_free_list surplus_regions[kind_count]; + for (int kind = basic_free_region; kind < kind_count; kind++) + { + // we may still have regions left on the regions_to_decommit list - + // use these to fill the budget as well + surplus_regions[kind].transfer_regions (&global_regions_to_decommit[kind]); + } #ifdef MULTIPLE_HEAPS for (int i = 0; i < n_heaps; i++) { @@ -13423,87 +13335,167 @@ void gc_heap::distribute_free_regions() #else //MULTIPLE_HEAPS { gc_heap* hp = pGenGCHeap; + // just to reduce the number of #ifdefs in the code below + const int i = 0; + const int n_heaps = 1; #endif //MULTIPLE_HEAPS + for (int kind = basic_free_region; kind < kind_count; kind++) + { + // If there are regions in free that haven't been used in AGE_IN_FREE_TO_DECOMMIT GCs we always decommit them. + region_free_list& region_list = hp->free_regions[kind]; + heap_segment* next_region = nullptr; + for (heap_segment* region = region_list.get_first_free_region(); region != nullptr; region = next_region) + { + next_region = heap_segment_next (region); + int age_in_free_to_decommit = min (max (AGE_IN_FREE_TO_DECOMMIT, n_heaps), MAX_AGE_IN_FREE); + // when we are about to get OOM, we'd like to discount the free regions that just have the initial page commit as they are not useful + if ((heap_segment_age_in_free (region) >= age_in_free_to_decommit) || + ((get_region_committed_size (region) == GC_PAGE_SIZE) && joined_last_gc_before_oom)) + { + num_decommit_regions_by_time++; + size_decommit_regions_by_time += get_region_committed_size (region); + dprintf (REGIONS_LOG, ("h%2d region %p age %2d, decommit", + i, heap_segment_mem (region), heap_segment_age_in_free (region))); + region_free_list::unlink_region (region); + region_free_list::add_region (region, global_regions_to_decommit); + } + } + + total_num_free_regions[kind] += region_list.get_num_free_regions(); + } + global_free_huge_regions.transfer_regions (&hp->free_regions[huge_free_region]); + + heap_budget_in_region_units[i][basic_free_region] = 0; + min_heap_budget_in_region_units[i] = 0; + heap_budget_in_region_units[i][large_free_region] = 0; } - move_all_aged_regions(total_num_free_regions, aged_regions, joined_last_gc_before_oom); - // For now, we just decommit right away, but eventually these will be used in move_highest_free_regions - move_regions_to_decommit(aged_regions); + for (int gen = soh_gen0; gen < total_generation_count; gen++) + { + if ((gen <= soh_gen2) && + total_budget_in_region_units[basic_free_region] >= (total_num_free_regions[basic_free_region] + + surplus_regions[basic_free_region].get_num_free_regions())) + { + // don't accumulate budget from higher soh generations if we cannot cover lower ones + dprintf (REGIONS_LOG, ("out of free regions - skipping gen %d budget = %zd >= avail %zd", + gen, + total_budget_in_region_units[basic_free_region], + total_num_free_regions[basic_free_region] + surplus_regions[basic_free_region].get_num_free_regions())); + continue; + } +#ifdef MULTIPLE_HEAPS + for (int i = 0; i < n_heaps; i++) + { + gc_heap* hp = g_heaps[i]; +#else //MULTIPLE_HEAPS + { + gc_heap* hp = pGenGCHeap; + // just to reduce the number of #ifdefs in the code below + const int i = 0; + const int n_heaps = 1; +#endif //MULTIPLE_HEAPS + ptrdiff_t budget_gen = max (hp->estimate_gen_growth (gen), 0); + int kind = gen >= loh_generation; + size_t budget_gen_in_region_units = (budget_gen + (region_size[kind] - 1)) / region_size[kind]; + dprintf (REGIONS_LOG, ("h%2d gen %d has an estimated growth of %zd bytes (%zd regions)", i, gen, budget_gen, budget_gen_in_region_units)); + if (gen <= soh_gen2) + { + // preserve the budget for the previous generation - we should not go below that + min_heap_budget_in_region_units[i] = heap_budget_in_region_units[i][kind]; + } + heap_budget_in_region_units[i][kind] += budget_gen_in_region_units; + total_budget_in_region_units[kind] += budget_gen_in_region_units; + } + } + + dprintf (1, ("moved %2zd regions (%8zd) to decommit based on time", num_decommit_regions_by_time, size_decommit_regions_by_time)); - size_t total_basic_free_regions = total_num_free_regions[basic_free_region] + surplus_regions[basic_free_region].get_num_free_regions(); - total_budget_in_region_units[basic_free_region] = compute_basic_region_budgets(heap_budget_in_region_units[basic_free_region], min_heap_budget_in_region_units[basic_free_region], total_basic_free_regions); + global_free_huge_regions.transfer_regions (&global_regions_to_decommit[huge_free_region]); - bool aggressive_decommit_large_p = joined_last_gc_before_oom || dt_high_memory_load_p() || near_heap_hard_limit_p(); + size_t free_space_in_huge_regions = global_free_huge_regions.get_size_free_regions(); - int region_factor[count_distributed_free_region_kinds] = { 1, LARGE_REGION_FACTOR }; + ptrdiff_t num_regions_to_decommit[kind_count]; + int region_factor[kind_count] = { 1, LARGE_REGION_FACTOR }; +#ifdef TRACE_GC + const char* kind_name[count_free_region_kinds] = { "basic", "large", "huge"}; +#endif // TRACE_GC #ifndef MULTIPLE_HEAPS // just to reduce the number of #ifdefs in the code below const int n_heaps = 1; #endif //!MULTIPLE_HEAPS - for (int kind = basic_free_region; kind < count_distributed_free_region_kinds; kind++) + size_t num_huge_region_units_to_consider[kind_count] = { 0, free_space_in_huge_regions / region_size[large_free_region] }; + + for (int kind = basic_free_region; kind < kind_count; kind++) { - dprintf(REGIONS_LOG, ("%zd %s free regions, %zd regions budget, %zd regions on surplus list", + num_regions_to_decommit[kind] = surplus_regions[kind].get_num_free_regions(); + + dprintf(REGIONS_LOG, ("%zd %s free regions, %zd regions budget, %zd regions on decommit list, %zd huge regions to consider", total_num_free_regions[kind], - free_region_kind_name[kind], + kind_name[kind], total_budget_in_region_units[kind], - surplus_regions[kind].get_num_free_regions())); + num_regions_to_decommit[kind], + num_huge_region_units_to_consider[kind])); // check if the free regions exceed the budget // if so, put the highest free regions on the decommit list - total_num_free_regions[kind] += surplus_regions[kind].get_num_free_regions(); + total_num_free_regions[kind] += num_regions_to_decommit[kind]; - ptrdiff_t balance_to_distribute = total_num_free_regions[kind] - total_budget_in_region_units[kind]; + ptrdiff_t balance = total_num_free_regions[kind] + num_huge_region_units_to_consider[kind] - total_budget_in_region_units[kind]; - if (distribute_surplus_p(balance_to_distribute, kind, aggressive_decommit_large_p)) + if ( +#ifdef BACKGROUND_GC + background_running_p() || +#endif + (balance < 0)) { + dprintf (REGIONS_LOG, ("distributing the %zd %s regions deficit", -balance, kind_name[kind])); + #ifdef MULTIPLE_HEAPS - // we may have a deficit or - for large regions or if background GC is going on - a surplus. + // we may have a deficit or - if background GC is going on - a surplus. // adjust the budget per heap accordingly - if (balance_to_distribute != 0) + if (balance != 0) { - dprintf (REGIONS_LOG, ("distributing the %zd %s regions deficit", -balance_to_distribute, free_region_kind_name[kind])); - ptrdiff_t curr_balance = 0; ptrdiff_t rem_balance = 0; for (int i = 0; i < n_heaps; i++) { - curr_balance += balance_to_distribute; + curr_balance += balance; ptrdiff_t adjustment_per_heap = curr_balance / n_heaps; curr_balance -= adjustment_per_heap * n_heaps; - ptrdiff_t new_budget = (ptrdiff_t)heap_budget_in_region_units[kind][i] + adjustment_per_heap; - ptrdiff_t min_budget = (ptrdiff_t)min_heap_budget_in_region_units[kind][i]; + ptrdiff_t new_budget = (ptrdiff_t)heap_budget_in_region_units[i][kind] + adjustment_per_heap; + ptrdiff_t min_budget = (kind == basic_free_region) ? (ptrdiff_t)min_heap_budget_in_region_units[i] : 0; dprintf (REGIONS_LOG, ("adjusting the budget for heap %d from %zd %s regions by %zd to %zd", i, - heap_budget_in_region_units[kind][i], - free_region_kind_name[kind], + heap_budget_in_region_units[i][kind], + kind_name[kind], adjustment_per_heap, max (min_budget, new_budget))); - heap_budget_in_region_units[kind][i] = max (min_budget, new_budget); - rem_balance += new_budget - heap_budget_in_region_units[kind][i]; + heap_budget_in_region_units[i][kind] = max (min_budget, new_budget); + rem_balance += new_budget - heap_budget_in_region_units[i][kind]; } assert (rem_balance <= 0); - dprintf (REGIONS_LOG, ("remaining balance: %zd %s regions", rem_balance, free_region_kind_name[kind])); + dprintf (REGIONS_LOG, ("remaining balance: %zd %s regions", rem_balance, kind_name[kind])); // if we have a left over deficit, distribute that to the heaps that still have more than the minimum while (rem_balance < 0) { for (int i = 0; i < n_heaps; i++) { - size_t min_budget = min_heap_budget_in_region_units[kind][i]; - if (heap_budget_in_region_units[kind][i] > min_budget) + size_t min_budget = (kind == basic_free_region) ? min_heap_budget_in_region_units[i] : 0; + if (heap_budget_in_region_units[i][kind] > min_budget) { dprintf (REGIONS_LOG, ("adjusting the budget for heap %d from %zd %s regions by %d to %zd", i, - heap_budget_in_region_units[kind][i], - free_region_kind_name[kind], + heap_budget_in_region_units[i][kind], + kind_name[kind], -1, - heap_budget_in_region_units[kind][i] - 1)); + heap_budget_in_region_units[i][kind] - 1)); - heap_budget_in_region_units[kind][i] -= 1; + heap_budget_in_region_units[i][kind] -= 1; rem_balance += 1; if (rem_balance == 0) break; @@ -13515,44 +13507,35 @@ void gc_heap::distribute_free_regions() } else { - assert (balance_to_distribute >= 0); - - ptrdiff_t balance_to_decommit = balance_to_distribute; - if (kind == large_free_region) - { - // huge regions aren't part of balance_to_distribute because they are kept in a global list - // and therefore can't be distributed across heaps - balance_to_decommit += global_free_huge_regions.get_size_free_regions() / global_region_allocator.get_large_region_alignment(); - } - + num_regions_to_decommit[kind] = balance; dprintf(REGIONS_LOG, ("distributing the %zd %s regions, removing %zd regions", total_budget_in_region_units[kind], - free_region_kind_name[kind], - balance_to_decommit)); + kind_name[kind], + num_regions_to_decommit[kind])); - if (balance_to_decommit > 0) + if (num_regions_to_decommit[kind] > 0) { // remember how many regions we had on the decommit list already due to aging size_t num_regions_to_decommit_before = global_regions_to_decommit[kind].get_num_free_regions(); // put the highest regions on the decommit list - global_region_allocator.move_highest_free_regions (balance_to_decommit * region_factor[kind], + global_region_allocator.move_highest_free_regions (num_regions_to_decommit[kind]*region_factor[kind], kind == basic_free_region, global_regions_to_decommit); dprintf (REGIONS_LOG, ("Moved %zd %s regions to decommit list", - global_regions_to_decommit[kind].get_num_free_regions(), free_region_kind_name[kind])); + global_regions_to_decommit[kind].get_num_free_regions(), kind_name[kind])); if (kind == basic_free_region) { - // we should now have 'balance' regions more on the decommit list + // we should now have num_regions_to_decommit[kind] regions more on the decommit list assert (global_regions_to_decommit[kind].get_num_free_regions() == - num_regions_to_decommit_before + (size_t)balance_to_decommit); + num_regions_to_decommit_before + (size_t)num_regions_to_decommit[kind]); } else { dprintf (REGIONS_LOG, ("Moved %zd %s regions to decommit list", - global_regions_to_decommit[huge_free_region].get_num_free_regions(), free_region_kind_name[huge_free_region])); + global_regions_to_decommit[huge_free_region].get_num_free_regions(), kind_name[huge_free_region])); // cannot assert we moved any regions because there may be a single huge region with more than we want to decommit } @@ -13560,7 +13543,7 @@ void gc_heap::distribute_free_regions() } } - for (int kind = basic_free_region; kind < count_distributed_free_region_kinds; kind++) + for (int kind = basic_free_region; kind < kind_count; kind++) { #ifdef MULTIPLE_HEAPS // now go through all the heaps and remove any free regions above the target count @@ -13568,16 +13551,16 @@ void gc_heap::distribute_free_regions() { gc_heap* hp = g_heaps[i]; - if (hp->free_regions[kind].get_num_free_regions() > heap_budget_in_region_units[kind][i]) + if (hp->free_regions[kind].get_num_free_regions() > heap_budget_in_region_units[i][kind]) { dprintf (REGIONS_LOG, ("removing %zd %s regions from heap %d with %zd regions, budget is %zd", - hp->free_regions[kind].get_num_free_regions() - heap_budget_in_region_units[kind][i], - free_region_kind_name[kind], + hp->free_regions[kind].get_num_free_regions() - heap_budget_in_region_units[i][kind], + kind_name[kind], i, hp->free_regions[kind].get_num_free_regions(), - heap_budget_in_region_units[kind][i])); + heap_budget_in_region_units[i][kind])); - trim_region_list (&surplus_regions[kind], &hp->free_regions[kind], heap_budget_in_region_units[kind][i]); + remove_surplus_regions (&hp->free_regions[kind], &surplus_regions[kind], heap_budget_in_region_units[i][kind]); } } // finally go through all the heaps and distribute any surplus regions to heaps having too few free regions @@ -13591,15 +13574,15 @@ void gc_heap::distribute_free_regions() #endif //MULTIPLE_HEAPS // second pass: fill all the regions having less than budget - if (hp->free_regions[kind].get_num_free_regions() < heap_budget_in_region_units[kind][i]) + if (hp->free_regions[kind].get_num_free_regions() < heap_budget_in_region_units[i][kind]) { - int64_t num_added_regions = grow_region_list (&hp->free_regions[kind], &surplus_regions[kind], heap_budget_in_region_units[kind][i]); + int64_t num_added_regions = add_regions (&hp->free_regions[kind], &surplus_regions[kind], heap_budget_in_region_units[i][kind]); dprintf (REGIONS_LOG, ("added %zd %s regions to heap %d - now has %zd, budget is %zd", (size_t)num_added_regions, - free_region_kind_name[kind], + kind_name[kind], i, hp->free_regions[kind].get_num_free_regions(), - heap_budget_in_region_units[kind][i])); + heap_budget_in_region_units[i][kind])); } hp->free_regions[kind].sort_by_committed_and_age(); } @@ -13611,221 +13594,7 @@ void gc_heap::distribute_free_regions() } } - decide_on_decommit_strategy(aggressive_decommit_large_p); -} - -void gc_heap::move_all_aged_regions(size_t total_num_free_regions[count_distributed_free_region_kinds], region_free_list aged_regions[count_free_region_kinds], bool joined_last_gc_before_oom) -{ - move_aged_regions(aged_regions, global_free_huge_regions, huge_free_region, joined_last_gc_before_oom); - #ifdef MULTIPLE_HEAPS - for (int i = 0; i < n_heaps; i++) - { - gc_heap* hp = g_heaps[i]; -#else //MULTIPLE_HEAPS - { - gc_heap* hp = pGenGCHeap; -#endif //MULTIPLE_HEAPS - - for (int kind = basic_free_region; kind < count_distributed_free_region_kinds; kind++) - { - move_aged_regions(aged_regions, hp->free_regions[kind], static_cast(kind), joined_last_gc_before_oom); - total_num_free_regions[kind] += hp->free_regions[kind].get_num_free_regions(); - } - } -} - -void gc_heap::move_aged_regions(region_free_list dest[count_free_region_kinds], region_free_list& src, free_region_kind kind, bool joined_last_gc_before_oom) -{ - heap_segment* next_region = nullptr; - for (heap_segment* region = src.get_first_free_region(); region != nullptr; region = next_region) - { - next_region = heap_segment_next (region); - // when we are about to get OOM, we'd like to discount the free regions that just have the initial page commit as they are not useful - if (aged_region_p(region, kind) || - ((get_region_committed_size (region) == GC_PAGE_SIZE) && joined_last_gc_before_oom)) - { - region_free_list::unlink_region (region); - region_free_list::add_region (region, dest); - } - } -} - -bool gc_heap::aged_region_p(heap_segment* region, free_region_kind kind) -{ -#ifndef MULTIPLE_HEAPS - const int n_heaps = 1; -#endif - - int age_in_free_to_decommit; - switch (kind) - { - case basic_free_region: - age_in_free_to_decommit = max(AGE_IN_FREE_TO_DECOMMIT_BASIC, n_heaps); - break; - case large_free_region: - age_in_free_to_decommit = AGE_IN_FREE_TO_DECOMMIT_LARGE; - break; - case huge_free_region: - age_in_free_to_decommit = AGE_IN_FREE_TO_DECOMMIT_HUGE; - break; - default: - assert(!"unexpected kind"); - age_in_free_to_decommit = 0; - } - - age_in_free_to_decommit = min (age_in_free_to_decommit, MAX_AGE_IN_FREE); - return (heap_segment_age_in_free (region) >= age_in_free_to_decommit); -} - -void gc_heap::move_regions_to_decommit(region_free_list regions[count_free_region_kinds]) -{ - for (int kind = basic_free_region; kind < count_free_region_kinds; kind++) - { - dprintf (1, ("moved %2zd %s regions (%8zd) to decommit based on time", - regions[kind].get_num_free_regions(), free_region_kind_name[kind], regions[kind].get_size_committed_in_free())); - } - for (int kind = basic_free_region; kind < count_free_region_kinds; kind++) - { - heap_segment* next_region = nullptr; - for (heap_segment* region = regions[kind].get_first_free_region(); region != nullptr; region = next_region) - { - next_region = heap_segment_next (region); - dprintf (REGIONS_LOG, ("region %p age %2d, decommit", - heap_segment_mem (region), heap_segment_age_in_free (region))); - region_free_list::unlink_region (region); - region_free_list::add_region (region, global_regions_to_decommit); - } - } - for (int kind = basic_free_region; kind < count_free_region_kinds; kind++) - { - assert(regions[kind].get_num_free_regions() == 0); - } -} - -size_t gc_heap::compute_basic_region_budgets( - size_t heap_basic_budget_in_region_units[MAX_SUPPORTED_CPUS], - size_t min_heap_basic_budget_in_region_units[MAX_SUPPORTED_CPUS], - size_t total_basic_free_regions) -{ - const size_t region_size = global_region_allocator.get_region_alignment(); - size_t total_budget_in_region_units = 0; - - for (int gen = soh_gen0; gen <= max_generation; gen++) - { - if (total_budget_in_region_units >= total_basic_free_regions) - { - // don't accumulate budget from higher soh generations if we cannot cover lower ones - dprintf (REGIONS_LOG, ("out of free regions - skipping gen %d budget = %zd >= avail %zd", - gen, - total_budget_in_region_units, - total_basic_free_regions)); - break; - } - -#ifdef MULTIPLE_HEAPS - for (int i = 0; i < n_heaps; i++) - { - gc_heap* hp = g_heaps[i]; -#else //MULTIPLE_HEAPS - { - gc_heap* hp = pGenGCHeap; - // just to reduce the number of #ifdefs in the code below - const int i = 0; -#endif //MULTIPLE_HEAPS - ptrdiff_t budget_gen = max (hp->estimate_gen_growth (gen), (ptrdiff_t)0); - size_t budget_gen_in_region_units = (budget_gen + (region_size - 1)) / region_size; - dprintf (REGIONS_LOG, ("h%2d gen %d has an estimated growth of %zd bytes (%zd regions)", i, gen, budget_gen, budget_gen_in_region_units)); - - // preserve the budget for the previous generation - we should not go below that - min_heap_basic_budget_in_region_units[i] = heap_basic_budget_in_region_units[i]; - - heap_basic_budget_in_region_units[i] += budget_gen_in_region_units; - total_budget_in_region_units += budget_gen_in_region_units; - } - } - - return total_budget_in_region_units; -} - -bool gc_heap::near_heap_hard_limit_p() -{ - if (heap_hard_limit) - { - int current_percent_heap_hard_limit = (int)((float)current_total_committed * 100.0 / (float)heap_hard_limit); - dprintf (REGIONS_LOG, ("committed %zd is %d%% of limit %zd", - current_total_committed, current_percent_heap_hard_limit, heap_hard_limit)); - if (current_percent_heap_hard_limit >= 90) - { - return true; - } - } - - return false; -} - -bool gc_heap::distribute_surplus_p(ptrdiff_t balance, int kind, bool aggressive_decommit_large_p) -{ - if (balance < 0) - { - return true; - } - - if (kind == basic_free_region) - { -#ifdef BACKGROUND_GC - // This is detecting FGCs that run during BGCs. It is not detecting ephemeral GCs that - // (possibly) run right before a BGC as background_running_p() is not yet true at that point. - return (background_running_p() && (settings.condemned_generation != max_generation)); -#else - return false; -#endif - } - - return !aggressive_decommit_large_p; -} - -void gc_heap::decide_on_decommit_strategy(bool joined_last_gc_before_oom) -{ -#ifdef MULTIPLE_HEAPS - if (joined_last_gc_before_oom || g_low_memory_status) - { - dprintf (REGIONS_LOG, ("low memory - decommitting everything (last_gc_before_oom=%d, g_low_memory_status=%d)", joined_last_gc_before_oom, g_low_memory_status)); - - while (decommit_step(DECOMMIT_TIME_STEP_MILLISECONDS)) - { - } - return; - } - - ptrdiff_t size_to_decommit_for_heap_hard_limit = 0; - if (heap_hard_limit) - { - size_to_decommit_for_heap_hard_limit = (ptrdiff_t)(current_total_committed - (heap_hard_limit * (MAX_ALLOWED_MEM_LOAD / 100.0f))); - size_to_decommit_for_heap_hard_limit = max(size_to_decommit_for_heap_hard_limit, (ptrdiff_t)0); - } - - // For the various high memory load situations, we're not using the process size at all. In - // particular, if we had a large process and smaller processes running in the same container, - // then we will treat them the same if the container reaches reaches high_memory_load_th. In - // the future, we could consider additional complexity to try to reclaim more memory from - // larger processes than smaller ones. - ptrdiff_t size_to_decommit_for_physical = 0; - if (settings.entry_memory_load >= high_memory_load_th) - { - size_t entry_used_physical_mem = total_physical_mem - entry_available_physical_mem; - size_t goal_used_physical_mem = (size_t)(((almost_high_memory_load_th) / 100.0) * total_physical_mem); - size_to_decommit_for_physical = entry_used_physical_mem - goal_used_physical_mem; - } - - size_t size_to_decommit = max(size_to_decommit_for_heap_hard_limit, size_to_decommit_for_physical); - if (size_to_decommit > 0) - { - dprintf (REGIONS_LOG, ("low memory - decommitting %zd (for heap_hard_limit: %zd, for physical: %zd)", size_to_decommit, size_to_decommit_for_heap_hard_limit, size_to_decommit_for_physical)); - - decommit_step(size_to_decommit / DECOMMIT_SIZE_PER_MILLISECOND); - } - for (int kind = basic_free_region; kind < count_free_region_kinds; kind++) { if (global_regions_to_decommit[kind].get_num_free_regions() != 0) @@ -13847,7 +13616,7 @@ void gc_heap::decide_on_decommit_strategy(bool joined_last_gc_before_oom) if (ephemeral_elapsed >= DECOMMIT_TIME_STEP_MILLISECONDS) { gc_last_ephemeral_decommit_time = dd_time_clock (dd0); - size_t decommit_step_milliseconds = min (ephemeral_elapsed, (size_t)(10 * 1000)); + size_t decommit_step_milliseconds = min (ephemeral_elapsed, (10*1000)); decommit_step (decommit_step_milliseconds); } @@ -13860,8 +13629,8 @@ void gc_heap::decide_on_decommit_strategy(bool joined_last_gc_before_oom) } } #endif //MULTIPLE_HEAPS -} #endif //USE_REGIONS +} #ifdef WRITE_WATCH uint8_t* g_addresses [array_size+2]; // to get around the bug in GetWriteWatch @@ -38144,16 +37913,6 @@ void gc_heap::background_mark_phase () if (bgc_t_join.joined()) #endif //MULTIPLE_HEAPS { -#ifdef USE_REGIONS - // There's no need to distribute a second time if we just did an ephemeral GC, and we don't want to - // age the free regions twice. - if (!do_ephemeral_gc_p) - { - distribute_free_regions (); - age_free_regions ("BGC"); - } -#endif //USE_REGIONS - #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP // Resetting write watch for software write watch is pretty fast, much faster than for hardware write watch. Reset // can be done while the runtime is suspended or after the runtime is restarted, the preference was to reset while @@ -52891,8 +52650,8 @@ bool gc_heap::compute_memory_settings(bool is_initialization, uint32_t& nhp, uin uint32_t highmem_th_from_config = (uint32_t)GCConfig::GetGCHighMemPercent(); if (highmem_th_from_config) { - high_memory_load_th = min (99u, highmem_th_from_config); - v_high_memory_load_th = min (99u, (high_memory_load_th + 7)); + high_memory_load_th = min (99, highmem_th_from_config); + v_high_memory_load_th = min (99, (highmem_th_from_config + 7)); #ifdef FEATURE_EVENT_TRACE high_mem_percent_from_config = highmem_th_from_config; #endif //FEATURE_EVENT_TRACE @@ -52917,7 +52676,6 @@ bool gc_heap::compute_memory_settings(bool is_initialization, uint32_t& nhp, uin } m_high_memory_load_th = min ((high_memory_load_th + 5), v_high_memory_load_th); - almost_high_memory_load_th = (high_memory_load_th > 5) ? (high_memory_load_th - 5) : 1; // avoid underflow of high_memory_load_th - 5 return true; } diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 317264023a81a0..cce6c5ee28adf0 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -1403,19 +1403,14 @@ enum interesting_data_point #ifdef USE_REGIONS enum free_region_kind { - basic_free_region = 0, - large_free_region = 1, - count_distributed_free_region_kinds = 2, - huge_free_region = 2, - count_free_region_kinds = 3, + basic_free_region, + large_free_region, + huge_free_region, + count_free_region_kinds, }; static_assert(count_free_region_kinds == FREE_REGION_KINDS, "Keep count_free_region_kinds in sync with FREE_REGION_KINDS, changing this is not a version breaking change."); -#ifdef TRACE_GC -static const char * const free_region_kind_name[count_free_region_kinds] = { "basic", "large", "huge"}; -#endif // TRACE_GC - class region_free_list { size_t num_free_regions; @@ -1722,19 +1717,6 @@ class gc_heap PER_HEAP_ISOLATED_METHOD void verify_region_to_generation_map(); PER_HEAP_ISOLATED_METHOD void compute_gc_and_ephemeral_range (int condemned_gen_number, bool end_of_gc_p); - - PER_HEAP_ISOLATED_METHOD void distribute_free_regions(); - PER_HEAP_ISOLATED_METHOD void move_all_aged_regions(size_t total_num_free_regions[count_distributed_free_region_kinds], region_free_list aged_regions[count_free_region_kinds], bool joined_last_gc_before_oom); - PER_HEAP_ISOLATED_METHOD void move_aged_regions(region_free_list dest[count_free_region_kinds], region_free_list& src, free_region_kind kind, bool joined_last_gc_before_oom); - PER_HEAP_ISOLATED_METHOD bool aged_region_p(heap_segment* region, free_region_kind kind); - PER_HEAP_ISOLATED_METHOD void move_regions_to_decommit(region_free_list oregions[count_free_region_kinds]); - PER_HEAP_ISOLATED_METHOD size_t compute_basic_region_budgets(size_t heap_basic_budget_in_region_units[MAX_SUPPORTED_CPUS], size_t min_heap_basic_budget_in_region_units[MAX_SUPPORTED_CPUS], size_t total_basic_free_regions); - PER_HEAP_ISOLATED_METHOD bool near_heap_hard_limit_p(); - PER_HEAP_ISOLATED_METHOD bool distribute_surplus_p(ptrdiff_t balance, int kind, bool aggressive_decommit_large_p); - PER_HEAP_ISOLATED_METHOD void decide_on_decommit_strategy(bool joined_last_gc_before_oom); - - PER_HEAP_ISOLATED_METHOD void age_free_regions (const char* msg); - #ifdef STRESS_REGIONS PER_HEAP_METHOD void pin_by_gc (uint8_t* object); #endif //STRESS_REGIONS @@ -2438,6 +2420,7 @@ class gc_heap PER_HEAP_METHOD void rearrange_heap_segments(BOOL compacting); #endif //!USE_REGIONS PER_HEAP_METHOD void delay_free_segments(); + PER_HEAP_ISOLATED_METHOD void distribute_free_regions(); #ifdef BACKGROUND_GC PER_HEAP_ISOLATED_METHOD void reset_write_watch_for_gc_heap(void* base_address, size_t region_size); PER_HEAP_ISOLATED_METHOD void get_write_watch_for_gc_heap(bool reset, void *base_address, size_t region_size, void** dirty_pages, uintptr_t* dirty_page_count_ref, bool is_runtime_suspended); @@ -4369,7 +4352,6 @@ class gc_heap PER_HEAP_ISOLATED_FIELD_INIT_ONLY uint32_t high_memory_load_th; PER_HEAP_ISOLATED_FIELD_INIT_ONLY uint32_t m_high_memory_load_th; PER_HEAP_ISOLATED_FIELD_INIT_ONLY uint32_t v_high_memory_load_th; - PER_HEAP_ISOLATED_FIELD_INIT_ONLY uint32_t almost_high_memory_load_th; PER_HEAP_ISOLATED_FIELD_INIT_ONLY bool is_restricted_physical_mem; PER_HEAP_ISOLATED_FIELD_INIT_ONLY uint64_t mem_one_percent; PER_HEAP_ISOLATED_FIELD_INIT_ONLY uint64_t total_physical_mem; @@ -5283,9 +5265,7 @@ class heap_segment // GCs. We stop at 99. It's initialized to 0 when a region is added to // the region's free list. #define MAX_AGE_IN_FREE 99 - #define AGE_IN_FREE_TO_DECOMMIT_BASIC 20 - #define AGE_IN_FREE_TO_DECOMMIT_LARGE 5 - #define AGE_IN_FREE_TO_DECOMMIT_HUGE 2 + #define AGE_IN_FREE_TO_DECOMMIT 20 int age_in_free; // This is currently only used by regions that are swept in plan - // we then thread this list onto the generation's free list. diff --git a/src/coreclr/vm/olevariant.cpp b/src/coreclr/vm/olevariant.cpp index e4d9174dc3edcd..0531d17fe7c802 100644 --- a/src/coreclr/vm/olevariant.cpp +++ b/src/coreclr/vm/olevariant.cpp @@ -4756,11 +4756,8 @@ void OleVariant::ConvertValueClassToVariant(OBJECTREF *pBoxedValueClass, VARIANT // Marshal the contents of the value class into the record. MethodDesc* pStructMarshalStub; { - GCPROTECT_BEGIN(*pBoxedValueClass); GCX_PREEMP(); - pStructMarshalStub = NDirect::CreateStructMarshalILStub(pValueClassMT); - GCPROTECT_END(); } MarshalStructViaILStub(pStructMarshalStub, (*pBoxedValueClass)->GetData(), (BYTE*)V_RECORD(pRecHolder), StructMarshalStubs::MarshalOperation::Marshal); diff --git a/src/installer/tests/HostActivation.Tests/PortableAppActivation.cs b/src/installer/tests/HostActivation.Tests/PortableAppActivation.cs index bd25648c5a46b5..9c54eb1b767ee6 100644 --- a/src/installer/tests/HostActivation.Tests/PortableAppActivation.cs +++ b/src/installer/tests/HostActivation.Tests/PortableAppActivation.cs @@ -216,6 +216,33 @@ public void AppHost_FrameworkDependent_GlobalLocation_Succeeds(bool useRegistere } } + [Fact] + [PlatformSpecific(TestPlatforms.Windows)] + public void AppHost_DotNetRoot_DevicePath() + { + string appExe = sharedTestState.PortableAppFixture_Published.TestProject.AppExe; + + string dotnetPath = $@"\\?\{sharedTestState.PortableAppFixture_Published.BuiltDotnet.BinPath}"; + Command.Create(appExe) + .CaptureStdErr() + .CaptureStdOut() + .DotNetRoot(dotnetPath, sharedTestState.RepoDirectories.BuildArchitecture) + .Execute() + .Should().Pass() + .And.HaveStdOutContaining("Hello World") + .And.HaveStdOutContaining(sharedTestState.RepoDirectories.MicrosoftNETCoreAppVersion); + + dotnetPath = $@"\\.\{sharedTestState.PortableAppFixture_Published.BuiltDotnet.BinPath}"; + Command.Create(appExe) + .CaptureStdErr() + .CaptureStdOut() + .DotNetRoot(dotnetPath, sharedTestState.RepoDirectories.BuildArchitecture) + .Execute() + .Should().Pass() + .And.HaveStdOutContaining("Hello World") + .And.HaveStdOutContaining(sharedTestState.RepoDirectories.MicrosoftNETCoreAppVersion); + } + [Fact] public void RuntimeConfig_FilePath_Breaks_MAX_PATH_Threshold() { diff --git a/src/installer/tests/HostActivation.Tests/StandaloneAppActivation.cs b/src/installer/tests/HostActivation.Tests/StandaloneAppActivation.cs index b5ad105a353051..04fbba687c4633 100644 --- a/src/installer/tests/HostActivation.Tests/StandaloneAppActivation.cs +++ b/src/installer/tests/HostActivation.Tests/StandaloneAppActivation.cs @@ -254,6 +254,29 @@ public void Running_Publish_Output_Standalone_EXE_with_Bound_AppHost_Succeeds() } } + [Fact] + [PlatformSpecific(TestPlatforms.Windows)] + public void DevicePath() + { + string appExe = $@"\\?\{sharedTestState.StandaloneAppFixture_Published.TestProject.AppExe}"; + Command.Create(appExe) + .CaptureStdErr() + .CaptureStdOut() + .Execute() + .Should().Pass() + .And.HaveStdOutContaining("Hello World") + .And.HaveStdOutContaining(sharedTestState.RepoDirectories.MicrosoftNETCoreAppVersion); + + appExe = $@"\\.\{sharedTestState.StandaloneAppFixture_Published.TestProject.AppExe}"; + Command.Create(appExe) + .CaptureStdErr() + .CaptureStdOut() + .Execute() + .Should().Pass() + .And.HaveStdOutContaining("Hello World") + .And.HaveStdOutContaining(sharedTestState.RepoDirectories.MicrosoftNETCoreAppVersion); + } + [Fact] [PlatformSpecific(TestPlatforms.Windows)] // GUI app host is only supported on Windows. public void Running_AppHost_with_GUI_No_Console() diff --git a/src/libraries/Common/src/Interop/OSX/Interop.libobjc.cs b/src/libraries/Common/src/Interop/OSX/Interop.libobjc.cs index 7274cb569051d7..73c57af7ec249c 100644 --- a/src/libraries/Common/src/Interop/OSX/Interop.libobjc.cs +++ b/src/libraries/Common/src/Interop/OSX/Interop.libobjc.cs @@ -46,14 +46,17 @@ internal static Version GetOperatingSystemVersion() } } - if (major == 10 && minor == 16) +#if TARGET_OSX + if (major == 16) { - // We get "compat" version for 11.0 unless we build with updated SDK. - // Hopefully that will be before 11.x comes out - // For now, this maps 10.16 to 11.0. - major = 11; - minor = 0; + // MacOS Tahoe returns a compatibility version unless it is built with a new SDK. Map the compatibility + // version to the "correct" version. This assumes the minor versions will map unchanged. + // 16.0 => 26.0 + // 16.1 => 26.1 + // etc + major = 26; } +#endif return new Version(major, minor, patch); } diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs index abf945b2d46bcc..50752f52b5708e 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs @@ -1,17 +1,20 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; using System.Collections.Generic; using System.Linq; using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; +using Microsoft.DotNet.XUnitExtensions; using Microsoft.Win32.SafeHandles; using Xunit; namespace System.IO.Tests { [SkipOnPlatform(TestPlatforms.Browser, "async file IO is not supported on browser")] + [Collection(nameof(DisableParallelization))] // don't run in parallel, as some of these tests use a LOT of resources public class RandomAccess_WriteGatherAsync : RandomAccess_Base { protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) @@ -133,5 +136,172 @@ public async Task DuplicatedBufferDuplicatesContentAsync(FileOptions options) Assert.Equal(repeatCount, actualContent.Length); Assert.All(actualContent, actual => Assert.Equal(value, actual)); } + + [OuterLoop("It consumes a lot of resources (disk space and memory).")] + [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.Is64BitProcess), nameof(PlatformDetection.IsReleaseRuntime))] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, true)] + [InlineData(true, false)] + public async Task NoInt32OverflowForLargeInputs(bool asyncFile, bool asyncMethod) + { + // We need to write more than Int32.MaxValue bytes to the disk to reproduce the problem. + // To reduce the number of used memory, we allocate only one write buffer and simply repeat it multiple times. + // For reading, we need unique buffers to ensure that all of them are getting populated with the right data. + + const int BufferCount = 1002; + const int BufferSize = int.MaxValue / 1000; + const long FileSize = (long)BufferCount * BufferSize; + string filePath = GetTestFilePath(); + + FileOptions options = asyncFile ? FileOptions.Asynchronous : FileOptions.None; // we need to test both code paths + options |= FileOptions.DeleteOnClose; + + SafeFileHandle? sfh; + try + { + sfh = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, options, preallocationSize: FileSize); + } + catch (IOException) + { + throw new SkipTestException("Not enough disk space."); + } + + using (sfh) + { + ReadOnlyMemory writeBuffer = RandomNumberGenerator.GetBytes(BufferSize); + List> writeBuffers = Enumerable.Repeat(writeBuffer, BufferCount).ToList(); + + List memoryManagers = new List(BufferCount); + List> readBuffers = new List>(BufferCount); + + try + { + try + { + for (int i = 0; i < BufferCount; i++) + { + // We are using native memory here to get OOM as soon as possible. + NativeMemoryManager nativeMemoryManager = new(BufferSize); + memoryManagers.Add(nativeMemoryManager); + readBuffers.Add(nativeMemoryManager.Memory); + } + } + catch (OutOfMemoryException) + { + throw new SkipTestException("Not enough memory."); + } + + await Verify(asyncMethod, FileSize, sfh, writeBuffer, writeBuffers, readBuffers); + } + finally + { + foreach (IDisposable memoryManager in memoryManagers) + { + memoryManager.Dispose(); + } + } + } + + static async Task Verify(bool asyncMethod, long FileSize, SafeFileHandle sfh, ReadOnlyMemory writeBuffer, List> writeBuffers, List> readBuffers) + { + if (asyncMethod) + { + await RandomAccess.WriteAsync(sfh, writeBuffers, 0); + } + else + { + RandomAccess.Write(sfh, writeBuffers, 0); + } + + Assert.Equal(FileSize, RandomAccess.GetLength(sfh)); + + long fileOffset = 0; + while (fileOffset < FileSize) + { + long bytesRead = asyncMethod + ? await RandomAccess.ReadAsync(sfh, readBuffers, fileOffset) + : RandomAccess.Read(sfh, readBuffers, fileOffset); + + Assert.InRange(bytesRead, 0, FileSize); + + while (bytesRead > 0) + { + Memory readBuffer = readBuffers[0]; + if (bytesRead >= readBuffer.Length) + { + AssertExtensions.SequenceEqual(writeBuffer.Span, readBuffer.Span); + + bytesRead -= readBuffer.Length; + fileOffset += readBuffer.Length; + + readBuffers.RemoveAt(0); + } + else + { + // A read has finished somewhere in the middle of one of the read buffers. + // Example: buffer had 30 bytes and only 10 were read. + // We don't read the missing part, but try to read the whole buffer again. + // It's not optimal from performance perspective, but it keeps the test logic simple. + break; + } + } + } + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, true)] + [InlineData(true, false)] + public async Task IovLimitsAreRespected(bool asyncFile, bool asyncMethod) + { + // We need to write and read more than IOV_MAX buffers at a time. + // IOV_MAX typical value is 1024. + const int BufferCount = 1026; + const int BufferSize = 1; // the less resources we use, the better + const int FileSize = BufferCount * BufferSize; + + ReadOnlyMemory writeBuffer = RandomNumberGenerator.GetBytes(BufferSize); + ReadOnlyMemory[] writeBuffers = Enumerable.Repeat(writeBuffer, BufferCount).ToArray(); + Memory[] readBuffers = Enumerable.Range(0, BufferCount).Select(_ => new byte[BufferSize].AsMemory()).ToArray(); + + FileOptions options = asyncFile ? FileOptions.Asynchronous : FileOptions.None; // we need to test both code paths + options |= FileOptions.DeleteOnClose; + + using SafeFileHandle sfh = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, options); + + if (asyncMethod) + { + await RandomAccess.WriteAsync(sfh, writeBuffers, 0); + } + else + { + RandomAccess.Write(sfh, writeBuffers, 0); + } + + Assert.Equal(FileSize, RandomAccess.GetLength(sfh)); + + long fileOffset = 0; + int bufferOffset = 0; + while (fileOffset < FileSize) + { + ArraySegment> left = new ArraySegment>(readBuffers, bufferOffset, readBuffers.Length - bufferOffset); + + long bytesRead = asyncMethod + ? await RandomAccess.ReadAsync(sfh, left, fileOffset) + : RandomAccess.Read(sfh, left, fileOffset); + + fileOffset += bytesRead; + // The following operation is correct only because the BufferSize is 1. + bufferOffset += (int)bytesRead; + } + + for (int i = 0; i < BufferCount; ++i) + { + AssertExtensions.SequenceEqual(writeBuffers[i].Span, readBuffers[i].Span); + } + } } } diff --git a/src/libraries/System.Net.NameResolution/tests/FunctionalTests/LoggingTest.cs b/src/libraries/System.Net.NameResolution/tests/FunctionalTests/LoggingTest.cs index 1e7b168c711c9b..b968b6a0ab1473 100644 --- a/src/libraries/System.Net.NameResolution/tests/FunctionalTests/LoggingTest.cs +++ b/src/libraries/System.Net.NameResolution/tests/FunctionalTests/LoggingTest.cs @@ -6,6 +6,7 @@ using System.Linq; using System.Net.Sockets; using System.Threading.Tasks; +using Microsoft.DotNet.RemoteExecutor; using Microsoft.DotNet.XUnitExtensions; using Xunit; @@ -28,72 +29,92 @@ public static void EventSource_ExistsWithCorrectId() Assert.NotEmpty(EventSource.GenerateManifest(esType, "assemblyPathToIncludeInManifest")); } - [ConditionalFact] + [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] public void GetHostEntry_InvalidHost_LogsError() { - using (var listener = new TestEventListener("Private.InternalDiagnostics.System.Net.NameResolution", EventLevel.Error)) + try { - var events = new ConcurrentQueue(); - - listener.RunWithCallback(ev => events.Enqueue(ev), () => + RemoteExecutor.Invoke(static () => { - try - { - Dns.GetHostEntry(Configuration.Sockets.InvalidHost); - throw new SkipTestException("GetHostEntry should fail but it did not."); - } - catch (SocketException e) when (e.SocketErrorCode == SocketError.HostNotFound) + using (var listener = new TestEventListener("Private.InternalDiagnostics.System.Net.NameResolution", EventLevel.Error)) { + var events = new ConcurrentQueue(); + + listener.RunWithCallback(ev => events.Enqueue(ev), () => + { + try + { + Dns.GetHostEntry(Configuration.Sockets.InvalidHost); + throw new SkipTestException("GetHostEntry should fail but it did not."); + } + catch (SocketException e) when (e.SocketErrorCode == SocketError.HostNotFound) + { + } + catch (Exception e) + { + throw new SkipTestException($"GetHostEntry failed unexpectedly: {e.Message}"); + } + }); + + Assert.True(events.Count > 0, "events.Count should be > 0"); + foreach (EventWrittenEventArgs ev in events) + { + Assert.True(ev.Payload.Count >= 3); + Assert.NotNull(ev.Payload[0]); + Assert.NotNull(ev.Payload[1]); + Assert.NotNull(ev.Payload[2]); + } } - catch (Exception e) - { - throw new SkipTestException($"GetHostEntry failed unexpectedly: {e.Message}"); - } - }); - - Assert.True(events.Count > 0, "events.Count should be > 0"); - foreach (EventWrittenEventArgs ev in events) - { - Assert.True(ev.Payload.Count >= 3); - Assert.NotNull(ev.Payload[0]); - Assert.NotNull(ev.Payload[1]); - Assert.NotNull(ev.Payload[2]); - } + }).Dispose(); + } + catch (Exception ex) when (ex.ToString().Contains(nameof(SkipTestException), StringComparison.Ordinal)) + { + throw new SkipTestException(ex.ToString()); } } - [ConditionalFact] - public async Task GetHostEntryAsync_InvalidHost_LogsError() + [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + public void GetHostEntryAsync_InvalidHost_LogsError() { - using (var listener = new TestEventListener("Private.InternalDiagnostics.System.Net.NameResolution", EventLevel.Error)) + try { - var events = new ConcurrentQueue(); - - await listener.RunWithCallbackAsync(ev => events.Enqueue(ev), async () => + RemoteExecutor.Invoke(static async () => { - try - { - await Dns.GetHostEntryAsync(Configuration.Sockets.InvalidHost).ConfigureAwait(false); - throw new SkipTestException("GetHostEntryAsync should fail but it did not."); - } - catch (SocketException e) when (e.SocketErrorCode == SocketError.HostNotFound) + using (var listener = new TestEventListener("Private.InternalDiagnostics.System.Net.NameResolution", EventLevel.Error)) { - await WaitForErrorEventAsync(events); + var events = new ConcurrentQueue(); + + await listener.RunWithCallbackAsync(ev => events.Enqueue(ev), async () => + { + try + { + await Dns.GetHostEntryAsync(Configuration.Sockets.InvalidHost).ConfigureAwait(false); + throw new SkipTestException("GetHostEntryAsync should fail but it did not."); + } + catch (SocketException e) when (e.SocketErrorCode == SocketError.HostNotFound) + { + await WaitForErrorEventAsync(events); + } + catch (Exception e) + { + throw new SkipTestException($"GetHostEntryAsync failed unexpectedly: {e.Message}"); + } + }).ConfigureAwait(false); + + Assert.True(events.Count > 0, "events.Count should be > 0"); + foreach (EventWrittenEventArgs ev in events) + { + Assert.True(ev.Payload.Count >= 3); + Assert.NotNull(ev.Payload[0]); + Assert.NotNull(ev.Payload[1]); + Assert.NotNull(ev.Payload[2]); + } } - catch (Exception e) - { - throw new SkipTestException($"GetHostEntryAsync failed unexpectedly: {e.Message}"); - } - }).ConfigureAwait(false); - - Assert.True(events.Count > 0, "events.Count should be > 0"); - foreach (EventWrittenEventArgs ev in events) - { - Assert.True(ev.Payload.Count >= 3); - Assert.NotNull(ev.Payload[0]); - Assert.NotNull(ev.Payload[1]); - Assert.NotNull(ev.Payload[2]); - } + }).Dispose(); + } + catch (Exception ex) when (ex.ToString().Contains(nameof(SkipTestException), StringComparison.Ordinal)) + { + throw new SkipTestException(ex.ToString()); } static async Task WaitForErrorEventAsync(ConcurrentQueue events) @@ -110,32 +131,42 @@ static async Task WaitForErrorEventAsync(ConcurrentQueue } } - [ConditionalFact] + [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] public void GetHostEntry_ValidName_NoErrors() { - using (var listener = new TestEventListener("Private.InternalDiagnostics.System.Net.NameResolution", EventLevel.Verbose)) + try { - var events = new ConcurrentQueue(); - - listener.RunWithCallback(ev => events.Enqueue(ev), () => + RemoteExecutor.Invoke(static () => { - try - { - Dns.GetHostEntryAsync("localhost").GetAwaiter().GetResult(); - Dns.GetHostEntryAsync(IPAddress.Loopback).GetAwaiter().GetResult(); - Dns.GetHostEntry("localhost"); - Dns.GetHostEntry(IPAddress.Loopback); - } - catch (Exception e) + using (var listener = new TestEventListener("Private.InternalDiagnostics.System.Net.NameResolution", EventLevel.Verbose)) { - throw new SkipTestException($"Localhost lookup failed unexpectedly: {e.Message}"); + var events = new ConcurrentQueue(); + + listener.RunWithCallback(ev => events.Enqueue(ev), () => + { + try + { + Dns.GetHostEntryAsync("localhost").GetAwaiter().GetResult(); + Dns.GetHostEntryAsync(IPAddress.Loopback).GetAwaiter().GetResult(); + Dns.GetHostEntry("localhost"); + Dns.GetHostEntry(IPAddress.Loopback); + } + catch (Exception e) + { + throw new SkipTestException($"Localhost lookup failed unexpectedly: {e.Message}"); + } + }); + + // We get some traces. + Assert.True(events.Count() > 0); + // No errors or warning for successful query. + Assert.True(events.Count(ev => (int)ev.Level > (int)EventLevel.Informational) == 0); } - }); - - // We get some traces. - Assert.True(events.Count() > 0); - // No errors or warning for successful query. - Assert.True(events.Count(ev => (int)ev.Level > (int)EventLevel.Informational) == 0); + }).Dispose(); + } + catch (Exception ex) when (ex.ToString().Contains(nameof(SkipTestException), StringComparison.Ordinal)) + { + throw new SkipTestException(ex.ToString()); } } } diff --git a/src/libraries/System.Net.Ping/tests/FunctionalTests/PingTest.cs b/src/libraries/System.Net.Ping/tests/FunctionalTests/PingTest.cs index 49a95c858a9af8..323829662c315d 100644 --- a/src/libraries/System.Net.Ping/tests/FunctionalTests/PingTest.cs +++ b/src/libraries/System.Net.Ping/tests/FunctionalTests/PingTest.cs @@ -745,53 +745,64 @@ public async Task SendPingToExternalHostWithLowTtlTest() Assert.NotEqual(IPAddress.Any, pingReply.Address); } - [Fact] - [OuterLoop] - public void Ping_TimedOut_Sync_Success() + private async Task Ping_TimedOut_Core(Func> sendPing) { - var sender = new Ping(); - PingReply reply = sender.Send(TestSettings.UnreachableAddress); + Ping sender = new Ping(); + PingReply reply = await sendPing(sender, TestSettings.UnreachableAddress); + if (reply.Status == IPStatus.DestinationNetworkUnreachable) + { + // A network middleware has dropped the packed and replied with DestinationNetworkUnreachable. Repeat the PING attempt on another address. + reply = await sendPing(sender, TestSettings.UnreachableAddress2); + } + + if (reply.Status == IPStatus.DestinationNetworkUnreachable) + { + // Do yet another attempt. + reply = await sendPing(sender, TestSettings.UnreachableAddress3); + } + Assert.Equal(IPStatus.TimedOut, reply.Status); } [Fact] [OuterLoop] - public async Task Ping_TimedOut_EAP_Success() - { - var sender = new Ping(); - sender.PingCompleted += (s, e) => - { - var tcs = (TaskCompletionSource)e.UserState; + public Task Ping_TimedOut_Sync_Success() + => Ping_TimedOut_Core((sender, address) => Task.Run(() => sender.Send(address))); - if (e.Cancelled) - { - tcs.TrySetCanceled(); - } - else if (e.Error != null) - { - tcs.TrySetException(e.Error); - } - else + [Fact] + [OuterLoop] + public Task Ping_TimedOut_EAP_Success() + => Ping_TimedOut_Core(async (sender, address) => + { + static void PingCompleted(object sender, PingCompletedEventArgs e) { - tcs.TrySetResult(e.Reply); - } - }; + var tcs = (TaskCompletionSource)e.UserState; - var tcs = new TaskCompletionSource(); - sender.SendAsync(TestSettings.UnreachableAddress, tcs); - - PingReply reply = await tcs.Task; - Assert.Equal(IPStatus.TimedOut, reply.Status); - } + if (e.Cancelled) + { + tcs.TrySetCanceled(); + } + else if (e.Error != null) + { + tcs.TrySetException(e.Error); + } + else + { + tcs.TrySetResult(e.Reply); + } + } + sender.PingCompleted += PingCompleted; + var tcs = new TaskCompletionSource(); + sender.SendAsync(address, tcs); + PingReply reply = await tcs.Task; + sender.PingCompleted -= PingCompleted; + return reply; + }); [Fact] [OuterLoop] - public async Task Ping_TimedOut_TAP_Success() - { - var sender = new Ping(); - PingReply reply = await sender.SendPingAsync(TestSettings.UnreachableAddress); - Assert.Equal(IPStatus.TimedOut, reply.Status); - } + public Task Ping_TimedOut_TAP_Success() + => Ping_TimedOut_Core((sender, address) => sender.SendPingAsync(address)); private static bool IsRemoteExecutorSupportedAndPrivilegedProcess => RemoteExecutor.IsSupported && PlatformDetection.IsPrivilegedProcess; diff --git a/src/libraries/System.Net.Ping/tests/FunctionalTests/TestSettings.cs b/src/libraries/System.Net.Ping/tests/FunctionalTests/TestSettings.cs index 0e48bb4777b2e1..78b2be986651e6 100644 --- a/src/libraries/System.Net.Ping/tests/FunctionalTests/TestSettings.cs +++ b/src/libraries/System.Net.Ping/tests/FunctionalTests/TestSettings.cs @@ -11,6 +11,9 @@ internal static class TestSettings { public static readonly string LocalHost = "localhost"; public static readonly string UnreachableAddress = "192.0.2.0"; // TEST-NET-1 + public static readonly string UnreachableAddress2 = "100.64.0.1"; // CGNAT block + public static readonly string UnreachableAddress3 = "10.255.255.1"; // High address in the private 10.0.0.0/8 range. Likely unused and unrouted. + public const int PingTimeout = 10 * 1000; public const string PayloadAsString = "'Post hoc ergo propter hoc'. 'After it, therefore because of it'. It means one thing follows the other, therefore it was caused by the other. But it's not always true. In fact it's hardly ever true."; diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs index e06187b5a3de54..147e3815812d0c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs @@ -168,37 +168,30 @@ internal static unsafe void WriteGatherAtOffset(SafeFileHandle handle, IReadOnly var handles = new MemoryHandle[buffersCount]; Span vectors = buffersCount <= IovStackThreshold ? - stackalloc Interop.Sys.IOVector[IovStackThreshold] : + stackalloc Interop.Sys.IOVector[IovStackThreshold].Slice(0, buffersCount) : new Interop.Sys.IOVector[buffersCount]; try { - int buffersOffset = 0, firstBufferOffset = 0; - while (true) + long totalBytesToWrite = 0; + for (int i = 0; i < buffersCount; i++) { - long totalBytesToWrite = 0; - - for (int i = buffersOffset; i < buffersCount; i++) - { - ReadOnlyMemory buffer = buffers[i]; - totalBytesToWrite += buffer.Length; - - MemoryHandle memoryHandle = buffer.Pin(); - vectors[i] = new Interop.Sys.IOVector { Base = firstBufferOffset + (byte*)memoryHandle.Pointer, Count = (UIntPtr)(buffer.Length - firstBufferOffset) }; - handles[i] = memoryHandle; - - firstBufferOffset = 0; - } + ReadOnlyMemory buffer = buffers[i]; + totalBytesToWrite += buffer.Length; - if (totalBytesToWrite == 0) - { - break; - } + MemoryHandle memoryHandle = buffer.Pin(); + vectors[i] = new Interop.Sys.IOVector { Base = (byte*)memoryHandle.Pointer, Count = (UIntPtr)buffer.Length }; + handles[i] = memoryHandle; + } + int buffersOffset = 0; + while (totalBytesToWrite > 0) + { long bytesWritten; - fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(vectors)) + Span left = vectors.Slice(buffersOffset); + fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(left)) { - bytesWritten = Interop.Sys.PWriteV(handle, pinnedVectors, buffersCount, fileOffset); + bytesWritten = Interop.Sys.PWriteV(handle, pinnedVectors, left.Length, fileOffset); } FileStreamHelpers.CheckFileCall(bytesWritten, handle.Path); @@ -208,22 +201,31 @@ internal static unsafe void WriteGatherAtOffset(SafeFileHandle handle, IReadOnly } // The write completed successfully but for fewer bytes than requested. + // We need to perform next write where the previous one has finished. + fileOffset += bytesWritten; + totalBytesToWrite -= bytesWritten; // We need to try again for the remainder. - for (int i = 0; i < buffersCount; i++) + while (buffersOffset < buffersCount && bytesWritten > 0) { - int n = buffers[i].Length; + int n = (int)vectors[buffersOffset].Count; if (n <= bytesWritten) { - buffersOffset++; bytesWritten -= n; - if (bytesWritten == 0) - { - break; - } + buffersOffset++; } else { - firstBufferOffset = (int)(bytesWritten - n); + // A partial read: the vector needs to point to the new offset. + // But that offset needs to be relative to the previous attempt. + // Example: we have a single buffer with 30 bytes and the first read returned 10. + // The next read should try to read the remaining 20 bytes, but in case it also reads just 10, + // the third attempt should read last 10 bytes (not 20 again). + Interop.Sys.IOVector current = vectors[buffersOffset]; + vectors[buffersOffset] = new Interop.Sys.IOVector + { + Base = current.Base + (int)(bytesWritten), + Count = current.Count - (UIntPtr)(bytesWritten) + }; break; } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index e5c8e17da23e9e..6439cf4b531e59 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -437,7 +437,7 @@ internal static long ReadScatterAtOffset(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset) { // WriteFileGather does not support sync handles, so we just call WriteFile in a loop - int bytesWritten = 0; + long bytesWritten = 0; int buffersCount = buffers.Count; for (int i = 0; i < buffersCount; i++) { diff --git a/src/libraries/System.Runtime.Numerics/src/System/Numerics/BigInteger.cs b/src/libraries/System.Runtime.Numerics/src/System/Numerics/BigInteger.cs index 5f4cf82f75a941..539e1c3175955b 100644 --- a/src/libraries/System.Runtime.Numerics/src/System/Numerics/BigInteger.cs +++ b/src/libraries/System.Runtime.Numerics/src/System/Numerics/BigInteger.cs @@ -1707,7 +1707,7 @@ private static BigInteger Add(ReadOnlySpan leftBits, int leftSign, ReadOnl } if (bitsFromPool != null) - ArrayPool.Shared.Return(bitsFromPool); + ArrayPool.Shared.Return(bitsFromPool); return result; } @@ -2642,7 +2642,7 @@ public static implicit operator BigInteger(nuint value) if (zdFromPool != null) ArrayPool.Shared.Return(zdFromPool); - exit: + exit: if (xdFromPool != null) ArrayPool.Shared.Return(xdFromPool); @@ -3255,7 +3255,27 @@ public static BigInteger PopCount(BigInteger value) public static BigInteger RotateLeft(BigInteger value, int rotateAmount) { value.AssertValid(); - int byteCount = (value._bits is null) ? sizeof(int) : (value._bits.Length * 4); + + bool negx = value._sign < 0; + uint smallBits = NumericsHelpers.Abs(value._sign); + scoped ReadOnlySpan bits = value._bits; + if (bits.IsEmpty) + { + bits = new ReadOnlySpan(in smallBits); + } + + int xl = bits.Length; + if (negx && (bits[^1] >= kuMaskHighBit) && ((bits[^1] != kuMaskHighBit) || bits.IndexOfAnyExcept(0u) != (bits.Length - 1))) + { + // We check for a special case where its sign bit could be outside the uint array after 2's complement conversion. + // For example given [0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF], its 2's complement is [0x01, 0x00, 0x00] + // After a 32 bit right shift, it becomes [0x00, 0x00] which is [0x00, 0x00] when converted back. + // The expected result is [0x00, 0x00, 0xFFFFFFFF] (2's complement) or [0x00, 0x00, 0x01] when converted back + // If the 2's component's last element is a 0, we will track the sign externally + ++xl; + } + + int byteCount = xl * 4; // Normalize the rotate amount to drop full rotations rotateAmount = (int)(rotateAmount % (byteCount * 8L)); @@ -3272,14 +3292,13 @@ public static BigInteger RotateLeft(BigInteger value, int rotateAmount) (int digitShift, int smallShift) = Math.DivRem(rotateAmount, kcbitUint); uint[]? xdFromPool = null; - int xl = value._bits?.Length ?? 1; - Span xd = (xl <= BigIntegerCalculator.StackAllocThreshold) ? stackalloc uint[BigIntegerCalculator.StackAllocThreshold] : xdFromPool = ArrayPool.Shared.Rent(xl); xd = xd.Slice(0, xl); + xd[^1] = 0; - bool negx = value.GetPartsForBitManipulation(xd); + bits.CopyTo(xd); int zl = xl; uint[]? zdFromPool = null; @@ -3390,7 +3409,28 @@ public static BigInteger RotateLeft(BigInteger value, int rotateAmount) public static BigInteger RotateRight(BigInteger value, int rotateAmount) { value.AssertValid(); - int byteCount = (value._bits is null) ? sizeof(int) : (value._bits.Length * 4); + + + bool negx = value._sign < 0; + uint smallBits = NumericsHelpers.Abs(value._sign); + scoped ReadOnlySpan bits = value._bits; + if (bits.IsEmpty) + { + bits = new ReadOnlySpan(in smallBits); + } + + int xl = bits.Length; + if (negx && (bits[^1] >= kuMaskHighBit) && ((bits[^1] != kuMaskHighBit) || bits.IndexOfAnyExcept(0u) != (bits.Length - 1))) + { + // We check for a special case where its sign bit could be outside the uint array after 2's complement conversion. + // For example given [0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF], its 2's complement is [0x01, 0x00, 0x00] + // After a 32 bit right shift, it becomes [0x00, 0x00] which is [0x00, 0x00] when converted back. + // The expected result is [0x00, 0x00, 0xFFFFFFFF] (2's complement) or [0x00, 0x00, 0x01] when converted back + // If the 2's component's last element is a 0, we will track the sign externally + ++xl; + } + + int byteCount = xl * 4; // Normalize the rotate amount to drop full rotations rotateAmount = (int)(rotateAmount % (byteCount * 8L)); @@ -3407,14 +3447,13 @@ public static BigInteger RotateRight(BigInteger value, int rotateAmount) (int digitShift, int smallShift) = Math.DivRem(rotateAmount, kcbitUint); uint[]? xdFromPool = null; - int xl = value._bits?.Length ?? 1; - Span xd = (xl <= BigIntegerCalculator.StackAllocThreshold) ? stackalloc uint[BigIntegerCalculator.StackAllocThreshold] : xdFromPool = ArrayPool.Shared.Rent(xl); xd = xd.Slice(0, xl); + xd[^1] = 0; - bool negx = value.GetPartsForBitManipulation(xd); + bits.CopyTo(xd); int zl = xl; uint[]? zdFromPool = null; @@ -3461,19 +3500,12 @@ public static BigInteger RotateRight(BigInteger value, int rotateAmount) { int carryShift = kcbitUint - smallShift; - int dstIndex = 0; - int srcIndex = digitShift; + int dstIndex = xd.Length - 1; + int srcIndex = digitShift == 0 + ? xd.Length - 1 + : digitShift - 1; - uint carry = 0; - - if (digitShift == 0) - { - carry = xd[^1] << carryShift; - } - else - { - carry = xd[srcIndex - 1] << carryShift; - } + uint carry = xd[digitShift] << carryShift; do { @@ -3482,22 +3514,22 @@ public static BigInteger RotateRight(BigInteger value, int rotateAmount) zd[dstIndex] = (part >> smallShift) | carry; carry = part << carryShift; - dstIndex++; - srcIndex++; + dstIndex--; + srcIndex--; } - while (srcIndex < xd.Length); + while ((uint)srcIndex < (uint)xd.Length); // is equivalent to (srcIndex >= 0 && srcIndex < xd.Length) - srcIndex = 0; + srcIndex = xd.Length - 1; - while (dstIndex < zd.Length) + while ((uint)dstIndex < (uint)zd.Length) // is equivalent to (dstIndex >= 0 && dstIndex < zd.Length) { uint part = xd[srcIndex]; zd[dstIndex] = (part >> smallShift) | carry; carry = part << carryShift; - dstIndex++; - srcIndex++; + dstIndex--; + srcIndex--; } } @@ -5252,13 +5284,32 @@ static bool INumberBase.TryConvertToTruncating(BigInteger va BigInteger result; + bool negx = value._sign < 0; + uint smallBits = NumericsHelpers.Abs(value._sign); + scoped ReadOnlySpan bits = value._bits; + if (bits.IsEmpty) + { + bits = new ReadOnlySpan(in smallBits); + } + + int xl = bits.Length; + if (negx && (bits[^1] >= kuMaskHighBit) && ((bits[^1] != kuMaskHighBit) || bits.IndexOfAnyExcept(0u) != (bits.Length - 1))) + { + // For a shift of N x 32 bit, + // We check for a special case where its sign bit could be outside the uint array after 2's complement conversion. + // For example given [0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF], its 2's complement is [0x01, 0x00, 0x00] + // After a 32 bit right shift, it becomes [0x00, 0x00] which is [0x00, 0x00] when converted back. + // The expected result is [0x00, 0x00, 0xFFFFFFFF] (2's complement) or [0x00, 0x00, 0x01] when converted back + // If the 2's component's last element is a 0, we will track the sign externally + ++xl; + } + uint[]? xdFromPool = null; - int xl = value._bits?.Length ?? 1; Span xd = (xl <= BigIntegerCalculator.StackAllocThreshold ? stackalloc uint[BigIntegerCalculator.StackAllocThreshold] : xdFromPool = ArrayPool.Shared.Rent(xl)).Slice(0, xl); - - bool negx = value.GetPartsForBitManipulation(xd); + xd[^1] = 0; + bits.CopyTo(xd); if (negx) { diff --git a/src/libraries/System.Runtime.Numerics/tests/BigInteger/MyBigInt.cs b/src/libraries/System.Runtime.Numerics/tests/BigInteger/MyBigInt.cs index cd4a578fab910e..8bb69d9f5e25bd 100644 --- a/src/libraries/System.Runtime.Numerics/tests/BigInteger/MyBigInt.cs +++ b/src/libraries/System.Runtime.Numerics/tests/BigInteger/MyBigInt.cs @@ -108,8 +108,14 @@ public static BigInteger DoBinaryOperatorMine(BigInteger num1, BigInteger num2, return new BigInteger(Max(bytes1, bytes2).ToArray()); case "b>>": return new BigInteger(ShiftLeft(bytes1, Negate(bytes2)).ToArray()); + case "b>>>": + return new BigInteger(ShiftRightUnsigned(bytes1, bytes2).ToArray()); case "b<<": return new BigInteger(ShiftLeft(bytes1, bytes2).ToArray()); + case "bRotateLeft": + return new BigInteger(RotateLeft(bytes1, bytes2).ToArray()); + case "bRotateRight": + return new BigInteger(RotateLeft(bytes1, Negate(bytes2)).ToArray()); case "b^": return new BigInteger(Xor(bytes1, bytes2).ToArray()); case "b|": @@ -637,11 +643,68 @@ public static List Not(List bytes) return bnew; } + public static List ShiftRightUnsigned(List bytes1, List bytes2) + { + int byteShift = (int)new BigInteger(Divide(Copy(bytes2), new List(new byte[] { 8 })).ToArray()); + sbyte bitShift = (sbyte)new BigInteger(Remainder(Copy(bytes2), new List(new byte[] { 8 })).ToArray()); + + if (byteShift == 0 && bitShift == 0) + return bytes1; + + if (byteShift < 0 || bitShift < 0) + return ShiftLeft(bytes1, Negate(bytes2)); + + Trim(bytes1); + + byte fill = (bytes1[bytes1.Count - 1] & 0x80) != 0 ? byte.MaxValue : (byte)0; + + if (fill == byte.MaxValue) + { + while (bytes1.Count % 4 != 0) + { + bytes1.Add(fill); + } + } + + if (byteShift >= bytes1.Count) + { + return [fill]; + } + + if (fill == byte.MaxValue) + { + bytes1.Add(0); + } + + for (int i = 0; i < bitShift; i++) + { + bytes1 = ShiftRight(bytes1); + } + + List temp = new List(); + for (int i = byteShift; i < bytes1.Count; i++) + { + temp.Add(bytes1[i]); + } + bytes1 = temp; + + if (fill == byte.MaxValue && bytes1.Count % 4 == 1) + { + bytes1.RemoveAt(bytes1.Count - 1); + } + + Trim(bytes1); + + return bytes1; + } + public static List ShiftLeft(List bytes1, List bytes2) { int byteShift = (int)new BigInteger(Divide(Copy(bytes2), new List(new byte[] { 8 })).ToArray()); sbyte bitShift = (sbyte)new BigInteger(Remainder(bytes2, new List(new byte[] { 8 })).ToArray()); + Trim(bytes1); + for (int i = 0; i < Math.Abs(bitShift); i++) { if (bitShift < 0) @@ -774,6 +837,105 @@ public static List ShiftRight(List bytes) return bresult; } + public static List RotateRight(List bytes) + { + List bresult = new List(); + + byte bottom = (byte)(bytes[0] & 0x01); + + for (int i = 0; i < bytes.Count; i++) + { + byte newbyte = bytes[i]; + + newbyte = (byte)(newbyte / 2); + if ((i != (bytes.Count - 1)) && ((bytes[i + 1] & 0x01) == 1)) + { + newbyte += 128; + } + if ((i == (bytes.Count - 1)) && (bottom != 0)) + { + newbyte += 128; + } + bresult.Add(newbyte); + } + + return bresult; + } + + public static List RotateLeft(List bytes) + { + List bresult = new List(); + + bool prevHead = (bytes[bytes.Count - 1] & 0x80) != 0; + + for (int i = 0; i < bytes.Count; i++) + { + byte newbyte = bytes[i]; + + newbyte = (byte)(newbyte * 2); + if (prevHead) + { + newbyte += 1; + } + + bresult.Add(newbyte); + + prevHead = (bytes[i] & 0x80) != 0; + } + + return bresult; + } + + + public static List RotateLeft(List bytes1, List bytes2) + { + List bytes1Copy = Copy(bytes1); + int byteShift = (int)new BigInteger(Divide(Copy(bytes2), new List(new byte[] { 8 })).ToArray()); + sbyte bitShift = (sbyte)new BigInteger(Remainder(bytes2, new List(new byte[] { 8 })).ToArray()); + + Trim(bytes1); + + byte fill = (bytes1[bytes1.Count - 1] & 0x80) != 0 ? byte.MaxValue : (byte)0; + + if (fill == 0 && bytes1.Count > 1 && bytes1[bytes1.Count - 1] == 0) + bytes1.RemoveAt(bytes1.Count - 1); + + while (bytes1.Count % 4 != 0) + { + bytes1.Add(fill); + } + + byteShift %= bytes1.Count; + if (byteShift == 0 && bitShift == 0) + return bytes1Copy; + + for (int i = 0; i < Math.Abs(bitShift); i++) + { + if (bitShift < 0) + { + bytes1 = RotateRight(bytes1); + } + else + { + bytes1 = RotateLeft(bytes1); + } + } + + List temp = new List(); + for (int i = 0; i < bytes1.Count; i++) + { + temp.Add(bytes1[(i - byteShift + bytes1.Count) % bytes1.Count]); + } + bytes1 = temp; + + if (fill == 0) + bytes1.Add(0); + + Trim(bytes1); + + return bytes1; + } + public static List SetLength(List bytes, int size) { List bresult = new List(); diff --git a/src/libraries/System.Runtime.Numerics/tests/BigInteger/Rotate.cs b/src/libraries/System.Runtime.Numerics/tests/BigInteger/Rotate.cs new file mode 100644 index 00000000000000..564eb23935cda9 --- /dev/null +++ b/src/libraries/System.Runtime.Numerics/tests/BigInteger/Rotate.cs @@ -0,0 +1,629 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Globalization; +using Xunit; + +namespace System.Numerics.Tests +{ + public abstract class RotateTestBase + { + public abstract string opstring { get; } + private static int s_samples = 10; + private static Random s_random = new Random(100); + + [Fact] + public void RunRotateTests() + { + byte[] tempByteArray1; + byte[] tempByteArray2; + + // Rotate Method - Large BigIntegers - large + Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomByteArray(s_random); + tempByteArray2 = GetRandomPosByteArray(s_random, 2); + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - Large BigIntegers - small + Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomByteArray(s_random); + tempByteArray2 = new byte[] { (byte)s_random.Next(1, 32) }; + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - Large BigIntegers - 32 bit Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomByteArray(s_random); + tempByteArray2 = new byte[] { (byte)32 }; + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - All One Uint Large BigIntegers - 32 bit Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomLengthAllOnesUIntByteArray(s_random); + tempByteArray2 = new byte[] { (byte)32 }; + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - Uint 0xffffffff 0x8000000 ... Large BigIntegers - 32 bit Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomLengthFirstUIntMaxSecondUIntMSBMaxArray(s_random); + tempByteArray2 = new byte[] { (byte)32 }; + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - Large BigIntegers - large - Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomByteArray(s_random); + tempByteArray2 = GetRandomNegByteArray(s_random, 2); + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - Large BigIntegers - small - Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomByteArray(s_random); + tempByteArray2 = new byte[] { unchecked((byte)s_random.Next(-31, 0)) }; + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - Large BigIntegers - -32 bit Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomByteArray(s_random); + tempByteArray2 = new byte[] { (byte)0xe0 }; + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - Large BigIntegers - 0 bit Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomByteArray(s_random); + tempByteArray2 = new byte[] { (byte)0 }; + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - Small BigIntegers - large + Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomByteArray(s_random, 2); + tempByteArray2 = GetRandomPosByteArray(s_random, 2); + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - Small BigIntegers - small + Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomByteArray(s_random, 2); + tempByteArray2 = new byte[] { (byte)s_random.Next(1, 32) }; + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - Small BigIntegers - 32 bit Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomByteArray(s_random, 2); + tempByteArray2 = new byte[] { (byte)32 }; + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + // Rotate Method - Small BigIntegers - large - Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomByteArray(s_random, 2); + tempByteArray2 = GetRandomNegByteArray(s_random, 2); + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - Small BigIntegers - small - Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomByteArray(s_random, 2); + tempByteArray2 = new byte[] { unchecked((byte)s_random.Next(-31, 0)) }; + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - Small BigIntegers - -32 bit Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomByteArray(s_random, 2); + tempByteArray2 = new byte[] { (byte)0xe0 }; + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - Small BigIntegers - 0 bit Shift + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomByteArray(s_random, 2); + tempByteArray2 = new byte[] { (byte)0 }; + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - Positive BigIntegers - Shift to 0 + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomPosByteArray(s_random, 100); + tempByteArray2 = BitConverter.GetBytes(s_random.Next(8 * tempByteArray1.Length, 1000)); + if (!BitConverter.IsLittleEndian) + { + Array.Reverse(tempByteArray2); + } + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + + // Rotate Method - Negative BigIntegers - Shift to -1 + for (int i = 0; i < s_samples; i++) + { + tempByteArray1 = GetRandomNegByteArray(s_random, 100); + tempByteArray2 = BitConverter.GetBytes(s_random.Next(8 * tempByteArray1.Length, 1000)); + if (!BitConverter.IsLittleEndian) + { + Array.Reverse(tempByteArray2); + } + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + } + + [Fact] + public void RunSmallTests() + { + foreach (int i in new int[] { + 0, + 1, + 16, + 31, + 32, + 33, + 63, + 64, + 65, + 100, + 127, + 128, + }) + { + foreach (int shift in new int[] { + 0, + -1, 1, + -16, 16, + -31, 31, + -32, 32, + -33, 33, + -63, 63, + -64, 64, + -65, 65, + -100, 100, + -127, 127, + -128, 128, + }) + { + var num = Int128.One << i; + for (int k = -1; k <= 1; k++) + { + foreach (int sign in new int[] { -1, +1 }) + { + Int128 value128 = sign * (num + k); + + byte[] tempByteArray1 = GetRandomSmallByteArray(value128); + byte[] tempByteArray2 = GetRandomSmallByteArray(shift); + + VerifyRotateString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + } + } + } + } + + private static void VerifyRotateString(string opstring) + { + StackCalc sc = new StackCalc(opstring); + while (sc.DoNextOperation()) + { + Assert.Equal(sc.snCalc.Peek().ToString(), sc.myCalc.Peek().ToString()); + } + } + + private static byte[] GetRandomSmallByteArray(Int128 num) + { + byte[] value = new byte[16]; + + for (int i = 0; i < value.Length; i++) + { + value[i] = (byte)num; + num >>= 8; + } + + return value; + } + + private static byte[] GetRandomByteArray(Random random) + { + return GetRandomByteArray(random, random.Next(0, 1024)); + } + + private static byte[] GetRandomByteArray(Random random, int size) + { + return MyBigIntImp.GetRandomByteArray(random, size); + } + + private static byte[] GetRandomPosByteArray(Random random, int size) + { + byte[] value = new byte[size]; + + for (int i = 0; i < value.Length; ++i) + { + value[i] = (byte)random.Next(0, 256); + } + value[value.Length - 1] &= 0x7F; + + return value; + } + + private static byte[] GetRandomNegByteArray(Random random, int size) + { + byte[] value = new byte[size]; + + for (int i = 0; i < value.Length; ++i) + { + value[i] = (byte)random.Next(0, 256); + } + value[value.Length - 1] |= 0x80; + + return value; + } + + private static byte[] GetRandomLengthAllOnesUIntByteArray(Random random) + { + int gap = random.Next(0, 128); + int byteLength = 4 + gap * 4 + 1; + byte[] array = new byte[byteLength]; + array[0] = 1; + array[^1] = 0xFF; + return array; + } + private static byte[] GetRandomLengthFirstUIntMaxSecondUIntMSBMaxArray(Random random) + { + int gap = random.Next(0, 128); + int byteLength = 4 + gap * 4 + 1; + byte[] array = new byte[byteLength]; + array[^5] = 0x80; + array[^1] = 0xFF; + return array; + } + + private static string Print(byte[] bytes) + { + return MyBigIntImp.Print(bytes); + } + } + + public class RotateLeftTest : RotateTestBase + { + public override string opstring => "bRotateLeft"; + + + public static TheoryData NegativeNumber_TestData = new TheoryData + { + + { + new BigInteger(unchecked((long)0xFFFF_FFFF_0000_0000)), + 1, + new BigInteger(unchecked((long)0xFFFF_FFFE_0000_0001)) + }, + { + new BigInteger(unchecked((long)0xFFFF_FFFF_0000_0000)), + 2, + new BigInteger(unchecked((long)0xFFFF_FFFC_0000_0003)) + }, + { + new BigInteger(unchecked((long)0xFFFF_FFFF_0000_0001)), + 1, + new BigInteger(unchecked((long)0xFFFF_FFFE_0000_0003)) + }, + { + new BigInteger(unchecked((long)0xFFFF_FFFF_0000_0001)), + 2, + new BigInteger(unchecked((long)0xFFFF_FFFC_0000_0007)) + }, + { + new BigInteger(unchecked((long)0xFFFF_FFFF_0000_0002)), + 1, + new BigInteger(unchecked((long)0xFFFF_FFFE_0000_0005)) + }, + { + new BigInteger(unchecked((long)0xFFFF_FFFF_0000_0002)), + 2, + new BigInteger(unchecked((long)0xFFFF_FFFC_0000_000B)) + }, + + { + new BigInteger(unchecked((long)0x8000_0000_0000_0000)), + 1, + new BigInteger(0x1) + }, + { + new BigInteger(unchecked((long)0x8000_0000_0000_0000)), + 2, + new BigInteger(0x2) + }, + { + new BigInteger(unchecked((long)0x8000_0000_0000_0001)), + 1, + new BigInteger(0x3) + }, + { + new BigInteger(unchecked((long)0x8000_0000_0000_0001)), + 2, + new BigInteger(0x6) + }, + { + new BigInteger(unchecked((long)0x8000_0000_0000_0002)), + 1, + new BigInteger(0x5) + }, + { + new BigInteger(unchecked((long)0x8000_0000_0000_0002)), + 2, + new BigInteger(0xA) + }, + + { + BigInteger.Parse("8000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber), + 1, + new BigInteger(0x1) + }, + { + BigInteger.Parse("8000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber), + 2, + new BigInteger(0x2) + }, + { + BigInteger.Parse("8000_0000_0000_0000_0000_0001".Replace("_", ""), NumberStyles.HexNumber), + 1, + new BigInteger(0x3) + }, + { + BigInteger.Parse("8000_0000_0000_0000_0000_0001".Replace("_", ""), NumberStyles.HexNumber), + 2, + new BigInteger(0x6) + }, + { + BigInteger.Parse("8000_0000_0000_0000_0000_0002".Replace("_", ""), NumberStyles.HexNumber), + 1, + new BigInteger(0x5) + }, + { + BigInteger.Parse("8000_0000_0000_0000_0000_0002".Replace("_", ""), NumberStyles.HexNumber), + 2, + new BigInteger(0xA) + }, + + { + BigInteger.Parse("________F_0000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber), + 1, + BigInteger.Parse("________E_0000_0000_0000_0000_0000_0001".Replace("_", ""), NumberStyles.HexNumber) + }, + { + BigInteger.Parse("________F_0000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber), + 2, + BigInteger.Parse("________C_0000_0000_0000_0000_0000_0003".Replace("_", ""), NumberStyles.HexNumber) + }, + { + BigInteger.Parse("________F_0000_0000_0000_0000_0000_0001".Replace("_", ""), NumberStyles.HexNumber), + 1, + BigInteger.Parse("________E_0000_0000_0000_0000_0000_0003".Replace("_", ""), NumberStyles.HexNumber) + }, + { + BigInteger.Parse("________F_0000_0000_0000_0000_0000_0001".Replace("_", ""), NumberStyles.HexNumber), + 2, + BigInteger.Parse("________C_0000_0000_0000_0000_0000_0007".Replace("_", ""), NumberStyles.HexNumber) + }, + { + BigInteger.Parse("________F_0000_0000_0000_0000_0000_0002".Replace("_", ""), NumberStyles.HexNumber), + 1, + BigInteger.Parse("________E_0000_0000_0000_0000_0000_0005".Replace("_", ""), NumberStyles.HexNumber) + }, + { + BigInteger.Parse("________F_0000_0000_0000_0000_0000_0002".Replace("_", ""), NumberStyles.HexNumber), + 2, + BigInteger.Parse("________C_0000_0000_0000_0000_0000_000B".Replace("_", ""), NumberStyles.HexNumber) + }, + }; + + [Theory] + [MemberData(nameof(NegativeNumber_TestData))] + public void NegativeNumber(BigInteger input, int rotateAmount, BigInteger expected) + { + Assert.Equal(expected, BigInteger.RotateLeft(input, rotateAmount)); + } + + [Fact] + public void PowerOfTwo() + { + for (int i = 0; i < 32; i++) + { + foreach (int k in new int[] { 1, 2, 3, 10 }) + { + BigInteger plus = BigInteger.One << (32 * k + i); + BigInteger minus = BigInteger.MinusOne << (32 * k + i); + + Assert.Equal(BigInteger.One << (i == 31 ? 0 : (32 * k + i + 1)), BigInteger.RotateLeft(plus, 1)); + Assert.Equal(BigInteger.One << i, BigInteger.RotateLeft(plus, 32)); + Assert.Equal(BigInteger.One << (32 * (k - 1) + i), BigInteger.RotateLeft(plus, 32 * k)); + + Assert.Equal(i == 31 ? BigInteger.One : (new BigInteger(-1 << (i + 1)) << 32 * k) + 1, + BigInteger.RotateLeft(minus, 1)); + Assert.Equal(new BigInteger(uint.MaxValue << i), BigInteger.RotateLeft(minus, 32)); + Assert.Equal(new BigInteger(uint.MaxValue << i) << (32 * (k - 1)), BigInteger.RotateLeft(minus, 32 * k)); + } + } + } + } + + public class RotateRightTest : RotateTestBase + { + public override string opstring => "bRotateRight"; + + public static TheoryData NegativeNumber_TestData = new TheoryData + { + + { + new BigInteger(unchecked((long)0xFFFF_FFFF_0000_0000)), + 1, + new BigInteger(unchecked((long)0x7FFF_FFFF_8000_0000)) + }, + { + new BigInteger(unchecked((long)0xFFFF_FFFF_0000_0000)), + 2, + new BigInteger(unchecked((long)0x3FFF_FFFF_C000_0000)) + }, + { + new BigInteger(unchecked((long)0xFFFF_FFFF_0000_0001)), + 1, + new BigInteger(unchecked((int)0x8000_0000)) + }, + { + new BigInteger(unchecked((long)0xFFFF_FFFF_0000_0001)), + 2, + new BigInteger(unchecked((long)0x7FFF_FFFF_C000_0000)) + }, + { + new BigInteger(unchecked((long)0xFFFF_FFFF_0000_0002)), + 1, + new BigInteger(unchecked((long)0x7FFF_FFFF_8000_0001)) + }, + { + new BigInteger(unchecked((long)0xFFFF_FFFF_0000_0002)), + 2, + new BigInteger(unchecked((long)0xBFFF_FFFF_C000_0000)) + }, + + { + new BigInteger(unchecked((long)0x8000_0000_0000_0000)), + 1, + new BigInteger(unchecked((long)0x4000_0000_0000_0000)) + }, + { + new BigInteger(unchecked((long)0x8000_0000_0000_0000)), + 2, + new BigInteger(unchecked((long)0x2000_0000_0000_0000)) + }, + { + new BigInteger(unchecked((long)0x8000_0000_0000_0001)), + 1, + new BigInteger(unchecked((long)0xC000_0000_0000_0000)) + }, + { + new BigInteger(unchecked((long)0x8000_0000_0000_0001)), + 2, + new BigInteger(unchecked((long)0x6000_0000_0000_0000)) + }, + { + new BigInteger(unchecked((long)0x8000_0000_0000_0002)), + 1, + new BigInteger(unchecked((long)0x4000_0000_0000_0001)) + }, + { + new BigInteger(unchecked((long)0x8000_0000_0000_0002)), + 2, + new BigInteger(unchecked((long)0xA000_0000_0000_0000)) + }, + + { + BigInteger.Parse("8000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber), + 1, + BigInteger.Parse("4000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber) + }, + { + BigInteger.Parse("8000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber), + 2, + BigInteger.Parse("2000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber) + }, + { + BigInteger.Parse("8000_0000_0000_0000_0000_0001".Replace("_", ""), NumberStyles.HexNumber), + 1, + BigInteger.Parse("C000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber) + }, + { + BigInteger.Parse("8000_0000_0000_0000_0000_0001".Replace("_", ""), NumberStyles.HexNumber), + 2, + BigInteger.Parse("6000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber) + }, + { + BigInteger.Parse("8000_0000_0000_0000_0000_0002".Replace("_", ""), NumberStyles.HexNumber), + 1, + BigInteger.Parse("4000_0000_0000_0000_0000_0001".Replace("_", ""), NumberStyles.HexNumber) + }, + { + BigInteger.Parse("8000_0000_0000_0000_0000_0002".Replace("_", ""), NumberStyles.HexNumber), + 2, + BigInteger.Parse("A000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber) + }, + + { + BigInteger.Parse("________F_0000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber), + 1, + BigInteger.Parse("7FFF_FFFF_8000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber) + }, + { + BigInteger.Parse("________F_0000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber), + 2, + BigInteger.Parse("3FFF_FFFF_C000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber) + }, + { + BigInteger.Parse("________F_0000_0000_0000_0000_0000_0001".Replace("_", ""), NumberStyles.HexNumber), + 1, + BigInteger.Parse("__________8000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber) + }, + { + BigInteger.Parse("________F_0000_0000_0000_0000_0000_0001".Replace("_", ""), NumberStyles.HexNumber), + 2, + BigInteger.Parse("7FFF_FFFF_C000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber) + }, + { + BigInteger.Parse("________F_0000_0000_0000_0000_0000_0002".Replace("_", ""), NumberStyles.HexNumber), + 1, + BigInteger.Parse("7FFF_FFFF_8000_0000_0000_0000_0000_0001".Replace("_", ""), NumberStyles.HexNumber) + }, + { + BigInteger.Parse("________F_0000_0000_0000_0000_0000_0002".Replace("_", ""), NumberStyles.HexNumber), + 2, + BigInteger.Parse("BFFF_FFFF_C000_0000_0000_0000_0000_0000".Replace("_", ""), NumberStyles.HexNumber) + }, + }; + + [Theory] + [MemberData(nameof(NegativeNumber_TestData))] + public void NegativeNumber(BigInteger input, int rotateAmount, BigInteger expected) + { + Assert.Equal(expected, BigInteger.RotateRight(input, rotateAmount)); + } + + [Fact] + public void PowerOfTwo() + { + for (int i = 0; i < 32; i++) + { + foreach (int k in new int[] { 1, 2, 3, 10 }) + { + BigInteger plus = BigInteger.One << (32 * k + i); + BigInteger minus = BigInteger.MinusOne << (32 * k + i); + + Assert.Equal(BigInteger.One << (32 * k + i - 1), BigInteger.RotateRight(plus, 1)); + Assert.Equal(BigInteger.One << (32 * (k - 1) + i), BigInteger.RotateRight(plus, 32)); + Assert.Equal(BigInteger.One << i, BigInteger.RotateRight(plus, 32 * k)); + + Assert.Equal(new BigInteger(uint.MaxValue << i) << (32 * k - 1), BigInteger.RotateRight(minus, 1)); + Assert.Equal(new BigInteger(uint.MaxValue << i) << (32 * (k - 1)), BigInteger.RotateRight(minus, 32)); + Assert.Equal(new BigInteger(uint.MaxValue << i), BigInteger.RotateRight(minus, 32 * k)); + } + } + } + } +} diff --git a/src/libraries/System.Runtime.Numerics/tests/BigInteger/op_leftshift.cs b/src/libraries/System.Runtime.Numerics/tests/BigInteger/op_leftshift.cs index 47b0a8ea7678cd..6c278bb63edb6e 100644 --- a/src/libraries/System.Runtime.Numerics/tests/BigInteger/op_leftshift.cs +++ b/src/libraries/System.Runtime.Numerics/tests/BigInteger/op_leftshift.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using Xunit; namespace System.Numerics.Tests @@ -151,6 +152,56 @@ public static void RunLeftShiftTests() } } + [Fact] + public void RunSmallTests() + { + foreach (int i in new int[] { + 0, + 1, + 16, + 31, + 32, + 33, + 63, + 64, + 65, + 100, + 127, + 128, + }) + { + foreach (int shift in new int[] { + 0, + -1, 1, + -16, 16, + -31, 31, + -32, 32, + -33, 33, + -63, 63, + -64, 64, + -65, 65, + -100, 100, + -127, 127, + -128, 128, + }) + { + var num = Int128.One << i; + for (int k = -1; k <= 1; k++) + { + foreach (int sign in new int[] { -1, +1 }) + { + Int128 value128 = sign * (num + k); + + byte[] tempByteArray1 = GetRandomSmallByteArray(value128); + byte[] tempByteArray2 = GetRandomSmallByteArray(shift); + + VerifyLeftShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b<<"); + } + } + } + } + } + private static void VerifyLeftShiftString(string opstring) { StackCalc sc = new StackCalc(opstring); @@ -160,6 +211,19 @@ private static void VerifyLeftShiftString(string opstring) } } + private static byte[] GetRandomSmallByteArray(Int128 num) + { + byte[] value = new byte[16]; + + for (int i = 0; i < value.Length; i++) + { + value[i] = (byte)num; + num >>= 8; + } + + return value; + } + private static byte[] GetRandomByteArray(Random random) { return GetRandomByteArray(random, random.Next(0, 1024)); diff --git a/src/libraries/System.Runtime.Numerics/tests/BigInteger/op_rightshift.cs b/src/libraries/System.Runtime.Numerics/tests/BigInteger/op_rightshift.cs index da6ac2615b9c1b..c725d52bc0ba4c 100644 --- a/src/libraries/System.Runtime.Numerics/tests/BigInteger/op_rightshift.cs +++ b/src/libraries/System.Runtime.Numerics/tests/BigInteger/op_rightshift.cs @@ -1,90 +1,29 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using Xunit; namespace System.Numerics.Tests { - public class op_rightshiftTest + public abstract class op_rightshiftTestBase { + public abstract string opstring { get; } private static int s_samples = 10; private static Random s_random = new Random(100); [Fact] - public static void BigShiftsTest() - { - BigInteger a = new BigInteger(1); - BigInteger b = new BigInteger(Math.Pow(2, 31)); - - for (int i = 0; i < 100; i++) - { - BigInteger a1 = (a << (i + 31)); - BigInteger a2 = a1 >> i; - - Assert.Equal(b, a2); - } - } - - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.Is64BitProcess))] // May fail on 32-bit due to a large memory requirement - public static void LargeNegativeBigIntegerShiftTest() - { - // Create a very large negative BigInteger - BigInteger bigInt = new BigInteger(-1) << int.MaxValue; - Assert.Equal(2147483647, bigInt.GetBitLength()); - Assert.Equal(-1, bigInt.Sign); - - // Validate internal representation. - // At this point, bigInt should be a 1 followed by int.MaxValue zeros. - // Given this, bigInt._bits is expected to be structured as follows: - // - _bits.Length == (int.MaxValue + 1) / (8 * sizeof(uint)) - // - First (_bits.Length - 1) elements: 0x00000000 - // - Last element: 0x80000000 - // ^------ (There's the leading '1') - - Assert.Equal(((uint)int.MaxValue + 1) / (8 * sizeof(uint)), (uint)bigInt._bits.Length); - - uint i = 0; - for (; i < (bigInt._bits.Length - 1); i++) { - Assert.Equal(0x00000000u, bigInt._bits[i]); - } - - Assert.Equal(0x80000000u, bigInt._bits[i]); - - // Right shift the BigInteger - BigInteger shiftedBigInt = bigInt >> 1; - Assert.Equal(2147483646, shiftedBigInt.GetBitLength()); - Assert.Equal(-1, shiftedBigInt.Sign); - - // Validate internal representation. - // At this point, shiftedBigInt should be a 1 followed by int.MaxValue - 1 zeros. - // Given this, shiftedBigInt._bits is expected to be structured as follows: - // - _bits.Length == (int.MaxValue + 1) / (8 * sizeof(uint)) - // - First (_bits.Length - 1) elements: 0x00000000 - // - Last element: 0x40000000 - // ^------ (the '1' is now one position to the right) - - Assert.Equal(((uint)int.MaxValue + 1) / (8 * sizeof(uint)), (uint)shiftedBigInt._bits.Length); - - i = 0; - for (; i < (shiftedBigInt._bits.Length - 1); i++) { - Assert.Equal(0x00000000u, shiftedBigInt._bits[i]); - } - - Assert.Equal(0x40000000u, shiftedBigInt._bits[i]); - } - - [Fact] - public static void RunRightShiftTests() + public void RunRightShiftTests() { - byte[] tempByteArray1 = new byte[0]; - byte[] tempByteArray2 = new byte[0]; + byte[] tempByteArray1; + byte[] tempByteArray2; // RightShift Method - Large BigIntegers - large + Shift for (int i = 0; i < s_samples; i++) { tempByteArray1 = GetRandomByteArray(s_random); tempByteArray2 = GetRandomPosByteArray(s_random, 2); - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Large BigIntegers - small + Shift @@ -92,7 +31,7 @@ public static void RunRightShiftTests() { tempByteArray1 = GetRandomByteArray(s_random); tempByteArray2 = new byte[] { (byte)s_random.Next(1, 32) }; - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Large BigIntegers - 32 bit Shift @@ -100,7 +39,7 @@ public static void RunRightShiftTests() { tempByteArray1 = GetRandomByteArray(s_random); tempByteArray2 = new byte[] { (byte)32 }; - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - All One Uint Large BigIntegers - 32 bit Shift @@ -108,7 +47,7 @@ public static void RunRightShiftTests() { tempByteArray1 = GetRandomLengthAllOnesUIntByteArray(s_random); tempByteArray2 = new byte[] { (byte)32 }; - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Uint 0xffffffff 0x8000000 ... Large BigIntegers - 32 bit Shift @@ -116,7 +55,7 @@ public static void RunRightShiftTests() { tempByteArray1 = GetRandomLengthFirstUIntMaxSecondUIntMSBMaxArray(s_random); tempByteArray2 = new byte[] { (byte)32 }; - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Large BigIntegers - large - Shift @@ -124,7 +63,7 @@ public static void RunRightShiftTests() { tempByteArray1 = GetRandomByteArray(s_random); tempByteArray2 = GetRandomNegByteArray(s_random, 2); - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Large BigIntegers - small - Shift @@ -132,7 +71,7 @@ public static void RunRightShiftTests() { tempByteArray1 = GetRandomByteArray(s_random); tempByteArray2 = new byte[] { unchecked((byte)s_random.Next(-31, 0)) }; - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Large BigIntegers - -32 bit Shift @@ -140,7 +79,7 @@ public static void RunRightShiftTests() { tempByteArray1 = GetRandomByteArray(s_random); tempByteArray2 = new byte[] { (byte)0xe0 }; - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Large BigIntegers - 0 bit Shift @@ -148,7 +87,7 @@ public static void RunRightShiftTests() { tempByteArray1 = GetRandomByteArray(s_random); tempByteArray2 = new byte[] { (byte)0 }; - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Small BigIntegers - large + Shift @@ -156,7 +95,7 @@ public static void RunRightShiftTests() { tempByteArray1 = GetRandomByteArray(s_random, 2); tempByteArray2 = GetRandomPosByteArray(s_random, 2); - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Small BigIntegers - small + Shift @@ -164,7 +103,7 @@ public static void RunRightShiftTests() { tempByteArray1 = GetRandomByteArray(s_random, 2); tempByteArray2 = new byte[] { (byte)s_random.Next(1, 32) }; - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Small BigIntegers - 32 bit Shift @@ -172,14 +111,14 @@ public static void RunRightShiftTests() { tempByteArray1 = GetRandomByteArray(s_random, 2); tempByteArray2 = new byte[] { (byte)32 }; - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Small BigIntegers - large - Shift for (int i = 0; i < s_samples; i++) { tempByteArray1 = GetRandomByteArray(s_random, 2); tempByteArray2 = GetRandomNegByteArray(s_random, 2); - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Small BigIntegers - small - Shift @@ -187,7 +126,7 @@ public static void RunRightShiftTests() { tempByteArray1 = GetRandomByteArray(s_random, 2); tempByteArray2 = new byte[] { unchecked((byte)s_random.Next(-31, 0)) }; - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Small BigIntegers - -32 bit Shift @@ -195,7 +134,7 @@ public static void RunRightShiftTests() { tempByteArray1 = GetRandomByteArray(s_random, 2); tempByteArray2 = new byte[] { (byte)0xe0 }; - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Small BigIntegers - 0 bit Shift @@ -203,7 +142,7 @@ public static void RunRightShiftTests() { tempByteArray1 = GetRandomByteArray(s_random, 2); tempByteArray2 = new byte[] { (byte)0 }; - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Positive BigIntegers - Shift to 0 @@ -215,7 +154,7 @@ public static void RunRightShiftTests() { Array.Reverse(tempByteArray2); } - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); } // RightShift Method - Negative BigIntegers - Shift to -1 @@ -227,7 +166,57 @@ public static void RunRightShiftTests() { Array.Reverse(tempByteArray2); } - VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + "b>>"); + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + } + + [Fact] + public void RunSmallTests() + { + foreach (int i in new int[] { + 0, + 1, + 16, + 31, + 32, + 33, + 63, + 64, + 65, + 100, + 127, + 128, + }) + { + foreach (int shift in new int[] { + 0, + -1, 1, + -16, 16, + -31, 31, + -32, 32, + -33, 33, + -63, 63, + -64, 64, + -65, 65, + -100, 100, + -127, 127, + -128, 128, + }) + { + var num = Int128.One << i; + for (int k = -1; k <= 1; k++) + { + foreach (int sign in new int[] { -1, +1 }) + { + Int128 value128 = sign * (num + k); + + byte[] tempByteArray1 = GetRandomSmallByteArray(value128); + byte[] tempByteArray2 = GetRandomSmallByteArray(shift); + + VerifyRightShiftString(Print(tempByteArray2) + Print(tempByteArray1) + opstring); + } + } + } } } @@ -240,6 +229,19 @@ private static void VerifyRightShiftString(string opstring) } } + private static byte[] GetRandomSmallByteArray(Int128 num) + { + byte[] value = new byte[16]; + + for (int i = 0; i < value.Length; i++) + { + value[i] = (byte)num; + num >>= 8; + } + + return value; + } + private static byte[] GetRandomByteArray(Random random) { return GetRandomByteArray(random, random.Next(0, 1024)); @@ -290,7 +292,7 @@ private static byte[] GetRandomLengthFirstUIntMaxSecondUIntMSBMaxArray(Random ra int gap = random.Next(0, 128); int byteLength = 4 + gap * 4 + 1; byte[] array = new byte[byteLength]; - array[^6] = 0x80; + array[^5] = 0x80; array[^1] = 0xFF; return array; } @@ -300,4 +302,93 @@ private static string Print(byte[] bytes) return MyBigIntImp.Print(bytes); } } + public class op_rightshiftTest : op_rightshiftTestBase + { + public override string opstring => "b>>"; + + [Fact] + public static void BigShiftsTest() + { + BigInteger a = new BigInteger(1); + BigInteger b = new BigInteger(Math.Pow(2, 31)); + + for (int i = 0; i < 100; i++) + { + BigInteger a1 = (a << (i + 31)); + BigInteger a2 = a1 >> i; + + Assert.Equal(b, a2); + } + } + + [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.Is64BitProcess))] // May fail on 32-bit due to a large memory requirement + public static void LargeNegativeBigIntegerShiftTest() + { + // Create a very large negative BigInteger + int bitsPerElement = 8 * sizeof(uint); + int maxBitLength = ((Array.MaxLength / bitsPerElement) * bitsPerElement); + BigInteger bigInt = new BigInteger(-1) << (maxBitLength - 1); + Assert.Equal(maxBitLength - 1, bigInt.GetBitLength()); + Assert.Equal(-1, bigInt.Sign); + + // Validate internal representation. + // At this point, bigInt should be a 1 followed by maxBitLength - 1 zeros. + // Given this, bigInt._bits is expected to be structured as follows: + // - _bits.Length == ceil(maxBitLength / bitsPerElement) + // - First (_bits.Length - 1) elements: 0x00000000 + // - Last element: 0x80000000 + // ^------ (There's the leading '1') + + Assert.Equal((maxBitLength + (bitsPerElement - 1)) / bitsPerElement, bigInt._bits.Length); + + uint i = 0; + for (; i < (bigInt._bits.Length - 1); i++) + { + Assert.Equal(0x00000000u, bigInt._bits[i]); + } + + Assert.Equal(0x80000000u, bigInt._bits[i]); + + // Right shift the BigInteger + BigInteger shiftedBigInt = bigInt >> 1; + Assert.Equal(maxBitLength - 2, shiftedBigInt.GetBitLength()); + Assert.Equal(-1, shiftedBigInt.Sign); + + // Validate internal representation. + // At this point, shiftedBigInt should be a 1 followed by maxBitLength - 2 zeros. + // Given this, shiftedBigInt._bits is expected to be structured as follows: + // - _bits.Length == ceil((maxBitLength - 1) / bitsPerElement) + // - First (_bits.Length - 1) elements: 0x00000000 + // - Last element: 0x40000000 + // ^------ (the '1' is now one position to the right) + + Assert.Equal(((maxBitLength - 1) + (bitsPerElement - 1)) / bitsPerElement, shiftedBigInt._bits.Length); + + i = 0; + for (; i < (shiftedBigInt._bits.Length - 1); i++) + { + Assert.Equal(0x00000000u, shiftedBigInt._bits[i]); + } + + Assert.Equal(0x40000000u, shiftedBigInt._bits[i]); + } + } + + public class op_UnsignedRightshiftTest : op_rightshiftTestBase + { + public override string opstring => "b>>>"; + + [Fact] + public void PowerOfTwo() + { + for (int i = 0; i < 32; i++) + { + foreach (int k in new int[] { 1, 2, 10 }) + { + Assert.Equal(BigInteger.One << i, (BigInteger.One << (32 * k + i)) >>> (32 * k)); + Assert.Equal(new BigInteger(unchecked((int)(uint.MaxValue << i))), (BigInteger.MinusOne << (32 * k + i)) >>> (32 * k)); + } + } + } + } } diff --git a/src/libraries/System.Runtime.Numerics/tests/BigInteger/stackcalculator.cs b/src/libraries/System.Runtime.Numerics/tests/BigInteger/stackcalculator.cs index c53e87a15a234f..cbd2a04127ef7f 100644 --- a/src/libraries/System.Runtime.Numerics/tests/BigInteger/stackcalculator.cs +++ b/src/libraries/System.Runtime.Numerics/tests/BigInteger/stackcalculator.cs @@ -191,8 +191,14 @@ private BigInteger DoBinaryOperatorSN(BigInteger num1, BigInteger num2, string o return BigInteger.Max(num1, num2); case "b>>": return num1 >> (int)num2; + case "b>>>": + return num1 >>> (int)num2; case "b<<": return num1 << (int)num2; + case "bRotateLeft": + return BigInteger.RotateLeft(num1, (int)num2); + case "bRotateRight": + return BigInteger.RotateRight(num1, (int)num2); case "b^": return num1 ^ num2; case "b|": @@ -254,7 +260,7 @@ public void VerifyOutParameter() private static string Print(byte[] bytes) { - return MyBigIntImp.PrintFormatX(bytes); + return MyBigIntImp.PrintFormatX(bytes); } } } diff --git a/src/libraries/System.Runtime.Numerics/tests/System.Runtime.Numerics.Tests.csproj b/src/libraries/System.Runtime.Numerics/tests/System.Runtime.Numerics.Tests.csproj index 0716f70c88e651..49ded253d98507 100644 --- a/src/libraries/System.Runtime.Numerics/tests/System.Runtime.Numerics.Tests.csproj +++ b/src/libraries/System.Runtime.Numerics/tests/System.Runtime.Numerics.Tests.csproj @@ -43,6 +43,7 @@ + diff --git a/src/libraries/System.Security.Cryptography/tests/X509Certificates/PfxIterationCountTests.X509Certificate2.cs b/src/libraries/System.Security.Cryptography/tests/X509Certificates/PfxIterationCountTests.X509Certificate2.cs index ce5c3be483da75..e997796b4c99cd 100644 --- a/src/libraries/System.Security.Cryptography/tests/X509Certificates/PfxIterationCountTests.X509Certificate2.cs +++ b/src/libraries/System.Security.Cryptography/tests/X509Certificates/PfxIterationCountTests.X509Certificate2.cs @@ -30,9 +30,10 @@ internal override X509Certificate Import(string fileName, SecureString password) [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] + [OuterLoop("Sensitive to timeouts and may fail in very busy environments.")] public static void Import_IterationCountLimitExceeded_ThrowsInAllottedTime() { - const int AllottedTime = 5000; + const int AllottedTime = 10_000; if (!PfxTests.Pkcs12PBES2Supported) { diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index 72aa247909efd2..8dfd49d489a134 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -8,7 +8,7 @@ CS8969 true true - true + false 6 Provides high-performance and low-allocating types that serialize objects to JavaScript Object Notation (JSON) text and deserialize JSON text to objects, with UTF-8 support built-in. Also provides types to read and write JSON text encoded as UTF-8, and to create an in-memory document object model (DOM), that is read-only, for random access of the JSON elements within a structured view of the data. diff --git a/src/native/corehost/apphost/standalone/hostfxr_resolver.cpp b/src/native/corehost/apphost/standalone/hostfxr_resolver.cpp index d06be0719551f5..5befe7ee7d37a1 100644 --- a/src/native/corehost/apphost/standalone/hostfxr_resolver.cpp +++ b/src/native/corehost/apphost/standalone/hostfxr_resolver.cpp @@ -38,7 +38,7 @@ hostfxr_resolver_t::hostfxr_resolver_t(const pal::string_t& app_root) { m_status_code = StatusCode::CoreHostLibMissingFailure; } - else if (!pal::is_path_rooted(m_fxr_path)) + else if (!pal::is_path_fully_qualified(m_fxr_path)) { // We should always be loading hostfxr from an absolute path m_status_code = StatusCode::CoreHostLibMissingFailure; diff --git a/src/native/corehost/corehost.cpp b/src/native/corehost/corehost.cpp index 5edc2fbf5d5219..2abc1bf0fa46b3 100644 --- a/src/native/corehost/corehost.cpp +++ b/src/native/corehost/corehost.cpp @@ -187,6 +187,7 @@ int exe_start(const int argc, const pal::char_t* argv[]) int rc = fxr.status_code(); if (rc != StatusCode::Success) { + trace::error(_X("Failed to resolve %s [%s]. Error code: 0x%x"), LIBFXR_NAME, fxr.fxr_path().empty() ? _X("not found") : fxr.fxr_path().c_str(), rc); return rc; } diff --git a/src/native/corehost/fxr/standalone/hostpolicy_resolver.cpp b/src/native/corehost/fxr/standalone/hostpolicy_resolver.cpp index bb0da3238f42b6..4ec39be9b1676e 100644 --- a/src/native/corehost/fxr/standalone/hostpolicy_resolver.cpp +++ b/src/native/corehost/fxr/standalone/hostpolicy_resolver.cpp @@ -181,7 +181,7 @@ int hostpolicy_resolver::load( } // We should always be loading hostpolicy from an absolute path - if (!pal::is_path_rooted(host_path)) + if (!pal::is_path_fully_qualified(host_path)) return StatusCode::CoreHostLibMissingFailure; // Load library diff --git a/src/native/corehost/fxr_resolver.h b/src/native/corehost/fxr_resolver.h index ecbf37c12b1ffc..9061939728d510 100644 --- a/src/native/corehost/fxr_resolver.h +++ b/src/native/corehost/fxr_resolver.h @@ -45,7 +45,7 @@ int load_fxr_and_get_delegate(hostfxr_delegate_type type, THostPathToConfigCallb } // We should always be loading hostfxr from an absolute path - if (!pal::is_path_rooted(fxr_path)) + if (!pal::is_path_fully_qualified(fxr_path)) return StatusCode::CoreHostLibMissingFailure; // Load library diff --git a/src/native/corehost/hostmisc/pal.h b/src/native/corehost/hostmisc/pal.h index aa050d1aeb8526..cb01462d090e29 100644 --- a/src/native/corehost/hostmisc/pal.h +++ b/src/native/corehost/hostmisc/pal.h @@ -328,6 +328,7 @@ namespace pal bool get_default_breadcrumb_store(string_t* recv); bool is_path_rooted(const string_t& path); + bool is_path_fully_qualified(const string_t& path); // Returns a platform-specific, user-private directory // that can be used for extracting out components of a single-file app. diff --git a/src/native/corehost/hostmisc/pal.unix.cpp b/src/native/corehost/hostmisc/pal.unix.cpp index 34520aefd7365a..7fe2e83327b1b1 100644 --- a/src/native/corehost/hostmisc/pal.unix.cpp +++ b/src/native/corehost/hostmisc/pal.unix.cpp @@ -192,7 +192,7 @@ bool pal::get_loaded_library( { pal::string_t library_name_local; #if defined(TARGET_OSX) - if (!pal::is_path_rooted(library_name)) + if (!pal::is_path_fully_qualified(library_name)) library_name_local.append("@rpath/"); #endif library_name_local.append(library_name); @@ -200,7 +200,7 @@ bool pal::get_loaded_library( dll_t dll_maybe = dlopen(library_name_local.c_str(), RTLD_LAZY | RTLD_NOLOAD); if (dll_maybe == nullptr) { - if (pal::is_path_rooted(library_name)) + if (pal::is_path_fully_qualified(library_name)) return false; // dlopen on some systems only finds loaded libraries when given the full path @@ -265,6 +265,11 @@ bool pal::is_path_rooted(const pal::string_t& path) return path.front() == '/'; } +bool pal::is_path_fully_qualified(const pal::string_t& path) +{ + return is_path_rooted(path); +} + bool pal::get_default_breadcrumb_store(string_t* recv) { recv->clear(); diff --git a/src/native/corehost/hostmisc/pal.windows.cpp b/src/native/corehost/hostmisc/pal.windows.cpp index ca0608cd56935e..339c7795a7fb02 100644 --- a/src/native/corehost/hostmisc/pal.windows.cpp +++ b/src/native/corehost/hostmisc/pal.windows.cpp @@ -585,9 +585,31 @@ pal::string_t pal::get_current_os_rid_platform() return ridOS; } +namespace +{ + bool is_directory_separator(pal::char_t c) + { + return c == DIR_SEPARATOR || c == L'/'; + } +} + bool pal::is_path_rooted(const string_t& path) { - return path.length() >= 2 && path[1] == L':'; + return (path.length() >= 1 && is_directory_separator(path[0])) // UNC or device paths + || (path.length() >= 2 && path[1] == L':'); // Drive letter paths +} + +bool pal::is_path_fully_qualified(const string_t& path) +{ + if (path.length() < 2) + return false; + + // Check for UNC and DOS device paths + if (is_directory_separator(path[0])) + return path[1] == L'?' || is_directory_separator(path[1]); + + // Check for drive absolute path - for example C:\. + return path.length() >= 3 && path[1] == L':' && is_directory_separator(path[2]); } // Returns true only if an env variable can be read successfully to be non-empty. diff --git a/src/native/corehost/hostpolicy/deps_resolver.cpp b/src/native/corehost/hostpolicy/deps_resolver.cpp index a00f4dabc71545..440e7cd98c59ff 100644 --- a/src/native/corehost/hostpolicy/deps_resolver.cpp +++ b/src/native/corehost/hostpolicy/deps_resolver.cpp @@ -601,7 +601,7 @@ void deps_resolver_t::init_known_entry_path(const deps_entry_t& entry, const pal return; } - assert(pal::is_path_rooted(path)); + assert(pal::is_path_fully_qualified(path)); if (m_coreclr_path.empty() && ends_with(path, DIR_SEPARATOR + pal::string_t(LIBCORECLR_NAME), false)) { m_coreclr_path = path; diff --git a/src/native/corehost/hostpolicy/standalone/coreclr_resolver.cpp b/src/native/corehost/hostpolicy/standalone/coreclr_resolver.cpp index 8df8e395e3f259..adcebeb9a029c5 100644 --- a/src/native/corehost/hostpolicy/standalone/coreclr_resolver.cpp +++ b/src/native/corehost/hostpolicy/standalone/coreclr_resolver.cpp @@ -14,7 +14,7 @@ bool coreclr_resolver_t::resolve_coreclr(const pal::string_t& libcoreclr_path, c append_path(&coreclr_dll_path, LIBCORECLR_NAME); // We should always be loading coreclr from an absolute path - if (!pal::is_path_rooted(coreclr_dll_path)) + if (!pal::is_path_fully_qualified(coreclr_dll_path)) return false; if (!pal::load_library(&coreclr_dll_path, &coreclr_resolver_contract.coreclr)) diff --git a/src/native/external/zlib-intel/inflate.c b/src/native/external/zlib-intel/inflate.c index 96306ef76570b7..47e3a25e4672b7 100644 --- a/src/native/external/zlib-intel/inflate.c +++ b/src/native/external/zlib-intel/inflate.c @@ -251,7 +251,7 @@ int stream_size; if (state->window == Z_NULL) { ZFREE(strm, state); strm->state = Z_NULL; - ret = Z_MEM_ERROR; + return Z_MEM_ERROR; } } state->whave = 0; @@ -727,7 +727,7 @@ int flush; if (state->window == Z_NULL) { ZFREE(strm, state); strm->state = Z_NULL; - ret = Z_MEM_ERROR; + return Z_MEM_ERROR; } LOAD(); } diff --git a/src/native/libs/System.Native/pal_io.c b/src/native/libs/System.Native/pal_io.c index d6d4a84e1faeb2..35858794dff1af 100644 --- a/src/native/libs/System.Native/pal_io.c +++ b/src/native/libs/System.Native/pal_io.c @@ -1855,6 +1855,53 @@ int32_t SystemNative_PWrite(intptr_t fd, void* buffer, int32_t bufferSize, int64 return (int32_t)count; } +#if (HAVE_PREADV || HAVE_PWRITEV) && !defined(TARGET_WASM) +static int GetAllowedVectorCount(IOVector* vectors, int32_t vectorCount) +{ +#if defined(IOV_MAX) + const int IovMax = IOV_MAX; +#else + // In theory all the platforms that we support define IOV_MAX, + // but we want to be extra safe and provde a fallback + // in case it turns out to not be true. + // 16 is low, but supported on every platform. + const int IovMax = 16; +#endif + + int allowedCount = (int)vectorCount; + + // We need to respect the limit of items that can be passed in iov. + // In case of writes, the managed code is responsible for handling incomplete writes. + // In case of reads, we simply returns the number of bytes read and it's up to the users. + if (IovMax < allowedCount) + { + allowedCount = IovMax; + } + +#if defined(TARGET_APPLE) + // For macOS preadv and pwritev can fail with EINVAL when the total length + // of all vectors overflows a 32-bit integer. + size_t totalLength = 0; + for (int i = 0; i < allowedCount; i++) + { + assert(INT_MAX >= vectors[i].Count); + + totalLength += vectors[i].Count; + + if (totalLength > INT_MAX) + { + allowedCount = i; + break; + } + } +#else + (void)vectors; +#endif + + return allowedCount; +} +#endif // (HAVE_PREADV || HAVE_PWRITEV) && !defined(TARGET_WASM) + int64_t SystemNative_PReadV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t fileOffset) { assert(vectors != NULL); @@ -1863,7 +1910,8 @@ int64_t SystemNative_PReadV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t count = 0; int fileDescriptor = ToFileDescriptor(fd); #if HAVE_PREADV && !defined(TARGET_WASM) // preadv is buggy on WASM - while ((count = preadv(fileDescriptor, (struct iovec*)vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); + int allowedVectorCount = GetAllowedVectorCount(vectors, vectorCount); + while ((count = preadv(fileDescriptor, (struct iovec*)vectors, allowedVectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); #else int64_t current; for (int i = 0; i < vectorCount; i++) @@ -1903,7 +1951,8 @@ int64_t SystemNative_PWriteV(intptr_t fd, IOVector* vectors, int32_t vectorCount int64_t count = 0; int fileDescriptor = ToFileDescriptor(fd); #if HAVE_PWRITEV && !defined(TARGET_WASM) // pwritev is buggy on WASM - while ((count = pwritev(fileDescriptor, (struct iovec*)vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); + int allowedVectorCount = GetAllowedVectorCount(vectors, vectorCount); + while ((count = pwritev(fileDescriptor, (struct iovec*)vectors, allowedVectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); #else int64_t current; for (int i = 0; i < vectorCount; i++)