From 9e200046edebf46e971527c935fe361b16f1499c Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 13 Apr 2021 19:59:15 +0200 Subject: [PATCH 01/11] initial Unix implementation --- .../Native/Unix/Common/pal_config.h.in | 2 ++ .../Native/Unix/System.Native/entrypoints.c | 1 + .../Native/Unix/System.Native/pal_io.c | 20 +++++++++++++++++-- .../Native/Unix/System.Native/pal_io.h | 7 +++++++ src/libraries/Native/Unix/configure.cmake | 10 ++++++++++ .../System.Private.CoreLib.Shared.projitems | 5 ++++- .../IO/Strategies/FileStreamHelpers.Unix.cs | 17 +++++++++++++++- 7 files changed, 58 insertions(+), 4 deletions(-) diff --git a/src/libraries/Native/Unix/Common/pal_config.h.in b/src/libraries/Native/Unix/Common/pal_config.h.in index 67158f915a7a48..b7bd484e629ffa 100644 --- a/src/libraries/Native/Unix/Common/pal_config.h.in +++ b/src/libraries/Native/Unix/Common/pal_config.h.in @@ -34,6 +34,8 @@ #cmakedefine01 HAVE_STRLCAT #cmakedefine01 HAVE_SHM_OPEN_THAT_WORKS_WELL_ENOUGH_WITH_MMAP #cmakedefine01 HAVE_POSIX_ADVISE +#cmakedefine01 HAVE_POSIX_FALLOCATE +#cmakedefine01 HAVE_POSIX_FALLOCATE64 #cmakedefine01 PRIORITY_REQUIRES_INT_WHO #cmakedefine01 KEVENT_REQUIRES_INT_PARAMS #cmakedefine01 HAVE_IOCTL diff --git a/src/libraries/Native/Unix/System.Native/entrypoints.c b/src/libraries/Native/Unix/System.Native/entrypoints.c index 5abcf2a91074dd..02ffd1916d6f95 100644 --- a/src/libraries/Native/Unix/System.Native/entrypoints.c +++ b/src/libraries/Native/Unix/System.Native/entrypoints.c @@ -89,6 +89,7 @@ static const Entry s_sysNative[] = DllImportEntry(SystemNative_FTruncate) DllImportEntry(SystemNative_Poll) DllImportEntry(SystemNative_PosixFAdvise) + DllImportEntry(SystemNative_PosixFAllocate) DllImportEntry(SystemNative_Read) DllImportEntry(SystemNative_ReadLink) DllImportEntry(SystemNative_Rename) diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index 96b69525471987..7368e020438b69 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -662,7 +662,7 @@ int32_t SystemNative_FSync(intptr_t fd) int fileDescriptor = ToFileDescriptor(fd); int32_t result; - while ((result = + while ((result = #if defined(TARGET_OSX) && HAVE_F_FULLFSYNC fcntl(fileDescriptor, F_FULLFSYNC) #else @@ -991,6 +991,22 @@ int32_t SystemNative_PosixFAdvise(intptr_t fd, int64_t offset, int64_t length, i #endif } +int32_t SystemNative_PosixFAllocate(intptr_t fd, int64_t offset, int64_t length) +{ + int32_t result; +#if HAVE_POSIX_FALLOCATE64 + while ((result = posix_fallocate64(ToFileDescriptor(fd), (off64_t)offset, (off64_t)length)) < 0 && errno == EINTR); +#elif HAVE_POSIX_FALLOCATE + while ((result = posix_fallocate(ToFileDescriptor(fd), (off_t)offset, (off_t)length)) < 0 && errno == EINTR); +#else + // Not supported on this platform. Caller can ignore this failure since it's just a hint. + (void)fd, (void)offset, (void)length; + result = ENOTSUP; +#endif + return result; +} + + int32_t SystemNative_Read(intptr_t fd, void* buffer, int32_t bufferSize) { return Common_Read(fd, buffer, bufferSize); @@ -1184,7 +1200,7 @@ int32_t SystemNative_CopyFile(intptr_t sourceFd, intptr_t destinationFd) #endif } // If we copied to a filesystem (eg EXFAT) that does not preserve POSIX ownership, all files appear - // to be owned by root. If we aren't running as root, then we won't be an owner of our new file, and + // to be owned by root. If we aren't running as root, then we won't be an owner of our new file, and // attempting to copy metadata to it will fail with EPERM. We have copied successfully, we just can't // copy metadata. The best thing we can do is skip copying the metadata. if (ret != 0 && errno != EPERM) diff --git a/src/libraries/Native/Unix/System.Native/pal_io.h b/src/libraries/Native/Unix/System.Native/pal_io.h index e82cffe8a8c3a9..7029eda300b69d 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.h +++ b/src/libraries/Native/Unix/System.Native/pal_io.h @@ -604,6 +604,13 @@ PALEXPORT int32_t SystemNative_Poll(PollEvent* pollEvents, uint32_t eventCount, */ PALEXPORT int32_t SystemNative_PosixFAdvise(intptr_t fd, int64_t offset, int64_t length, int32_t advice); +/** + * Ensures that disk space is allocated. + * + * Returns 0 on success; otherwise, the error code is returned and errno is NOT set. + */ +PALEXPORT int32_t SystemNative_PosixFAllocate(intptr_t fd, int64_t offset, int64_t length); + /** * Reads the number of bytes specified into the provided buffer from the specified, opened file descriptor. * diff --git a/src/libraries/Native/Unix/configure.cmake b/src/libraries/Native/Unix/configure.cmake index 674375cdd9ec52..0cdc12cca9f0e3 100644 --- a/src/libraries/Native/Unix/configure.cmake +++ b/src/libraries/Native/Unix/configure.cmake @@ -209,6 +209,16 @@ check_symbol_exists( fcntl.h HAVE_POSIX_ADVISE) +check_symbol_exists( + posix_fallocate + fcntl.h + HAVE_POSIX_FALLOCATE) + +check_symbol_exists( + posix_fallocate64 + fcntl.h + HAVE_POSIX_FALLOCATE64) + check_symbol_exists( ioctl sys/ioctl.h diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index 75c8c0f88d260a..3453aa06d0ddc1 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -1,4 +1,4 @@ - + true c5ed3c1d-b572-46f1-8f96-522a85ce1179 @@ -1821,6 +1821,9 @@ Common\Interop\Unix\System.Native\Interop.PosixFAdvise.cs + + Common\Interop\Unix\System.Native\Interop.PosixFAllocate.cs + Common\Interop\Unix\System.Native\Interop.Read.cs diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs index 418dadc99378fd..d4d6af0ae8b666 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs @@ -7,6 +7,7 @@ using System.Runtime.Versioning; using System.Threading; using System.Threading.Tasks; +using Internal.IO; namespace System.IO.Strategies { @@ -35,7 +36,21 @@ internal static SafeFileHandle OpenHandle(string path, FileMode mode, FileAccess Interop.Sys.Permissions.S_IROTH | Interop.Sys.Permissions.S_IWOTH; // Open the file and store the safe handle. - return SafeFileHandle.Open(path!, openFlags, (int)OpenPermissions); + SafeFileHandle handle = SafeFileHandle.Open(path!, openFlags, (int)OpenPermissions); + if (allocationSize > 0 && (mode == FileMode.Create || mode == FileMode.CreateNew || mode == FileMode.Truncate)) + { + int allocationResult = Interop.Sys.PosixFAllocate(handle, 0, allocationSize); + + if (allocationResult == (int)Interop.Error.ENOSPC) + { + Interop.Sys.Unlink(path); // remove the file to mimic Windows behaviour (atomic operation) + + throw new IOException(SR.Format(SR.IO_DiskFull_Path_AllocationSize, path, allocationSize)); + } + // ignore not supported and other failures (pipe etc) + } + + return handle; } internal static bool GetDefaultIsAsync(SafeFileHandle handle, bool defaultIsAsync) => handle.IsAsync ?? defaultIsAsync; From 33f1e961d62c96956452874e57314f4d2dcea5f1 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 13 Apr 2021 20:02:49 +0200 Subject: [PATCH 02/11] missing file --- .../Unix/System.Native/Interop.PosixFAllocate.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/libraries/Common/src/Interop/Unix/System.Native/Interop.PosixFAllocate.cs diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PosixFAllocate.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PosixFAllocate.cs new file mode 100644 index 00000000000000..902ba72e175436 --- /dev/null +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PosixFAllocate.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; +using Microsoft.Win32.SafeHandles; + +internal static partial class Interop +{ + internal static partial class Sys + { + [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_PosixFAllocate", SetLastError = false /* this is explicitly called out in the man page */)] + internal static extern int PosixFAllocate(SafeFileHandle fd, long offset, long length); + } +} From bc2bca1e4e6d9ed1a09f0a5bb35b7bcd271d9a41 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 13 Apr 2021 20:08:28 +0200 Subject: [PATCH 03/11] remove Posix from the name as it won't be pure POSIX --- .../{Interop.PosixFAllocate.cs => Interop.FAllocate.cs} | 4 ++-- src/libraries/Native/Unix/System.Native/entrypoints.c | 2 +- src/libraries/Native/Unix/System.Native/pal_io.c | 2 +- src/libraries/Native/Unix/System.Native/pal_io.h | 2 +- .../src/System.Private.CoreLib.Shared.projitems | 4 ++-- .../src/System/IO/Strategies/FileStreamHelpers.Unix.cs | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) rename src/libraries/Common/src/Interop/Unix/System.Native/{Interop.PosixFAllocate.cs => Interop.FAllocate.cs} (65%) diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PosixFAllocate.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.FAllocate.cs similarity index 65% rename from src/libraries/Common/src/Interop/Unix/System.Native/Interop.PosixFAllocate.cs rename to src/libraries/Common/src/Interop/Unix/System.Native/Interop.FAllocate.cs index 902ba72e175436..8554f1fe66a53a 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PosixFAllocate.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.FAllocate.cs @@ -8,7 +8,7 @@ internal static partial class Interop { internal static partial class Sys { - [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_PosixFAllocate", SetLastError = false /* this is explicitly called out in the man page */)] - internal static extern int PosixFAllocate(SafeFileHandle fd, long offset, long length); + [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_FAllocate", SetLastError = false /* this is explicitly called out in the man page */)] + internal static extern int FAllocate(SafeFileHandle fd, long offset, long length); } } diff --git a/src/libraries/Native/Unix/System.Native/entrypoints.c b/src/libraries/Native/Unix/System.Native/entrypoints.c index 02ffd1916d6f95..107c089c2cbbd0 100644 --- a/src/libraries/Native/Unix/System.Native/entrypoints.c +++ b/src/libraries/Native/Unix/System.Native/entrypoints.c @@ -89,7 +89,7 @@ static const Entry s_sysNative[] = DllImportEntry(SystemNative_FTruncate) DllImportEntry(SystemNative_Poll) DllImportEntry(SystemNative_PosixFAdvise) - DllImportEntry(SystemNative_PosixFAllocate) + DllImportEntry(SystemNative_FAllocate) DllImportEntry(SystemNative_Read) DllImportEntry(SystemNative_ReadLink) DllImportEntry(SystemNative_Rename) diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index 7368e020438b69..f18893860ce015 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -991,7 +991,7 @@ int32_t SystemNative_PosixFAdvise(intptr_t fd, int64_t offset, int64_t length, i #endif } -int32_t SystemNative_PosixFAllocate(intptr_t fd, int64_t offset, int64_t length) +int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length) { int32_t result; #if HAVE_POSIX_FALLOCATE64 diff --git a/src/libraries/Native/Unix/System.Native/pal_io.h b/src/libraries/Native/Unix/System.Native/pal_io.h index 7029eda300b69d..313aa5a6179f6d 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.h +++ b/src/libraries/Native/Unix/System.Native/pal_io.h @@ -609,7 +609,7 @@ PALEXPORT int32_t SystemNative_PosixFAdvise(intptr_t fd, int64_t offset, int64_t * * Returns 0 on success; otherwise, the error code is returned and errno is NOT set. */ -PALEXPORT int32_t SystemNative_PosixFAllocate(intptr_t fd, int64_t offset, int64_t length); +PALEXPORT int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length); /** * Reads the number of bytes specified into the provided buffer from the specified, opened file descriptor. diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index 3453aa06d0ddc1..ebd67d49a9c3d9 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -1821,8 +1821,8 @@ Common\Interop\Unix\System.Native\Interop.PosixFAdvise.cs - - Common\Interop\Unix\System.Native\Interop.PosixFAllocate.cs + + Common\Interop\Unix\System.Native\Interop.FAllocate.cs Common\Interop\Unix\System.Native\Interop.Read.cs diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs index d4d6af0ae8b666..bb5b4c9bedb6d0 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs @@ -39,7 +39,7 @@ internal static SafeFileHandle OpenHandle(string path, FileMode mode, FileAccess SafeFileHandle handle = SafeFileHandle.Open(path!, openFlags, (int)OpenPermissions); if (allocationSize > 0 && (mode == FileMode.Create || mode == FileMode.CreateNew || mode == FileMode.Truncate)) { - int allocationResult = Interop.Sys.PosixFAllocate(handle, 0, allocationSize); + int allocationResult = Interop.Sys.FAllocate(handle, 0, allocationSize); if (allocationResult == (int)Interop.Error.ENOSPC) { From fbef3bd567cefd0bcf1abae8c49305f43282f286 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 13 Apr 2021 20:21:18 +0200 Subject: [PATCH 04/11] implement macOS support using F_PREALLOCATE --- .../Native/Unix/System.Native/pal_io.c | 20 +++++++++++++++++++ src/libraries/Native/Unix/configure.cmake | 5 +++++ 2 files changed, 25 insertions(+) diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index f18893860ce015..0071d78f0090fd 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -998,6 +998,26 @@ int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length) while ((result = posix_fallocate64(ToFileDescriptor(fd), (off64_t)offset, (off64_t)length)) < 0 && errno == EINTR); #elif HAVE_POSIX_FALLOCATE while ((result = posix_fallocate(ToFileDescriptor(fd), (off_t)offset, (off_t)length)) < 0 && errno == EINTR); +#elif defined(TARGET_OSX) && HAVE_F_PREALLOCATE + struct fstore_t fstore; + fstore.fst_flags = F_ALLOCATECONTIG; // ensure contiguous space + fstore.fst_posmode = F_VOLPOSMODE; // allocate from the offset + fstore.fst_offset = (off_t)offset; + fstore.fst_length = (off_t)length; + + while ((result = fcntl(ToFileDescriptor(fd), F_PREALLOCATE, &fstore)) < 0 && errno == EINTR); + + if (result == -1 && errno != ENOSPC) + { + // we have failed to allocate contiguous space, let's try non-contiguous + fstore.fst_flags = F_ALLOCATEALL; // all or nothing + while ((result = fcntl(ToFileDescriptor(fd), F_PREALLOCATE, &fstore)) < 0 && errno == EINTR); + + if (result == -1) + { + result = errno; + } + } #else // Not supported on this platform. Caller can ignore this failure since it's just a hint. (void)fd, (void)offset, (void)length; diff --git a/src/libraries/Native/Unix/configure.cmake b/src/libraries/Native/Unix/configure.cmake index 0cdc12cca9f0e3..f3b7ea11ce5577 100644 --- a/src/libraries/Native/Unix/configure.cmake +++ b/src/libraries/Native/Unix/configure.cmake @@ -219,6 +219,11 @@ check_symbol_exists( fcntl.h HAVE_POSIX_FALLOCATE64) +check_symbol_exists( + F_PREALLOCATE + fcntl.h + HAVE_F_PREALLOCATE) + check_symbol_exists( ioctl sys/ioctl.h From 071fc3357d0a14e86b42ac01a1fe19b9047a5623 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 13 Apr 2021 21:17:00 +0200 Subject: [PATCH 05/11] ensure file does not exist after allocation fails --- .../tests/FileStream/FileStreamConformanceTests.cs | 11 ++++++++--- .../System/IO/Strategies/FileStreamHelpers.Unix.cs | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.IO.FileSystem/tests/FileStream/FileStreamConformanceTests.cs b/src/libraries/System.IO.FileSystem/tests/FileStream/FileStreamConformanceTests.cs index 5ab668f760724c..a8ac77595566d8 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileStream/FileStreamConformanceTests.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileStream/FileStreamConformanceTests.cs @@ -62,7 +62,7 @@ public async Task FileOffsetIsPreservedWhenFileStreamIsCreatedFromSafeFileHandle using FileStream createdFromHandle = new FileStream(stream.SafeFileHandle, FileAccess.Write); - Assert.Equal(buffer.Length, stream.Position); + Assert.Equal(buffer.Length, stream.Position); Assert.Equal(stream.Position, createdFromHandle.Position); } @@ -187,17 +187,22 @@ public async Task WriteByteFlushesTheBufferWhenItBecomesFull() byte[] allBytes = File.ReadAllBytes(filePath); Assert.Equal(writtenBytes.ToArray(), allBytes); } - + [Fact] public void WhenFileStreamFailsToPreallocateDiskSpaceTheErrorMessageContainsAllTheDetails() { const long tooMuch = 1024L * 1024L * 1024L * 1024L; // 1 TB string filePath = GetTestFilePath(); - IOException ex = Assert.Throws(() => new FileStream(filePath, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None, BufferSize, Options, tooMuch)); + + Assert.False(File.Exists(filePath)); + + IOException ex = Assert.Throws(() => new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None, BufferSize, Options, tooMuch)); Assert.Contains("disk was full", ex.Message); Assert.Contains(filePath, ex.Message); Assert.Contains(AllocationSize.ToString(), ex.Message); + + Assert.False(File.Exists(filePath)); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs index bb5b4c9bedb6d0..24a8d59d2850a3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs @@ -37,7 +37,7 @@ internal static SafeFileHandle OpenHandle(string path, FileMode mode, FileAccess // Open the file and store the safe handle. SafeFileHandle handle = SafeFileHandle.Open(path!, openFlags, (int)OpenPermissions); - if (allocationSize > 0 && (mode == FileMode.Create || mode == FileMode.CreateNew || mode == FileMode.Truncate)) + if (allocationSize > 0 && (mode == FileMode.Create || mode == FileMode.CreateNew || mode == FileMode.Truncate || mode == FileMode.OpenOrCreate)) { int allocationResult = Interop.Sys.FAllocate(handle, 0, allocationSize); From 25e3d19ab10960e0dd0108c0dc5eaccf7c50efa7 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 14 Apr 2021 08:12:33 +0200 Subject: [PATCH 06/11] posix_fallocate does not set errno --- src/libraries/Native/Unix/System.Native/pal_io.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index 0071d78f0090fd..4c04309e397b06 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -995,9 +995,9 @@ int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length) { int32_t result; #if HAVE_POSIX_FALLOCATE64 - while ((result = posix_fallocate64(ToFileDescriptor(fd), (off64_t)offset, (off64_t)length)) < 0 && errno == EINTR); + while ((result = posix_fallocate64(ToFileDescriptor(fd), (off64_t)offset, (off64_t)length)) == EINTR); #elif HAVE_POSIX_FALLOCATE - while ((result = posix_fallocate(ToFileDescriptor(fd), (off_t)offset, (off_t)length)) < 0 && errno == EINTR); + while ((result = posix_fallocate(ToFileDescriptor(fd), (off_t)offset, (off_t)length)) == EINTR); #elif defined(TARGET_OSX) && HAVE_F_PREALLOCATE struct fstore_t fstore; fstore.fst_flags = F_ALLOCATECONTIG; // ensure contiguous space From 991c4f3d6168189b658ba07d559a7a4346aad6f3 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 14 Apr 2021 15:53:56 +0200 Subject: [PATCH 07/11] working Linux implementation --- .../Native/Unix/System.Native/pal_io.c | 30 +++++++++++++++---- .../IO/Strategies/FileStreamHelpers.Unix.cs | 9 ++++-- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index 4c04309e397b06..529a90dbc1a6d7 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -993,11 +993,12 @@ int32_t SystemNative_PosixFAdvise(intptr_t fd, int64_t offset, int64_t length, i int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length) { + int fileDescriptor = ToFileDescriptor(fd); int32_t result; #if HAVE_POSIX_FALLOCATE64 - while ((result = posix_fallocate64(ToFileDescriptor(fd), (off64_t)offset, (off64_t)length)) == EINTR); + while ((result = posix_fallocate64(fileDescriptor, (off64_t)offset, (off64_t)length)) == EINTR); #elif HAVE_POSIX_FALLOCATE - while ((result = posix_fallocate(ToFileDescriptor(fd), (off_t)offset, (off_t)length)) == EINTR); + while ((result = posix_fallocate(fileDescriptor, (off_t)offset, (off_t)length)) == EINTR); #elif defined(TARGET_OSX) && HAVE_F_PREALLOCATE struct fstore_t fstore; fstore.fst_flags = F_ALLOCATECONTIG; // ensure contiguous space @@ -1005,13 +1006,13 @@ int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length) fstore.fst_offset = (off_t)offset; fstore.fst_length = (off_t)length; - while ((result = fcntl(ToFileDescriptor(fd), F_PREALLOCATE, &fstore)) < 0 && errno == EINTR); + while ((result = fcntl(fileDescriptor, F_PREALLOCATE, &fstore)) < 0 && errno == EINTR); if (result == -1 && errno != ENOSPC) { // we have failed to allocate contiguous space, let's try non-contiguous fstore.fst_flags = F_ALLOCATEALL; // all or nothing - while ((result = fcntl(ToFileDescriptor(fd), F_PREALLOCATE, &fstore)) < 0 && errno == EINTR); + while ((result = fcntl(fileDescriptor, F_PREALLOCATE, &fstore)) < 0 && errno == EINTR); if (result == -1) { @@ -1020,13 +1021,30 @@ int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length) } #else // Not supported on this platform. Caller can ignore this failure since it's just a hint. - (void)fd, (void)offset, (void)length; + (void)offset, (void)length; result = ENOTSUP; #endif + +#if HAVE_POSIX_FALLOCATE64 || HAVE_POSIX_FALLOCATE + if (result == ENOSPC) + { + // POSIX specification does not mention what should happen when the allocation fails due to lack of free space. + // Most of the Linux distros don't truncate the file and it has non-zero size (but less than what was requested. + // To mimic the Windows behaviour of the atomic NtCreateFile syscall we just remove the file. + + ftruncate(fileDescriptor, 0); + // the managed code has a reference to handle and it's responsible for Disposing it and Deleting the file + } +#elif defined(TARGET_OSX) && HAVE_F_PREALLOCATE + if (result == 0) + { + ftruncate(fileDescriptor, length); + } +#endif + return result; } - int32_t SystemNative_Read(intptr_t fd, void* buffer, int32_t bufferSize) { return Common_Read(fd, buffer, bufferSize); diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs index 24a8d59d2850a3..5138b36c10b0ea 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs @@ -14,6 +14,8 @@ namespace System.IO.Strategies // this type defines a set of stateless FileStream/FileStreamStrategy helper methods internal static partial class FileStreamHelpers { + private const int ENOSPC_Linux = 28; + // in the future we are most probably going to introduce more strategies (io_uring etc) private static FileStreamStrategy ChooseStrategyCore(SafeFileHandle handle, FileAccess access, FileShare share, int bufferSize, bool isAsync) => new Net5CompatFileStreamStrategy(handle, access, bufferSize, isAsync); @@ -37,12 +39,13 @@ internal static SafeFileHandle OpenHandle(string path, FileMode mode, FileAccess // Open the file and store the safe handle. SafeFileHandle handle = SafeFileHandle.Open(path!, openFlags, (int)OpenPermissions); - if (allocationSize > 0 && (mode == FileMode.Create || mode == FileMode.CreateNew || mode == FileMode.Truncate || mode == FileMode.OpenOrCreate)) + // If allocationSize has been provided for a creatable and writeable file + if (allocationSize > 0 && (access & FileAccess.Write) != 0 && mode != FileMode.Open && mode != FileMode.Append) { int allocationResult = Interop.Sys.FAllocate(handle, 0, allocationSize); - - if (allocationResult == (int)Interop.Error.ENOSPC) + if (allocationResult == (int)Interop.Error.ENOSPC || allocationResult == ENOSPC_Linux) { + handle.Dispose(); Interop.Sys.Unlink(path); // remove the file to mimic Windows behaviour (atomic operation) throw new IOException(SR.Format(SR.IO_DiskFull_Path_AllocationSize, path, allocationSize)); From efc861d04cb3f3de369c0f311b7dd0389cce96a1 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 20 Apr 2021 12:29:31 +0200 Subject: [PATCH 08/11] FreeBSD support --- .../Native/Unix/System.Native/pal_io.c | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index 529a90dbc1a6d7..8c5d63eb5f603e 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -995,11 +995,25 @@ int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length) { int fileDescriptor = ToFileDescriptor(fd); int32_t result; -#if HAVE_POSIX_FALLOCATE64 +#if HAVE_POSIX_FALLOCATE64 // 64-bit Linux while ((result = posix_fallocate64(fileDescriptor, (off64_t)offset, (off64_t)length)) == EINTR); -#elif HAVE_POSIX_FALLOCATE +#elif HAVE_POSIX_FALLOCATE // 32-bit Linux while ((result = posix_fallocate(fileDescriptor, (off_t)offset, (off_t)length)) == EINTR); -#elif defined(TARGET_OSX) && HAVE_F_PREALLOCATE +#elif defined(F_ALLOCSP) || defined(F_ALLOCSP64) // FreeBSD + #if HAVE_FLOCK64 + struct flock64 lockArgs; + int command = F_ALLOCSP64; + #else + struct flock lockArgs; + int command = F_ALLOCSP; + #endif + + lockArgs.l_whence = SEEK_SET; + lockArgs.l_start = (off_t)offset; + lockArgs.l_len = (off_t)length; + + while ((result = fcntl(fileDescriptor, command, &lockArgs)) < 0 && errno == EINTR) ; +#elif defined(TARGET_OSX) && HAVE_F_PREALLOCATE // macOS struct fstore_t fstore; fstore.fst_flags = F_ALLOCATECONTIG; // ensure contiguous space fstore.fst_posmode = F_VOLPOSMODE; // allocate from the offset From f5166130f02692703051560b37039b52665f42db Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 20 Apr 2021 12:53:03 +0200 Subject: [PATCH 09/11] improve the macOS logic handling --- .../Native/Unix/System.Native/pal_io.c | 43 ++++++++----------- src/libraries/Native/Unix/configure.cmake | 5 --- 2 files changed, 17 insertions(+), 31 deletions(-) diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index 8c5d63eb5f603e..3d7c024adfa817 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -999,6 +999,22 @@ int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length) while ((result = posix_fallocate64(fileDescriptor, (off64_t)offset, (off64_t)length)) == EINTR); #elif HAVE_POSIX_FALLOCATE // 32-bit Linux while ((result = posix_fallocate(fileDescriptor, (off_t)offset, (off_t)length)) == EINTR); +#elif defined(TARGET_OSX) && defined(F_PREALLOCATE) // macOS + fstore_t fstore; + fstore.fst_flags = F_ALLOCATECONTIG; // ensure contiguous space + fstore.fst_posmode = F_PEOFPOSMODE; // allocate from the physical end of file, as offset MUST NOT be 0 for F_VOLPOSMODE + fstore.fst_offset = (off_t)offset; + fstore.fst_length = (off_t)length; + fstore.fst_bytesalloc = 0; // output size, can be > length + + while ((result = fcntl(fileDescriptor, F_PREALLOCATE, &fstore)) == -1 && errno == EINTR) ; + + if (result == -1) + { + // we have failed to allocate contiguous space, let's try non-contiguous + fstore.fst_flags = F_ALLOCATEALL; // all or nothing + while ((result = fcntl(fileDescriptor, F_PREALLOCATE, &fstore)) == -1 && errno == EINTR) ; + } #elif defined(F_ALLOCSP) || defined(F_ALLOCSP64) // FreeBSD #if HAVE_FLOCK64 struct flock64 lockArgs; @@ -1012,27 +1028,7 @@ int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length) lockArgs.l_start = (off_t)offset; lockArgs.l_len = (off_t)length; - while ((result = fcntl(fileDescriptor, command, &lockArgs)) < 0 && errno == EINTR) ; -#elif defined(TARGET_OSX) && HAVE_F_PREALLOCATE // macOS - struct fstore_t fstore; - fstore.fst_flags = F_ALLOCATECONTIG; // ensure contiguous space - fstore.fst_posmode = F_VOLPOSMODE; // allocate from the offset - fstore.fst_offset = (off_t)offset; - fstore.fst_length = (off_t)length; - - while ((result = fcntl(fileDescriptor, F_PREALLOCATE, &fstore)) < 0 && errno == EINTR); - - if (result == -1 && errno != ENOSPC) - { - // we have failed to allocate contiguous space, let's try non-contiguous - fstore.fst_flags = F_ALLOCATEALL; // all or nothing - while ((result = fcntl(fileDescriptor, F_PREALLOCATE, &fstore)) < 0 && errno == EINTR); - - if (result == -1) - { - result = errno; - } - } + while ((result = fcntl(fileDescriptor, command, &lockArgs)) == -1 && errno == EINTR) ; #else // Not supported on this platform. Caller can ignore this failure since it's just a hint. (void)offset, (void)length; @@ -1049,11 +1045,6 @@ int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length) ftruncate(fileDescriptor, 0); // the managed code has a reference to handle and it's responsible for Disposing it and Deleting the file } -#elif defined(TARGET_OSX) && HAVE_F_PREALLOCATE - if (result == 0) - { - ftruncate(fileDescriptor, length); - } #endif return result; diff --git a/src/libraries/Native/Unix/configure.cmake b/src/libraries/Native/Unix/configure.cmake index f3b7ea11ce5577..0cdc12cca9f0e3 100644 --- a/src/libraries/Native/Unix/configure.cmake +++ b/src/libraries/Native/Unix/configure.cmake @@ -219,11 +219,6 @@ check_symbol_exists( fcntl.h HAVE_POSIX_FALLOCATE64) -check_symbol_exists( - F_PREALLOCATE - fcntl.h - HAVE_F_PREALLOCATE) - check_symbol_exists( ioctl sys/ioctl.h From 691ce88b5a68c49e2f246aeba6f624855f4bffa9 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 20 Apr 2021 12:53:48 +0200 Subject: [PATCH 10/11] remove the unsupported ifdef as in theory every supported OS should work fine now --- src/libraries/Native/Unix/System.Native/pal_io.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index 3d7c024adfa817..fbb228bba1aa8f 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -1029,10 +1029,6 @@ int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length) lockArgs.l_len = (off_t)length; while ((result = fcntl(fileDescriptor, command, &lockArgs)) == -1 && errno == EINTR) ; -#else - // Not supported on this platform. Caller can ignore this failure since it's just a hint. - (void)offset, (void)length; - result = ENOTSUP; #endif #if HAVE_POSIX_FALLOCATE64 || HAVE_POSIX_FALLOCATE From 169465061235b362459ceabdc9c4ecaf86a3c4d3 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 20 Apr 2021 16:09:51 +0200 Subject: [PATCH 11/11] all tests are passing on macOS! --- src/libraries/Native/Unix/System.Native/pal_io.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index fbb228bba1aa8f..89fa7c2108fac5 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -999,7 +999,7 @@ int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length) while ((result = posix_fallocate64(fileDescriptor, (off64_t)offset, (off64_t)length)) == EINTR); #elif HAVE_POSIX_FALLOCATE // 32-bit Linux while ((result = posix_fallocate(fileDescriptor, (off_t)offset, (off_t)length)) == EINTR); -#elif defined(TARGET_OSX) && defined(F_PREALLOCATE) // macOS +#elif defined(F_PREALLOCATE) // macOS fstore_t fstore; fstore.fst_flags = F_ALLOCATECONTIG; // ensure contiguous space fstore.fst_posmode = F_PEOFPOSMODE; // allocate from the physical end of file, as offset MUST NOT be 0 for F_VOLPOSMODE @@ -1031,15 +1031,12 @@ int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length) while ((result = fcntl(fileDescriptor, command, &lockArgs)) == -1 && errno == EINTR) ; #endif -#if HAVE_POSIX_FALLOCATE64 || HAVE_POSIX_FALLOCATE - if (result == ENOSPC) +#if defined(F_PREALLOCATE) || defined(F_ALLOCSP) || defined(F_ALLOCSP64) + // most of the Unixes implement posix_fallocate which does NOT set the last error + // fctnl does, but to mimic the posix_fallocate behaviour we just return error + if (result == -1) { - // POSIX specification does not mention what should happen when the allocation fails due to lack of free space. - // Most of the Linux distros don't truncate the file and it has non-zero size (but less than what was requested. - // To mimic the Windows behaviour of the atomic NtCreateFile syscall we just remove the file. - - ftruncate(fileDescriptor, 0); - // the managed code has a reference to handle and it's responsible for Disposing it and Deleting the file + result = errno; } #endif