diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 674b6ccf34f95..5f790d8fa448f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,6 +146,11 @@ jobs: - name: ensure the channel matches the target branch run: src/ci/scripts/verify-channel.sh + - name: print kernel/libc versions + if: runner.os == 'Linux' + run: | + uname -a + - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh diff --git a/library/std/src/fs/tests.rs b/library/std/src/fs/tests.rs index c81e3af2f0d4c..2b5774b0d2b40 100644 --- a/library/std/src/fs/tests.rs +++ b/library/std/src/fs/tests.rs @@ -8,6 +8,7 @@ use rand::RngCore; target_vendor = "apple", ))] use crate::assert_matches::assert_matches; +use crate::bstr::ByteStr; use crate::char::MAX_LEN_UTF8; #[cfg(any( windows, @@ -490,6 +491,25 @@ fn file_test_io_read_write_at() { check!(fs::remove_file(&filename)); } +#[test] +#[cfg(unix)] +fn file_test_append_write_at() { + use crate::os::unix::fs::FileExt; + + let tmpdir = tmpdir(); + let filename = tmpdir.join("file_test_append_write_at.txt"); + let msg = b"it's not working!"; + check!(fs::write(&filename, &msg)); + // write_at should work even in in append mode + let mut f = check!(fs::File::options().append(true).open(&filename)); + assert_eq!(check!(f.stream_position()), 0); + assert_eq!(check!(f.write_at(b" ", 5)), 3); + assert_eq!(check!(f.stream_position()), 0); + + let content = check!(fs::read(&filename)); + assert_eq!(ByteStr::new(&content), ByteStr::new(b"it's working!")); +} + #[test] #[cfg(unix)] fn set_get_unix_permissions() { diff --git a/library/std/src/os/unix/fs.rs b/library/std/src/os/unix/fs.rs index b776df3dde1da..2c5ada7deb54e 100644 --- a/library/std/src/os/unix/fs.rs +++ b/library/std/src/os/unix/fs.rs @@ -147,9 +147,10 @@ pub trait FileExt { /// /// # Bug /// On some systems, `write_at` utilises [`pwrite64`] to write to files. - /// However, this syscall has a [bug] where files opened with the `O_APPEND` + /// However, on linux this syscall has a [bug] where files opened with the `O_APPEND` /// flag fail to respect the offset parameter, always appending to the end - /// of the file instead. + /// of the file instead. When available `pwritev2(..., RWF_NOAPPEND)` will + /// be used instead to avoid this bug. /// /// It is possible to inadvertently set this flag, like in the example below. /// Therefore, it is important to be vigilant while changing options to mitigate diff --git a/library/std/src/os/unix/fs/tests.rs b/library/std/src/os/unix/fs/tests.rs index 1840bb38c17c8..ecd799049282a 100644 --- a/library/std/src/os/unix/fs/tests.rs +++ b/library/std/src/os/unix/fs/tests.rs @@ -1,4 +1,6 @@ use super::*; +use crate::bstr::ByteStr; +use crate::io::Seek; #[test] fn read_vectored_at() { @@ -39,13 +41,16 @@ fn write_vectored_at() { file.write_all(msg).unwrap(); } let expected = { - let file = fs::File::options().write(true).open(&filename).unwrap(); + // Open in append mode to test that positioned writes bypass O_APPEND. + let mut file = fs::File::options().append(true).open(&filename).unwrap(); let buf0 = b" "; let buf1 = b"great "; let iovec = [io::IoSlice::new(buf0), io::IoSlice::new(buf1)]; + assert_eq!(file.stream_position().unwrap(), 0); let n = file.write_vectored_at(&iovec, 11).unwrap(); + assert_eq!(file.stream_position().unwrap(), 0); assert!(n == 4 || n == 11); @@ -53,7 +58,7 @@ fn write_vectored_at() { }; let content = fs::read(&filename).unwrap(); - assert_eq!(&content, expected); + assert_eq!(ByteStr::new(&content), ByteStr::new(expected)); } #[test] diff --git a/library/std/src/sys/fd/unix.rs b/library/std/src/sys/fd/unix.rs index cdca73cdca11e..026d92f54430d 100644 --- a/library/std/src/sys/fd/unix.rs +++ b/library/std/src/sys/fd/unix.rs @@ -22,9 +22,12 @@ use crate::cmp; use crate::io::{self, BorrowedCursor, IoSlice, IoSliceMut, Read}; use crate::os::unix::io::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; use crate::sys::cvt; -#[cfg(all(target_os = "android", target_pointer_width = "64"))] +#[cfg(all(any(target_os = "android", target_os = "linux"), target_pointer_width = "64"))] use crate::sys::pal::weak::syscall; -#[cfg(any(all(target_os = "android", target_pointer_width = "32"), target_vendor = "apple"))] +#[cfg(any( + all(any(target_os = "android", target_os = "linux"), target_pointer_width = "32"), + target_vendor = "apple" +))] use crate::sys::pal::weak::weak; use crate::sys_common::{AsInner, FromInner, IntoInner}; @@ -384,6 +387,16 @@ impl FileDesc { ))] use libc::pwrite64; + // Work around linux deviating from POSIX where it ignores the + // offset of pwrite when the file was opened with O_APPEND. + #[cfg(any(target_os = "linux", target_os = "android"))] + { + let iov = [IoSlice::new(buf)]; + if let Some(ret) = self.pwritev2(&iov, offset) { + return ret; + } + } + unsafe { cvt(pwrite64( self.as_raw_fd(), @@ -408,6 +421,13 @@ impl FileDesc { target_os = "openbsd", // OpenBSD 2.7 ))] pub fn write_vectored_at(&self, bufs: &[IoSlice<'_>], offset: u64) -> io::Result { + // Work around linux deviating from POSIX where it ignores the + // offset of pwrite when the file was opened with O_APPEND. + #[cfg(any(target_os = "linux", target_os = "android"))] + if let Some(ret) = self.pwritev2(bufs, offset) { + return ret; + } + let ret = cvt(unsafe { libc::pwritev( self.as_raw_fd(), @@ -611,6 +631,65 @@ impl FileDesc { pub fn duplicate(&self) -> io::Result { Ok(Self(self.0.try_clone()?)) } + + #[cfg(any(target_os = "linux", target_os = "android"))] + fn pwritev2(&self, bufs: &[IoSlice<'_>], offset: u64) -> Option> { + #[cfg(target_pointer_width = "64")] + syscall!( + fn pwritev2( + fd: libc::c_int, + iovec: *const libc::iovec, + n_iovec: libc::c_int, + offset: off64_t, + flags: libc::c_int, + ) -> isize; + ); + #[cfg(target_pointer_width = "32")] + let pwritev2 = { + weak!( + fn pwritev2( + fd: libc::c_int, + iovec: *const libc::iovec, + n_iovec: libc::c_int, + offset: off64_t, + flags: libc::c_int, + ) -> isize; + ); + let Some(pwritev2) = pwritev2.get() else { + return None; + }; + pwritev2 + }; + + use core::sync::atomic::AtomicBool; + + static NOAPPEND_SUPPORTED: AtomicBool = AtomicBool::new(true); + if NOAPPEND_SUPPORTED.load(core::sync::atomic::Ordering::Relaxed) { + let r = unsafe { + cvt(pwritev2( + self.as_raw_fd(), + bufs.as_ptr() as *const libc::iovec, + cmp::min(bufs.len(), max_iov()) as libc::c_int, + offset as off64_t, + libc::RWF_NOAPPEND, + )) + }; + match r { + Ok(ret) => return Some(Ok(ret as usize)), + Err(e) + if let Some(err) = e.raw_os_error() + && (err == libc::EOPNOTSUPP || err == libc::ENOSYS) => + { + eprintln!("pwritev2 NOAPPEND error: {err}"); + NOAPPEND_SUPPORTED.store(false, core::sync::atomic::Ordering::Relaxed); + return None; + } + Err(e) => return Some(Err(e)), + } + } + + return None; + } } impl<'a> Read for &'a FileDesc { diff --git a/src/ci/docker/host-x86_64/pr-check-2/Dockerfile b/src/ci/docker/host-x86_64/pr-check-2/Dockerfile index ce18a181d313d..876d63d2497eb 100644 --- a/src/ci/docker/host-x86_64/pr-check-2/Dockerfile +++ b/src/ci/docker/host-x86_64/pr-check-2/Dockerfile @@ -18,6 +18,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ xz-utils \ libssl-dev \ pkg-config \ + strace \ mingw-w64 \ && rm -rf /var/lib/apt/lists/* @@ -27,16 +28,6 @@ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh ENV SCRIPT \ - python3 ../x.py check && \ - python3 ../x.py clippy ci && \ - python3 ../x.py test --stage 1 core alloc std test proc_macro && \ - python3 ../x.py doc --stage 0 bootstrap && \ - # Build both public and internal documentation. - RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 0 compiler && \ - RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 1 library && \ - mkdir -p /checkout/obj/staging/doc && \ - cp -r build/x86_64-unknown-linux-gnu/doc /checkout/obj/staging && \ - RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 1 library/test && \ - # The BOOTSTRAP_TRACING flag is added to verify whether the - # bootstrap process compiles successfully with this flag enabled. - BOOTSTRAP_TRACING=1 python3 ../x.py --help + ldd --version && \ + python3 ../x.py test --stage 1 std -- "write_at" || \ + strace -ffe write,pwritev2,pwrite64,pwritev python3 ../x.py test --stage 1 std -- "write_at"