From 1e1205e65d5520b490e8b9f0250f3848999b3b27 Mon Sep 17 00:00:00 2001 From: Chris Kennelly CA Date: Wed, 11 Nov 2026 15:38:56 -0800 Subject: [PATCH] Unify PageFlags::Get page reads into a single batch loop. This folds ReadMany and the separate single-page and boundary-page reads in PageFlags::Get into one loop. PiperOrigin-RevId: 1019558985 --- tcmalloc/internal/pageflags.cc | 191 ++++++++++++--------------------- tcmalloc/internal/pageflags.h | 5 - 2 files changed, 66 insertions(+), 130 deletions(-) diff --git a/tcmalloc/internal/pageflags.cc b/tcmalloc/internal/pageflags.cc index 95d20f72d..f99497e44 100644 --- a/tcmalloc/internal/pageflags.cc +++ b/tcmalloc/internal/pageflags.cc @@ -193,39 +193,6 @@ absl::StatusCode PageFlags::MaybeReadOne(uintptr_t vaddr, uint64_t& flags, return absl::StatusCode::kOk; } -absl::StatusCode PageFlags::ReadMany(int64_t num_pages, PageStats& output) { - while (num_pages > 0) { - const size_t batch_size = std::min(kEntriesInBuf, num_pages); - const size_t to_read = kPagemapEntrySize * batch_size; - - // We read continuously. For the first read, this starts at wherever the - // first ReadOne ended. See above note for the reinterpret_cast. - auto status = - signal_safe_read(fd_, reinterpret_cast(buf_), to_read, nullptr); - if (status != to_read) { - return absl::StatusCode::kUnavailable; - } - for (int i = 0; i < batch_size; ++i) { - uint64_t flags = buf_[i]; - if (PageHead(flags)) { - last_head_read_ = flags; - } - - if (PageTail(flags)) { - if (ABSL_PREDICT_FALSE(last_head_read_ == -1)) { - TC_LOG("Did not see head page before tail page (i=%v, buf=%v)", i, - buf_[i]); - return absl::StatusCode::kFailedPrecondition; - } - flags = last_head_read_; - } - MaybeAddToStats(output, flags, kHardwarePageSize); - } - num_pages -= batch_size; - } - return absl::StatusCode::kOk; -} - std::optional PageFlags::IsHugepageBacked(const void* const addr) { if (fd_ < 0) { return std::nullopt; @@ -259,110 +226,84 @@ std::optional PageFlags::Get(const void* const addr, last_head_read_ = -1; PageStats ret; - if (size == 0) return ret; - uint64_t result_flags = 0; - bool is_huge = false; - uintptr_t uaddr = reinterpret_cast(addr); + const uintptr_t uaddr = reinterpret_cast(addr); + const uintptr_t uend = uaddr + size; // Round address down to get the start of the first page that has any bytes // corresponding to the span [addr, addr+size). - uintptr_t basePage = uaddr & ~(kHardwarePageSize - 1); + uintptr_t vaddr = uaddr & ~(kHardwarePageSize - 1); // Round end address up to get the start of the first page that does not // have any bytes corresponding to the span [addr, addr+size). - // The span is a subset of [basePage, endPage). - uintptr_t endPage = - (uaddr + size + kHardwarePageSize - 1) & ~(kHardwarePageSize - 1); - - int64_t remainingPages = (endPage - basePage) / kHardwarePageSize; - - if (remainingPages == 1) { - if (auto res = MaybeReadOne(basePage, result_flags, is_huge); - res != absl::StatusCode::kOk) { + // The span is a subset of [vaddr, endPage). + const uintptr_t endPage = + (uend + kHardwarePageSize - 1) & ~(kHardwarePageSize - 1); + + const uintptr_t basePage = vaddr; + // TODO(patrickx): Refactor PageFlags to use pread instead of Seek + + // signal_safe_read so we don't have to track and re-seek fd_'s offset. + bool need_seek = true; + int64_t remainingPages = (endPage - vaddr) / kHardwarePageSize; + while (remainingPages > 0) { + if (need_seek && Seek(vaddr) != absl::StatusCode::kOk) { return std::nullopt; } - MaybeAddToStats(ret, result_flags, size); - - if (ret.bytes_stale > 0) { - ret.stale_scan_seconds = MaybeReadStaleScanSeconds(); + need_seek = false; + const size_t batch_size = std::min(kEntriesInBuf, remainingPages); + const size_t to_read = kPagemapEntrySize * batch_size; + if (signal_safe_read(fd_, reinterpret_cast(buf_), to_read, + nullptr) != to_read) { + return std::nullopt; } - return ret; - } - - // Since the input address might not be page-aligned (it can possibly point - // to an arbitrary object), we read staleness about the first page separately - // with ReadOne, then read the complete pages with ReadMany, and then read the - // last page with ReadOne again if needed. - - // Handle the first page. - if (auto res = MaybeReadOne(basePage, result_flags, is_huge); - res != absl::StatusCode::kOk) { - return std::nullopt; - } - size_t firstPageSize = kHardwarePageSize - (uaddr - basePage); - if (is_huge) { - // The object starts in the middle of a native page, but the entire page - // might be stale. So the situation looks like, simplifying to four native - // pages per hugepage to make the diagram fit, an entire hugepage that looks - // like (where X is the span of interest): - // . basePage - // [....|..XX|XXXX|XXXX] - // ^^^^^^^ some other stale object(s) - // ^^ firstPageSize - // ^^^^^^^^^^^^^^ `pages_represented` pages, each of - // kHardwarePageSize - // The remainingPages <= 0 case covers the situation where the span ends - // before the hugepage. - const uint64_t base_page_offset = basePage & (kHugePageSize - 1); - const uint64_t base_page_index = base_page_offset / kHardwarePageSize; - const int64_t pages_represented = kPagesInHugePage - base_page_index; - - remainingPages -= pages_represented; - if (remainingPages <= 0) { - // This hugepage represents every single page that this object is on; - // we're done. - MaybeAddToStats(ret, result_flags, size); - - if (ret.bytes_stale > 0) { - ret.stale_scan_seconds = MaybeReadStaleScanSeconds(); + for (size_t i = 0; i < batch_size; ++i, vaddr += kHardwarePageSize) { + if (IsAlignedTo(vaddr, kHugePageSize)) { + last_head_read_ = -1; } - return ret; - } - - // pages_represented - 1 is the number of full pages represented (see - // diagram) - MaybeAddToStats( - ret, result_flags, - firstPageSize + (pages_represented - 1) * kHardwarePageSize); - - // We've read one uint64_t about a single page, but it represents 512 small - // pages. So the next page that is of interest is one hugepage away -- seek - // to make sure the next read doesn't double-count the native pages in - // between the two head pages. - if (auto res = Seek((basePage & kHugePageMask) + kHugePageSize); - res != absl::StatusCode::kOk) { - return std::nullopt; + uint64_t flags = buf_[i]; + if (ABSL_PREDICT_FALSE((PageHead(flags) || PageTail(flags)) && + !PageThp(flags))) { + // TODO(b/127849243): Support non-THP hugepages. + return std::nullopt; + } + if (PageHead(flags)) { + last_head_read_ = flags; + } else if (PageTail(flags)) { + // The object may start on a tail page of a hugepage, whose flags are + // only on the head page. Simplifying to four native pages per hugepage + // to make the diagram fit, an entire hugepage looks like (where X is + // the span of interest): + // . vaddr & kHugePageMask (head page) + // . basePage + // [....|..XX|XXXX|XXXX] + // ^^^^^^^ some other stale object(s) + if (vaddr == basePage) { + bool is_huge = false; + if (MaybeReadOne(vaddr & kHugePageMask, last_head_read_, is_huge) != + absl::StatusCode::kOk) { + return std::nullopt; + } + need_seek = true; + } else if (ABSL_PREDICT_FALSE(last_head_read_ == -1)) { + TC_LOG("Did not see head page before tail page (i=%v, buf=%v)", i, + buf_[i]); + return std::nullopt; + } + flags = last_head_read_; + } else { + last_head_read_ = -1; + } + // Intersect [uaddr, uend) (where X is the span of interest) with each + // page [vaddr, vaddr + kHardwarePageSize) (delimited by |) to get + // [max(uaddr, vaddr), min(uend, vaddr + kHardwarePageSize)): + // . . . . uaddr, uend + // [....|..XX|XXXX|XX..] or [.XX.] + // ^^ ^^^^ ^^ ^^ page_bytes + const size_t page_bytes = + std::min(uend, vaddr + kHardwarePageSize) - std::max(uaddr, vaddr); + MaybeAddToStats(ret, flags, page_bytes); } - } else { - remainingPages--; - MaybeAddToStats(ret, result_flags, firstPageSize); - } - - // Handle all pages but the last page. - if (auto res = ReadMany(remainingPages - 1, ret); - res != absl::StatusCode::kOk) { - return std::nullopt; - } - - // Check final page. It doesn't really matter if is_huge; we just want the - // statistics about the page that has the last byte of the object. - size_t lastPageSize = kHardwarePageSize - (endPage - uaddr - size); - if (auto res = - MaybeReadOne(endPage - kHardwarePageSize, result_flags, is_huge); - res != absl::StatusCode::kOk) { - return std::nullopt; + remainingPages -= batch_size; } - MaybeAddToStats(ret, result_flags, lastPageSize); if (ret.bytes_stale > 0) { ret.stale_scan_seconds = MaybeReadStaleScanSeconds(); diff --git a/tcmalloc/internal/pageflags.h b/tcmalloc/internal/pageflags.h index 63a30e9e4..6a8e62d7b 100644 --- a/tcmalloc/internal/pageflags.h +++ b/tcmalloc/internal/pageflags.h @@ -137,10 +137,6 @@ class PageFlags final : public PageFlagsBase { // Possibly seeks backwards in an effort to find head hugepages. [[nodiscard]] absl::StatusCode MaybeReadOne(uintptr_t vaddr, uint64_t& flags, bool& is_huge); - // This helper reads staleness information for `num_pages` worth of _full_ - // pages and puts the results into `output`. It continues the read from the - // last Seek() or last Read operation. - [[nodiscard]] absl::StatusCode ReadMany(int64_t num_pages, PageStats& output); static constexpr const char* kKstaledScanSeconds = "/sys/kernel/mm/kstaled/scan_seconds"; @@ -162,7 +158,6 @@ class PageFlags final : public PageFlagsBase { // assume any compound pages are 2 MB. static constexpr int kHugePageSize = (2 << 20); static constexpr uintptr_t kHugePageMask = ~(kHugePageSize - 1); - const size_t kPagesInHugePage = kHugePageSize / kHardwarePageSize; uint64_t buf_[kEntriesInBuf]; // Information about the previous head page. For any future-encountered tail