Fix virtualization handling of large dataset length changes. Fixes #37245 (#37250)

Backport of #37247 to release/6.0

/cc @SteveSandersonMS

## Description/Customer Impact

The `Virtualize` component exhibits pathological behavior in the following case:

 * User scrolls to the end of a very long list (e.g., 100000 items)
 * The list becomes much shorter (e.g., 100 items)

Instead of jumping back to the new maximum scroll position, it incrementally steps back to it one page at a time. In the above example, if the scroll region is tall enough for 20 items, that would take (100000-100)/20 = 4995 steps. On each step it requests a new data set (possibly causing an HTTP request and DB query) and a UI re-render.

Eventually this does complete, but it's extraordinarily inefficient. It's very trivial to fix this so that we move to the correct final scroll offset in a single step.

## Regression?
 - [ ] Yes
 - [X] No

## Risk
- [ ] High
- [ ] Medium
- [X] Low

* Change was verified manually and via a new E2E test
* The code change is trivial: we can detect and correct the invalid scroll offset in a single step. The new logic shouldn't affect any existing case where the scroll offset was valid.

## Verification
- [X] Manual (required)
- [X] Automated

## Packaging changes reviewed?
- [ ] Yes
- [ ] No
- [X] N/A

Fixes #37245

Author: github-actions[bot]

src/Components/Web/src/Virtualization/Virtualize.cs

@@ -308,6 +308,14 @@ private void CalcualteItemDistribution(
 
         private void UpdateItemDistribution(int itemsBefore, int visibleItemCapacity)
         {
+            // If the itemcount just changed to a lower number, and we're already scrolled past the end of the new
+            // reduced set of items, clamp the scroll position to the new maximum
+            if (itemsBefore + visibleItemCapacity > _itemCount)
+            {
+                itemsBefore = Math.Max(0, _itemCount - visibleItemCapacity);
+            }
+
+            // If anything about the offset changed, re-render
             if (itemsBefore != _itemsBefore || visibleItemCapacity != _visibleItemCapacity)
             {
                 _itemsBefore = itemsBefore;

src/Components/test/E2ETest/Tests/VirtualizationTest.cs

@@ -11,6 +11,7 @@
 using Microsoft.AspNetCore.Testing;
 using OpenQA.Selenium;
 using OpenQA.Selenium.Support.Extensions;
+using OpenQA.Selenium.Support.UI;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -366,10 +367,69 @@ public void CanRefreshItemsProviderResultsInPlace()
                 name => Assert.Equal("Person 3", name));
         }
 
+        [Fact]
+        public void CanExpandDataSetAndRetainScrollPosition()
+        {
+            Browser.MountTestComponent<VirtualizationDataChanges>();
+            var dataSetLengthSelector = new SelectElement(Browser.Exists(By.Id("large-dataset-length")));
+            var dataSetLengthLastRendered = () => int.Parse(Browser.FindElement(By.Id("large-dataset-length-lastrendered")).Text, CultureInfo.InvariantCulture);
+            var container = Browser.Exists(By.Id("removing-many"));
+
+            // Scroll to the end of a medium list
+            dataSetLengthSelector.SelectByText("1000");
+            Browser.Equal(1000, dataSetLengthLastRendered);
+            Browser.True(() =>
+            {
+                ScrollToEnd(Browser, container);
+                return GetPeopleNames(container).Contains("Person 1000");
+            });
+
+            // Expand the data set
+            dataSetLengthSelector.SelectByText("100000");
+            Browser.Equal(100000, dataSetLengthLastRendered);
+
+            // See that the old data is still visible, because the scroll position is preserved as a pixel count,
+            // not a scroll percentage
+            Browser.True(() => GetPeopleNames(container).Contains("Person 1000"));
+        }
+
+        [Fact]
+        public void CanHandleDataSetShrinkingWithExistingOffsetAlreadyBeyondNewListEnd()
+        {
+            // Represents https://github.com/dotnet/aspnetcore/issues/37245
+            Browser.MountTestComponent<VirtualizationDataChanges>();
+            var dataSetLengthSelector = new SelectElement(Browser.Exists(By.Id("large-dataset-length")));
+            var dataSetLengthLastRendered = () => int.Parse(Browser.FindElement(By.Id("large-dataset-length-lastrendered")).Text, CultureInfo.InvariantCulture);
+            var container = Browser.Exists(By.Id("removing-many"));
+
+            // Scroll to the end of a very long list
+            dataSetLengthSelector.SelectByText("100000");
+            Browser.Equal(100000, dataSetLengthLastRendered);
+            Browser.True(() =>
+            {
+                ScrollToEnd(Browser, container);
+                return GetPeopleNames(container).Contains("Person 100000");
+            });
+
+            // Now make the dataset much shorter
+            // We should automatically have the scroll position reduced to the new maximum
+            // Because the new data set is *so much* shorter than the previous one, if bug #37245 were still here,
+            // this would take over 30 minutes so the test would fail
+            dataSetLengthSelector.SelectByText("25");
+            Browser.Equal(25, dataSetLengthLastRendered);
+            Browser.True(() => GetPeopleNames(container).Contains("Person 25"));
+        }
+
         private string[] GetPeopleNames(IWebElement container)
         {
             var peopleElements = container.FindElements(By.CssSelector(".person span"));
             return peopleElements.Select(element => element.Text).ToArray();
         }
+
+        private static void ScrollToEnd(IWebDriver browser, IWebElement elem)
+        {
+            var js = (IJavaScriptExecutor)browser;
+            js.ExecuteScript("arguments[0].scrollTop = arguments[0].scrollHeight", elem);
+        }
     }
 }

src/Components/test/testassets/BasicTestApp/VirtualizationDataChanges.razor

@@ -38,11 +38,34 @@
     </Virtualize>
 </div>
 
+<h4>Large data set with size changes</h4>
+<p>
+    Data set length:
+    <select id="large-dataset-length" value="@largeDataSetLength" @onchange="@(e => SetLargeDataSetLengthAsync(int.Parse((string)e.Value)))">
+        <option>25</option>
+        <option>1000</option>
+        <option>100000</option>
+    </select>
+    Last rendered:
+    <span id="large-dataset-length-lastrendered">@largeDataSetLengthLastRendered</span>
+</p>
+<div id="removing-many" style="overflow-y: auto; height: 200px; border: 1px dashed gray;">
+    <Virtualize @ref="removeManyVirtualize" ItemsProvider="GetLargeDataSetAsync">
+        <div @key="@context.Name" class="person" style="border-bottom: 1px solid silver; padding: 4px;">
+            <span>@context.Name</span>
+        </div>
+    </Virtualize>
+</div>
+
 @code {
     Virtualize<Person> asyncVirtualize;
+    Virtualize<Person> removeManyVirtualize;
     List<Person> fixedPeople = Enumerable.Range(1, 3).Select(GeneratePerson).ToList();
     int numPeopleInItemsProvider = 3;
 
+    int largeDataSetLength = 25;
+    int? largeDataSetLengthLastRendered;
+
     void AddPersonToFixedList()
     {
         // When using Items (not ItemsProvider), the Virtualize component re-queries
@@ -69,6 +92,27 @@
             numPeopleInItemsProvider);
     }
 
+    async Task SetLargeDataSetLengthAsync(int length)
+    {
+        largeDataSetLength = length;
+        await removeManyVirtualize.RefreshDataAsync();
+    }
+
+    async ValueTask<ItemsProviderResult<Person>> GetLargeDataSetAsync(ItemsProviderRequest request)
+    {
+        await Task.Delay(500);
+        largeDataSetLengthLastRendered = largeDataSetLength;
+
+        // Behave like a .Skip(startIndex).Take(count), so that if you ask for data beyond the end of the
+        // set, you get back nothing
+        var lastIndexExcl = Math.Min(request.StartIndex + request.Count, largeDataSetLength);
+        var effectiveCount = lastIndexExcl - request.StartIndex;
+        var resultItems = effectiveCount <= 0
+            ? Enumerable.Empty<Person>()
+            : Enumerable.Range(1 + request.StartIndex, effectiveCount).Select(GeneratePerson).ToList();
+        return new ItemsProviderResult<Person>(resultItems, largeDataSetLength);
+    }
+
     class Person
     {
         public string Name { get; set; }