Reduce database calls for Microsoft profile page from 21.056 to 2 (#10411)

Author: ErikEJ

src/NuGetGallery.Services/PackageManagement/IPackageService.cs

@@ -103,6 +103,8 @@ Package FilterLatestPackageBySuffix(IReadOnlyCollection<Package> packages,
 
         IEnumerable<Package> FindPackagesByOwner(User user, bool includeUnlisted, bool includeVersions = false);
 
+        (IReadOnlyCollection<Package> Packages, long TotalDownloadCount, int PackageCount) FindPackagesByProfile(User user, int page, int pageSize);
+
         IEnumerable<Package> FindPackagesByAnyMatchingOwner(User user, bool includeUnlisted, bool includeVersions = false);
 
         IQueryable<PackageRegistration> FindPackageRegistrationsByOwner(User user);

src/NuGetGallery.Services/PackageManagement/PackageService.cs

@@ -428,6 +428,53 @@ private static Package FilterLatestPackageHelper(
             return package;
         }
 
+        public (IReadOnlyCollection<Package> Packages, long TotalDownloadCount, int PackageCount) FindPackagesByProfile(
+            User user,
+            int page,
+            int pageSize)
+        {
+            if (user == null)
+            {
+                throw new ArgumentNullException(nameof(user));
+            }
+
+            IQueryable<Package> packages = _packageRepository.GetAll()
+                .Where(p => p.PackageRegistration.Owners.Any(o => o.Key == user.Key));
+
+            var packageSummary = packages
+                .Where(p => p.IsLatestSemVer2)
+                .Include(p => p.PackageRegistration)
+                .GroupBy(r => 1)
+                .Select(g => new
+                {
+                    PackageCount = g.Sum(x => 1),
+                    DownloadCount = g.Sum(x => x.PackageRegistration.DownloadCount)
+                })
+                .FirstOrDefault();
+
+            var packageCount = packageSummary != null ? packageSummary.PackageCount : 0;
+            var downloadCount = packageSummary != null ? packageSummary.DownloadCount : 0;
+
+            packages = packages
+                .Where(p => p.Listed
+                    && p.PackageStatusKey == PackageStatus.Available);
+
+            packages = GetLatestVersion(packages);
+
+            return (packages
+                .OrderByDescending(p => p.PackageRegistration.DownloadCount)
+                .ThenBy(p => p.PackageRegistration.Id)
+                .Skip((page-1) * pageSize)
+                .Take(pageSize)
+                .Include(p => p.PackageRegistration.Owners)
+                .Include(p => p.PackageRegistration.RequiredSigners)
+                .Include(p => p.PackageRegistration.Packages.Select(p => p.SupportedFrameworks))
+                .Include(p => p.PackageRegistration.Packages.Select(p => p.Deprecations))
+                .ToList(),
+                downloadCount,
+                packageCount);
+        }
+
         public IEnumerable<Package> FindPackagesByOwner(User user, bool includeUnlisted, bool includeVersions = false)
         {
             return GetPackagesForOwners(new[] { user.Key }, includeUnlisted, includeVersions);
@@ -459,20 +506,7 @@ private IEnumerable<Package> GetPackagesForOwners(IEnumerable<int> ownerKeys, bo
 
             if (!includeVersions)
             {
-                // Do a best effort of retrieving the latest versions. Note that UpdateIsLatest has had concurrency issues
-                // where sometimes packages no rows with IsLatest set. In this case, we'll just select the last inserted
-                // row (descending [Key]) as opposed to reading all rows into memory and sorting on NuGetVersion.
-                packages = packages
-                    .GroupBy(p => p.PackageRegistrationKey)
-                    .Select(g => g
-                        // order booleans desc so that true (1) comes first
-                        .OrderByDescending(p => p.IsLatestStableSemVer2)
-                        .ThenByDescending(p => p.IsLatestStable)
-                        .ThenByDescending(p => p.IsLatestSemVer2)
-                        .ThenByDescending(p => p.IsLatest)
-                        .ThenByDescending(p => p.Listed)
-                        .ThenByDescending(p => p.Key)
-                        .FirstOrDefault());
+                packages = GetLatestVersion(packages);
             }
 
             return packages
@@ -482,6 +516,25 @@ private IEnumerable<Package> GetPackagesForOwners(IEnumerable<int> ownerKeys, bo
                 .ToList();
         }
 
+        private static IQueryable<Package> GetLatestVersion(IQueryable<Package> packages)
+        {
+            // Do a best effort of retrieving the latest versions. Note that UpdateIsLatest has had concurrency issues
+            // where sometimes packages no rows with IsLatest set. In this case, we'll just select the last inserted
+            // row (descending [Key]) as opposed to reading all rows into memory and sorting on NuGetVersion.
+            packages = packages
+                .GroupBy(p => p.PackageRegistrationKey)
+                .Select(g => g
+                    // order booleans desc so that true (1) comes first
+                    .OrderByDescending(p => p.IsLatestStableSemVer2)
+                    .ThenByDescending(p => p.IsLatestStable)
+                    .ThenByDescending(p => p.IsLatestSemVer2)
+                    .ThenByDescending(p => p.IsLatest)
+                    .ThenByDescending(p => p.Listed)
+                    .ThenByDescending(p => p.Key)
+                    .FirstOrDefault());
+            return packages;
+        }
+
         public IQueryable<PackageRegistration> FindPackageRegistrationsByOwner(User user)
         {
             return _packageRegistrationRepository.GetAll().Where(p => p.Owners.Any(o => o.Key == user.Key));

src/NuGetGallery/Controllers/UsersController.cs

@@ -751,17 +751,25 @@ public virtual ActionResult Profiles(string username, int page = 1)
                 return HttpNotFound();
             }
 
-            var packages = PackageService.FindPackagesByOwner(user, includeUnlisted: false)
-                .Where(p => p.PackageStatusKey == PackageStatus.Available)
-                .OrderByDescending(p => p.PackageRegistration.DownloadCount)
+            var packageResult = PackageService.FindPackagesByProfile(user, page, GalleryConstants.DefaultPackageListPageSize);
+
+            var packages = packageResult.Packages
                 .Select(p =>
                 {
                     var viewModel = _listPackageItemViewModelFactory.Create(p, currentUser, false);
                     viewModel.DownloadCount = p.PackageRegistration.DownloadCount;
                     return viewModel;
                 }).ToList();
 
-            var model = new UserProfileModel(user, currentUser, packages, page - 1, GalleryConstants.DefaultPackageListPageSize, Url);
+            var model = new UserProfileModel(
+                user,
+                currentUser,
+                packages,
+                page - 1,
+                GalleryConstants.DefaultPackageListPageSize,
+                Url,
+                packageResult.TotalDownloadCount,
+                packageResult.PackageCount);
 
             return View(model);
         }

src/NuGetGallery/ViewModels/UserProfileModel.cs

@@ -10,28 +10,27 @@ namespace NuGetGallery
 {
     public class UserProfileModel
     {
-        public UserProfileModel(User user, User currentUser, List<ListPackageItemViewModel> allPackages, int pageIndex, int pageSize, UrlHelper url)
+        public UserProfileModel(User user, User currentUser, List<ListPackageItemViewModel> allPackages, int pageIndex, int pageSize, UrlHelper url, long totalDownloadCount, int packageCount)
         {
             User = user;
             Username = user.Username;
             EmailAddress = user.EmailAddress;
             UnconfirmedEmailAddress = user.UnconfirmedEmailAddress;
             IsLocked = user.IsLocked;
             AllPackages = allPackages;
-            TotalPackages = allPackages.Count;
+            TotalPackages = packageCount;
             PackagePage = pageIndex;
             PackagePageSize = pageSize;
 
-            TotalPackageDownloadCount = AllPackages.Sum(p => p.TotalDownloadCount);
+            TotalPackageDownloadCount = totalDownloadCount;
 
             PackagePageTotalCount = (TotalPackages + PackagePageSize - 1) / PackagePageSize;
 
             var pager = new PreviousNextPagerViewModel<ListPackageItemViewModel>(allPackages, pageIndex, PackagePageTotalCount,
                 page => url.User(user, page));
 
             Pager = pager;
-            PagedPackages = AllPackages.Skip(PackagePageSize * pageIndex)
-                                       .Take(PackagePageSize).ToList();
+            PagedPackages = AllPackages;
 
             CanManageAccount = ActionsRequiringPermissions.ManageAccount.CheckPermissions(currentUser, user) == PermissionsCheckResult.Allowed;
             CanViewAccount = ActionsRequiringPermissions.ViewAccount.CheckPermissions(currentUser, user) == PermissionsCheckResult.Allowed;

src/NuGetGallery/Views/Users/Profiles.cshtml

@@ -25,8 +25,8 @@
                 responsive: true)
             <div class="statistics">
                 <div class="statistic">
-                    <div class="value">@Model.AllPackages.Count.ToNuGetNumberString()</div>
-                    <div class="description">@(Model.AllPackages.Count == 1 ? "Package" : "Packages")</div>
+                    <div class="value">@Model.TotalPackages.ToNuGetNumberString()</div>
+                    <div class="description">@(Model.TotalPackages == 1 ? "Package" : "Packages")</div>
                 </div>
                 <div class="statistic">
                     <div class="value">@Model.TotalPackageDownloadCount.ToNuGetNumberString()</div>

tests/NuGetGallery.Facts/Controllers/UsersControllerFacts.cs

@@ -1162,89 +1162,6 @@ public static IEnumerable<object[]> PossibleOwnershipScenarios_Data
                 }
             }
 
-            [Theory]
-            [MemberData(nameof(PossibleOwnershipScenarios_Data))]
-            public void ReturnsSinglePackageAsExpected(User currentUser, User owner)
-            {
-                // Arrange
-                var username = "test";
-
-                var package = new Package
-                {
-                    Version = "1.1.1",
-
-                    PackageRegistration = new PackageRegistration
-                    {
-                        Id = "package",
-                        Owners = new[] { owner },
-                        DownloadCount = 150
-                    },
-
-                    DownloadCount = 100,
-                    PackageStatusKey = PackageStatus.Available
-                };
-                var invalidatedPackage = new Package
-                {
-                    Version = "1.0.0",
-
-                    PackageRegistration = new PackageRegistration
-                    {
-                        Id = "packageFailedValidation",
-                        Owners = new[] { owner },
-                        DownloadCount = 0
-                    },
-
-                    DownloadCount = 0,
-                    PackageStatusKey = PackageStatus.FailedValidation
-                };
-                var validatingPackage = new Package
-                {
-                    Version = "1.0.0",
-
-                    PackageRegistration = new PackageRegistration
-                    {
-                        Id = "packageValidating",
-                        Owners = new[] { owner },
-                        DownloadCount = 0
-                    },
-
-                    DownloadCount = 0,
-                    PackageStatusKey = PackageStatus.Validating
-                };
-                var deletedPackage = new Package
-                {
-                    Version = "1.0.0",
-
-                    PackageRegistration = new PackageRegistration
-                    {
-                        Id = "packageDeleted",
-                        Owners = new[] { owner },
-                        DownloadCount = 0
-                    },
-
-                    DownloadCount = 0,
-                    PackageStatusKey = PackageStatus.Deleted
-                };
-
-                GetMock<IUserService>()
-                    .Setup(x => x.FindByUsername(username, false))
-                    .Returns(owner);
-
-                GetMock<IPackageService>()
-                    .Setup(x => x.FindPackagesByOwner(owner, false, false))
-                    .Returns(new[] { package, invalidatedPackage, validatingPackage, deletedPackage });
-
-                var controller = GetController<UsersController>();
-                controller.SetCurrentUser(currentUser);
-
-                // Act
-                var result = controller.Profiles(username);
-
-                // Assert
-                var model = ResultAssert.IsView<UserProfileModel>(result);
-                AssertUserProfileModel(model, currentUser, owner, package);
-            }
-
             [Theory]
             [MemberData(nameof(PossibleOwnershipScenarios_Data))]
             public void SortsPackagesByDownloadCount(User currentUser, User owner)
@@ -1284,9 +1201,11 @@ public void SortsPackagesByDownloadCount(User currentUser, User owner)
                     .Setup(x => x.FindByUsername(username, false))
                     .Returns(owner);
 
+                var packages = new List<Package> { package2, package1 };
+
                 GetMock<IPackageService>()
-                    .Setup(x => x.FindPackagesByOwner(owner, false, false))
-                    .Returns(new[] { package1, package2 });
+                    .Setup(x => x.FindPackagesByProfile(owner, 1, GalleryConstants.DefaultPackageListPageSize))
+                    .Returns((packages.AsReadOnly(), 350, 2));
 
                 var controller = GetController<UsersController>();
                 controller.SetCurrentUser(currentUser);
@@ -1327,8 +1246,8 @@ public void UsesProperIconUrl(User currentUser, User owner)
                     .Setup(x => x.FindByUsername(username, false))
                     .Returns(owner);
                 GetMock<IPackageService>()
-                    .Setup(x => x.FindPackagesByOwner(owner, false, false))
-                    .Returns(new[] { userPackage });
+                    .Setup(x => x.FindPackagesByProfile(owner, 1, GalleryConstants.DefaultPackageListPageSize))
+                    .Returns((userPackages.AsReadOnly(), 0, 0 ));
 
                 var controller = GetController<UsersController>();
                 controller.SetCurrentUser(currentUser);