From e3b0b7d3f60f6dc04da826442f0dfdf203b96680 Mon Sep 17 00:00:00 2001 From: GabrielDuf Date: Thu, 22 Oct 2026 11:59:34 -0400 Subject: [PATCH] Split NuGet V2 update checks so a long module list stops dropping every update The V2 update path put every installed package of a source into one GetUpdates() GET request, without deduplicating ids, so a module installed side by side appeared once per version. With about 450 PowerShell 7 modules the URL grew past what PSGallery accepts (about 11.8k characters returned 200, 15.7k returned 404, 19.6k returned 414). A non-success response was only logged, so the whole source contributed zero updates and PowerShell 7 disappeared from the Software Updates source filter. BuildV2UpdateUrls now deduplicates ids case-insensitively, keeps the lowest installed version of each (compared through the manager's CompareVersions with the same VersionStringToStruct fallback KeepUpdatesNewerThanInstalled uses), and splits the query into URLs of at most 8000 characters. Keeping the lowest version gives the same server answer as listing all of them: the latest version is newer than one installed version exactly when it is newer than the lowest. KeepUpdatesNewerThanInstalled still filters against the highest installed version. Each chunk is requested on a shared HttpClient with its own error handling, so a failed chunk is logged and counted without discarding the updates the other chunks found. The query string format and the last-wins version and scope maps are unchanged. Fixes #5555 Co-Authored-By: Claude Opus 5.5 --- .../BaseNuGet.cs | 152 +++++++++++---- .../NuGetV2UpdatesTests.cs | 178 ++++++++++++++++++ 2 files changed, 293 insertions(+), 37 deletions(-) create mode 100644 src/UniGetUI.PackageEngine.Tests/NuGetV2UpdatesTests.cs diff --git a/src/UniGetUI.PackageEngine.Managers.Generic.NuGet/BaseNuGet.cs b/src/UniGetUI.PackageEngine.Managers.Generic.NuGet/BaseNuGet.cs index 06ffa55949..f641d19428 100644 --- a/src/UniGetUI.PackageEngine.Managers.Generic.NuGet/BaseNuGet.cs +++ b/src/UniGetUI.PackageEngine.Managers.Generic.NuGet/BaseNuGet.cs @@ -493,52 +493,55 @@ out int v3Errors continue; } - var packageIds = new StringBuilder(); - var packageVers = new StringBuilder(); var packageIdVersion = new Dictionary(); foreach (var package in pair.Value) - { - packageIds.Append(package.Id + "|"); - packageVers.Append(package.VersionString + "|"); packageIdVersion[package.Id.ToLower()] = package.VersionString; - } var packageIdScope = BuildInstalledScopeMap(pair.Value); - var SearchUrl = - $"{pair.Key.Url.ToString().Trim('/')}/GetUpdates()" - + $"?packageIds=%27{HttpUtility.UrlEncode(packageIds.ToString().Trim('|'))}%27" - + $"&versions=%27{HttpUtility.UrlEncode(packageVers.ToString().Trim('|'))}%27" - + $"&includePrerelease={(canPrerelease ? "true" : "false")}" - + $"&includeAllVersions=0"; - using HttpClient client = new(CoreTools.GenericHttpClientParameters); client.DefaultRequestHeaders.UserAgent.ParseAdd(CoreData.UserAgentString); - using var request = new HttpRequestMessage(HttpMethod.Get, SearchUrl); - using HttpResponseMessage response = client.Send(request); - if (!response.IsSuccessStatusCode) - { - logger.Error( - $"Failed to fetch api at Url={SearchUrl} with status code {response.StatusCode}" - ); - errors++; - } - else + foreach ( + string SearchUrl in BuildV2UpdateUrls(pair.Key, pair.Value, canPrerelease) + ) { - string SearchResults = response - .Content.ReadAsStringAsync() - .GetAwaiter() - .GetResult(); - Packages.AddRange( - ParseUpdatesResponse( - SearchResults, - packageIdVersion, - packageIdScope, - pair.Key, - this, - logger - ) - ); + try + { + using var request = new HttpRequestMessage(HttpMethod.Get, SearchUrl); + using HttpResponseMessage response = client.Send(request); + + if (!response.IsSuccessStatusCode) + { + logger.Error( + $"Failed to fetch api at Url={SearchUrl} with status code {response.StatusCode}" + ); + errors++; + continue; + } + + string SearchResults = response + .Content.ReadAsStringAsync() + .GetAwaiter() + .GetResult(); + Packages.AddRange( + ParseUpdatesResponse( + SearchResults, + packageIdVersion, + packageIdScope, + pair.Key, + this, + logger + ) + ); + } + catch (Exception ex) + { + logger.Error( + $"Failed to fetch api at Url={SearchUrl} on source {pair.Key.Name}" + ); + logger.Error(ex); + errors++; + } } } catch (Exception ex) @@ -554,6 +557,81 @@ out int v3Errors return KeepUpdatesNewerThanInstalled(Packages, installedPackages); } + internal const int MaxV2UpdateUrlLength = 8000; + + internal IReadOnlyList BuildV2UpdateUrls( + IManagerSource source, + IReadOnlyList installedPackages, + bool canPrerelease + ) + { + var entries = new List<(string Id, string Version)>(); + var indexById = new Dictionary(); + foreach (var package in installedPackages) + { + string key = package.Id.ToLower(); + if (!indexById.TryGetValue(key, out int index)) + { + indexById[key] = entries.Count; + entries.Add((package.Id, package.VersionString)); + } + else if (IsOlderVersion(package.VersionString, entries[index].Version)) + { + entries[index] = (entries[index].Id, package.VersionString); + } + } + + string prefix = $"{source.Url.ToString().Trim('/')}/GetUpdates()?packageIds=%27"; + const string middle = "%27&versions=%27"; + string suffix = + $"%27&includePrerelease={(canPrerelease ? "true" : "false")}&includeAllVersions=0"; + string separator = HttpUtility.UrlEncode("|"); + int fixedLength = prefix.Length + middle.Length + suffix.Length; + + var urls = new List(); + var ids = new StringBuilder(); + var versions = new StringBuilder(); + int count = 0; + + foreach (var (id, version) in entries) + { + string encodedId = HttpUtility.UrlEncode(id); + string encodedVersion = HttpUtility.UrlEncode(version); + int added = + encodedId.Length + encodedVersion.Length + (count > 0 ? 2 * separator.Length : 0); + + if ( + count > 0 + && fixedLength + ids.Length + versions.Length + added > MaxV2UpdateUrlLength + ) + { + urls.Add(prefix + ids + middle + versions + suffix); + ids.Clear(); + versions.Clear(); + count = 0; + } + + if (count > 0) + { + ids.Append(separator); + versions.Append(separator); + } + ids.Append(encodedId); + versions.Append(encodedVersion); + count++; + } + + if (count > 0) + urls.Add(prefix + ids + middle + versions + suffix); + + return urls; + } + + private bool IsOlderVersion(string version, string other) => + CompareVersions(version, other) is { } comparison + ? comparison < 0 + : CoreTools.VersionStringToStruct(version) < CoreTools.VersionStringToStruct(other); + internal IReadOnlyList? GetAvailableUpdatesV3( IManagerSource source, IReadOnlyList installedPackages, diff --git a/src/UniGetUI.PackageEngine.Tests/NuGetV2UpdatesTests.cs b/src/UniGetUI.PackageEngine.Tests/NuGetV2UpdatesTests.cs new file mode 100644 index 0000000000..4236603eb7 --- /dev/null +++ b/src/UniGetUI.PackageEngine.Tests/NuGetV2UpdatesTests.cs @@ -0,0 +1,178 @@ +using System.Net; +using System.Text; +using UniGetUI.PackageEngine.Classes.Manager; +using UniGetUI.PackageEngine.Interfaces; +using UniGetUI.PackageEngine.ManagerClasses.Classes; +using UniGetUI.PackageEngine.ManagerClasses.Manager; +using UniGetUI.PackageEngine.Managers.PowerShellManager; +using UniGetUI.PackageEngine.PackageClasses; +using UniGetUI.PackageEngine.Tests.Infrastructure.Helpers; + +namespace UniGetUI.PackageEngine.Tests; + +public sealed class NuGetV2UpdatesTests +{ + private const int FeedMaxUrlLength = 11000; + + [Fact] + public void GetAvailableUpdatesReportsEveryModuleWhenTheInstalledListExceedsTheFeedUrlLimit() + { + using var feed = new V2UpdatesFeed(); + var manager = feed.CreateManager(InstalledModules(450, duplicatedVersions: 60)); + + var updates = manager.GetAvailableUpdates(); + + Assert.Equal(450, updates.Count); + Assert.Equal(450, updates.Select(update => update.Id.ToLowerInvariant()).Distinct().Count()); + Assert.All(updates, update => Assert.Equal("99.0.0", update.NewVersionString)); + Assert.True(feed.Server.RequestPaths.Count > 1); + Assert.All(feed.RequestedUrlLengths, length => Assert.True(length <= FeedMaxUrlLength)); + } + + [Fact] + public void GetAvailableUpdatesKeepsTheOtherChunksWhenOneRequestFails() + { + using var feed = new V2UpdatesFeed { FailingPackageId = "Microsoft.Graph.Beta.Module000" }; + var manager = feed.CreateManager(InstalledModules(450, duplicatedVersions: 0)); + + var updates = manager.GetAvailableUpdates(); + + Assert.NotEmpty(updates); + Assert.True(updates.Count < 450); + Assert.DoesNotContain(updates, update => update.Id == "Microsoft.Graph.Beta.Module000"); + Assert.Contains(updates, update => update.Id == "Microsoft.Graph.Beta.Module449"); + } + + private static List<(string Id, string Version)> InstalledModules(int count, int duplicatedVersions) + { + var modules = new List<(string Id, string Version)>(); + for (int i = 0; i < count; i++) + { + string id = $"Microsoft.Graph.Beta.Module{i:000}"; + modules.Add((id, "2.25.0")); + if (i < duplicatedVersions) + modules.Add((id, "2.24.0")); + } + + return modules; + } + + private sealed class V2UpdatesFeed : IDisposable + { + private TestHttpServer? _server; + private readonly List _requestedUrlLengths = []; + + public TestHttpServer Server => _server ??= new TestHttpServer(Handle); + + public string? FailingPackageId { get; init; } + + public IReadOnlyList RequestedUrlLengths + { + get + { + lock (_requestedUrlLengths) + { + return _requestedUrlLengths.ToArray(); + } + } + } + + public V2TestManager CreateManager(IReadOnlyList<(string Id, string Version)> installed) + { + var manager = new V2TestManager($"{Server.BaseUri.AbsoluteUri}api/v2/", installed); + manager.Initialize(); + Assert.True(manager.IsReady()); + return manager; + } + + public void Dispose() => _server?.Dispose(); + + private (int StatusCode, string Content, string ContentType) Handle( + HttpListenerRequest request + ) + { + string rawUrl = request.Url?.AbsoluteUri ?? string.Empty; + lock (_requestedUrlLengths) + { + _requestedUrlLengths.Add(rawUrl.Length); + } + + if (!(request.Url?.AbsolutePath ?? string.Empty).EndsWith("/GetUpdates()")) + return (404, string.Empty, "application/xml"); + + if (rawUrl.Length > FeedMaxUrlLength) + return (404, string.Empty, "application/xml"); + + string[] ids = (request.QueryString["packageIds"] ?? string.Empty) + .Trim('\'') + .Split('|', StringSplitOptions.RemoveEmptyEntries); + + if (FailingPackageId is not null && ids.Contains(FailingPackageId)) + return (500, string.Empty, "application/xml"); + + var feed = new StringBuilder(""); + foreach (string id in ids) + feed.Append($"{id}99.0.0"); + feed.Append(""); + + return (200, feed.ToString(), "application/atom+xml"); + } + } + + private sealed class V2TestManager : BaseNuGet + { + private readonly IReadOnlyList<(string Id, string Version)> _installed; + + public V2TestManager(string sourceUrl, IReadOnlyList<(string Id, string Version)> installed) + { + _installed = installed; + + Capabilities = new ManagerCapabilities + { + SupportsCustomVersions = true, + SupportsCustomPackageIcons = true, + CanListDependencies = true, + }; + + Properties = new ManagerProperties + { + Id = "test-nuget-v2", + Name = "TestNuGetV2", + DefaultSource = new ManagerSource(this, "test", new Uri(sourceUrl)), + }; + + DetailsHelper = new Helper(this); + } + + protected override IReadOnlyList _getInstalledPackages_UnSafe() => + _installed + .Select(module => new Package( + module.Id, + module.Id, + module.Version, + Properties.DefaultSource, + this + )) + .ToList(); + + public override IReadOnlyList FindCandidateExecutableFiles() => []; + + protected override void _loadManagerExecutableFile( + out bool found, + out string executablePath, + out string callArgs + ) + { + found = true; + executablePath = "test-nuget-v2"; + callArgs = string.Empty; + } + + protected override void _loadManagerVersion(out string version) => version = "0.0.0"; + + private sealed class Helper(BaseNuGet manager) : BaseNuGetDetailsHelper(manager) + { + protected override string? GetInstallLocation_UnSafe(IPackage package) => null; + } + } +}