Skip to content

Split NuGet V2 update checks so a long module list stops dropping every update #5577

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Gabriel Dufresne (GabrielDuf) merged 1 commit into from
Oct 22, 2026
+293 −37
Merged

Split NuGet V2 update checks so a long module list stops dropping every update #5577

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 115 additions & 37 deletions src/UniGetUI.PackageEngine.Managers.Generic.NuGet/BaseNuGet.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -493,52 +493,55 @@ out int v3Errors
continue;
}

var packageIds = new StringBuilder();
var packageVers = new StringBuilder();
var packageIdVersion = new Dictionary<string, string>();
foreach (var package in pair.Value)
{
packageIds.Append(package.Id + "|");
packageVers.Append(package.VersionString + "|");
packageIdVersion[package.Id.ToLower()] = package.VersionString;
}
var packageIdScope = BuildInstalledScopeMap(pair.Value);

var SearchUrl =
$"{pair.Key.Url.ToString().Trim('/')}/GetUpdates()"
+ $"?packageIds=%27{HttpUtility.UrlEncode(packageIds.ToString().Trim('|'))}%27"
+ $"&versions=%27{HttpUtility.UrlEncode(packageVers.ToString().Trim('|'))}%27"
+ $"&includePrerelease={(canPrerelease ? "true" : "false")}"
+ $"&includeAllVersions=0";

using HttpClient client = new(CoreTools.GenericHttpClientParameters);
client.DefaultRequestHeaders.UserAgent.ParseAdd(CoreData.UserAgentString);
using var request = new HttpRequestMessage(HttpMethod.Get, SearchUrl);
using HttpResponseMessage response = client.Send(request);

if (!response.IsSuccessStatusCode)
{
logger.Error(
$"Failed to fetch api at Url={SearchUrl} with status code {response.StatusCode}"
);
errors++;
}
else
foreach (
string SearchUrl in BuildV2UpdateUrls(pair.Key, pair.Value, canPrerelease)
)
{
string SearchResults = response
.Content.ReadAsStringAsync()
.GetAwaiter()
.GetResult();
Packages.AddRange(
ParseUpdatesResponse(
SearchResults,
packageIdVersion,
packageIdScope,
pair.Key,
this,
logger
)
);
try
{
using var request = new HttpRequestMessage(HttpMethod.Get, SearchUrl);
using HttpResponseMessage response = client.Send(request);

if (!response.IsSuccessStatusCode)
{
logger.Error(
$"Failed to fetch api at Url={SearchUrl} with status code {response.StatusCode}"
);
errors++;
continue;
}

string SearchResults = response
.Content.ReadAsStringAsync()
.GetAwaiter()
.GetResult();
Packages.AddRange(
ParseUpdatesResponse(
SearchResults,
packageIdVersion,
packageIdScope,
pair.Key,
this,
logger
)
);
}
catch (Exception ex)
{
logger.Error(
$"Failed to fetch api at Url={SearchUrl} on source {pair.Key.Name}"
);
logger.Error(ex);
errors++;
}
}
}
catch (Exception ex)
Expand All @@ -554,6 +557,81 @@ out int v3Errors
return KeepUpdatesNewerThanInstalled(Packages, installedPackages);
}

internal const int MaxV2UpdateUrlLength = 8000;

internal IReadOnlyList<string> BuildV2UpdateUrls(
IManagerSource source,
IReadOnlyList<IPackage> installedPackages,
bool canPrerelease
)
{
var entries = new List<(string Id, string Version)>();
var indexById = new Dictionary<string, int>();
foreach (var package in installedPackages)
{
string key = package.Id.ToLower();
if (!indexById.TryGetValue(key, out int index))
{
indexById[key] = entries.Count;
entries.Add((package.Id, package.VersionString));
}
else if (IsOlderVersion(package.VersionString, entries[index].Version))
{
entries[index] = (entries[index].Id, package.VersionString);
}
}

string prefix = $"{source.Url.ToString().Trim('/')}/GetUpdates()?packageIds=%27";
const string middle = "%27&versions=%27";
string suffix =
$"%27&includePrerelease={(canPrerelease ? "true" : "false")}&includeAllVersions=0";
string separator = HttpUtility.UrlEncode("|");
int fixedLength = prefix.Length + middle.Length + suffix.Length;

var urls = new List<string>();
var ids = new StringBuilder();
var versions = new StringBuilder();
int count = 0;

foreach (var (id, version) in entries)
{
string encodedId = HttpUtility.UrlEncode(id);
string encodedVersion = HttpUtility.UrlEncode(version);
int added =
encodedId.Length + encodedVersion.Length + (count > 0 ? 2 * separator.Length : 0);

if (
count > 0
&& fixedLength + ids.Length + versions.Length + added > MaxV2UpdateUrlLength
)
{
urls.Add(prefix + ids + middle + versions + suffix);
ids.Clear();
versions.Clear();
count = 0;
}

if (count > 0)
{
ids.Append(separator);
versions.Append(separator);
}
ids.Append(encodedId);
versions.Append(encodedVersion);
count++;
}

if (count > 0)
urls.Add(prefix + ids + middle + versions + suffix);

return urls;
}

private bool IsOlderVersion(string version, string other) =>
CompareVersions(version, other) is { } comparison
? comparison < 0
: CoreTools.VersionStringToStruct(version) < CoreTools.VersionStringToStruct(other);

internal IReadOnlyList<Package>? GetAvailableUpdatesV3(
IManagerSource source,
IReadOnlyList<IPackage> installedPackages,
Expand Down
178 changes: 178 additions & 0 deletions src/UniGetUI.PackageEngine.Tests/NuGetV2UpdatesTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
using System.Net;
using System.Text;
using UniGetUI.PackageEngine.Classes.Manager;
using UniGetUI.PackageEngine.Interfaces;
using UniGetUI.PackageEngine.ManagerClasses.Classes;
using UniGetUI.PackageEngine.ManagerClasses.Manager;
using UniGetUI.PackageEngine.Managers.PowerShellManager;
using UniGetUI.PackageEngine.PackageClasses;
using UniGetUI.PackageEngine.Tests.Infrastructure.Helpers;

namespace UniGetUI.PackageEngine.Tests;

public sealed class NuGetV2UpdatesTests
{
private const int FeedMaxUrlLength = 11000;

[Fact]
public void GetAvailableUpdatesReportsEveryModuleWhenTheInstalledListExceedsTheFeedUrlLimit()
{
using var feed = new V2UpdatesFeed();
var manager = feed.CreateManager(InstalledModules(450, duplicatedVersions: 60));

var updates = manager.GetAvailableUpdates();

Assert.Equal(450, updates.Count);
Assert.Equal(450, updates.Select(update => update.Id.ToLowerInvariant()).Distinct().Count());
Assert.All(updates, update => Assert.Equal("99.0.0", update.NewVersionString));
Assert.True(feed.Server.RequestPaths.Count > 1);
Assert.All(feed.RequestedUrlLengths, length => Assert.True(length <= FeedMaxUrlLength));
}

[Fact]
public void GetAvailableUpdatesKeepsTheOtherChunksWhenOneRequestFails()
{
using var feed = new V2UpdatesFeed { FailingPackageId = "Microsoft.Graph.Beta.Module000" };
var manager = feed.CreateManager(InstalledModules(450, duplicatedVersions: 0));

var updates = manager.GetAvailableUpdates();

Assert.NotEmpty(updates);
Assert.True(updates.Count < 450);
Assert.DoesNotContain(updates, update => update.Id == "Microsoft.Graph.Beta.Module000");
Assert.Contains(updates, update => update.Id == "Microsoft.Graph.Beta.Module449");
}

private static List<(string Id, string Version)> InstalledModules(int count, int duplicatedVersions)
{
var modules = new List<(string Id, string Version)>();
for (int i = 0; i < count; i++)
{
string id = $"Microsoft.Graph.Beta.Module{i:000}";
modules.Add((id, "2.25.0"));
if (i < duplicatedVersions)
modules.Add((id, "2.24.0"));
}

return modules;
}

private sealed class V2UpdatesFeed : IDisposable
{
private TestHttpServer? _server;
private readonly List<int> _requestedUrlLengths = [];

public TestHttpServer Server => _server ??= new TestHttpServer(Handle);

public string? FailingPackageId { get; init; }

public IReadOnlyList<int> RequestedUrlLengths
{
get
{
lock (_requestedUrlLengths)
{
return _requestedUrlLengths.ToArray();
}
}
}

public V2TestManager CreateManager(IReadOnlyList<(string Id, string Version)> installed)
{
var manager = new V2TestManager($"{Server.BaseUri.AbsoluteUri}api/v2/", installed);
manager.Initialize();
Assert.True(manager.IsReady());
return manager;
}

public void Dispose() => _server?.Dispose();

private (int StatusCode, string Content, string ContentType) Handle(
HttpListenerRequest request
)
{
string rawUrl = request.Url?.AbsoluteUri ?? string.Empty;
lock (_requestedUrlLengths)
{
_requestedUrlLengths.Add(rawUrl.Length);
}

if (!(request.Url?.AbsolutePath ?? string.Empty).EndsWith("/GetUpdates()"))
return (404, string.Empty, "application/xml");

if (rawUrl.Length > FeedMaxUrlLength)
return (404, string.Empty, "application/xml");

string[] ids = (request.QueryString["packageIds"] ?? string.Empty)
.Trim('\'')
.Split('|', StringSplitOptions.RemoveEmptyEntries);

if (FailingPackageId is not null && ids.Contains(FailingPackageId))
return (500, string.Empty, "application/xml");

var feed = new StringBuilder("<feed>");
foreach (string id in ids)
feed.Append($"<entry><d:Id>{id}</d:Id><d:Version>99.0.0</d:Version></entry>");
feed.Append("</feed>");

return (200, feed.ToString(), "application/atom+xml");
}
}

private sealed class V2TestManager : BaseNuGet
{
private readonly IReadOnlyList<(string Id, string Version)> _installed;

public V2TestManager(string sourceUrl, IReadOnlyList<(string Id, string Version)> installed)
{
_installed = installed;

Capabilities = new ManagerCapabilities
{
SupportsCustomVersions = true,
SupportsCustomPackageIcons = true,
CanListDependencies = true,
};

Properties = new ManagerProperties
{
Id = "test-nuget-v2",
Name = "TestNuGetV2",
DefaultSource = new ManagerSource(this, "test", new Uri(sourceUrl)),
};

DetailsHelper = new Helper(this);
}

protected override IReadOnlyList<Package> _getInstalledPackages_UnSafe() =>
_installed
.Select(module => new Package(
module.Id,
module.Id,
module.Version,
Properties.DefaultSource,
this
))
.ToList();

public override IReadOnlyList<string> FindCandidateExecutableFiles() => [];

protected override void _loadManagerExecutableFile(
out bool found,
out string executablePath,
out string callArgs
)
{
found = true;
executablePath = "test-nuget-v2";
callArgs = string.Empty;
}

protected override void _loadManagerVersion(out string version) => version = "0.0.0";

private sealed class Helper(BaseNuGet manager) : BaseNuGetDetailsHelper(manager)
{
protected override string? GetInstallLocation_UnSafe(IPackage package) => null;
}
}
}
Loading