Skip to content

Reduce updater download amplification #4936

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Gabriel Dufresne (GabrielDuf) merged 2 commits into from
Jun 16, 2026
+1,411 −44
Merged

Reduce updater download amplification #4936

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c75a994
Reduce redundant CDN downloads
mamoreau-devolutions Jun 16, 2026
649ec12
Reduce updater download amplification
mamoreau-devolutions Jun 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 129 additions & 16 deletions src/UniGetUI.Avalonia/Infrastructure/AvaloniaAutoUpdater.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ namespace UniGetUI.Avalonia.Infrastructure;
/// </summary>
internal static partial class AvaloniaAutoUpdater
{
private const string INSTALLER_VALIDATION_FAILURE_MARKER_SUFFIX = ".validation-failed";

// ------------------------------------------------------------------ constants
private const string REGISTRY_PATH = @"Software\Devolutions\UniGetUI";
private const string DEFAULT_PRODUCTINFO_URL = "https://devolutions.net/productinfo.json";
Expand Down Expand Up @@ -370,10 +372,10 @@ public static async Task UpdateCheckLoopAsync()
return;
}

bool success = await CheckAndInstallUpdatesAsync(autoLaunch: isFirstLaunch);
await CheckAndInstallUpdatesAsync(autoLaunch: isFirstLaunch);
isFirstLaunch = false;

await Task.Delay(TimeSpan.FromMinutes(success ? 60 : 10));
await Task.Delay(UpdaterDownloadEngine.DefaultAutomaticUpdateCheckInterval);
}
}

Expand Down Expand Up @@ -425,6 +427,11 @@ internal static async Task<bool> CheckAndInstallUpdatesAsync(bool autoLaunch = f
// macOS and Linux both ship as self-contained .tar.gz archives.
installerName = "UniGetUI Updater.tar.gz";
string installerPath = Path.Join(CoreData.UniGetUIDataDirectory, installerName);
UpdaterDownloadIdentity installerIdentity = new(
candidate.VersionName,
candidate.InstallerHash,
candidate.InstallerDownloadUrl
);

// Try cached installer first
if (
Expand All @@ -433,12 +440,44 @@ internal static async Task<bool> CheckAndInstallUpdatesAsync(bool autoLaunch = f
&& CheckInstallerSignerThumbprint(installerPath, overrides)
)
{
ClearInstallerValidationFailure(GetInstallerValidationFailureMarkerPath(installerPath));
UpdaterDownloadEngine.ClearFailureState(
UpdaterDownloadEngine.GetFailureStatePath(installerPath),
message => LogUpdateWarn(message)
);
LogUpdateInfo("Cached valid installer found, preparing to launch...");
return await PrepareAndLaunchAsync(installerPath, candidate.VersionName, autoLaunch, manualCheck);
}

// Delete invalid/outdated cached copy
try { File.Delete(installerPath); } catch { }
string installerValidationFailureMarkerPath =
GetInstallerValidationFailureMarkerPath(installerPath);
string installerDownloadFailureStatePath =
UpdaterDownloadEngine.GetFailureStatePath(installerPath);
if (File.Exists(installerPath))
{
LogUpdateWarn(
"Cached installer is invalid; it will be kept until a replacement is validated."
);
}

if (
!manualCheck
&& UpdaterDownloadEngine.IsFailureBackoffActive(
installerDownloadFailureStatePath,
installerIdentity,
DateTime.UtcNow,
out TimeSpan remainingBackoff,
out UpdaterDownloadFailureState? failureState,
message => LogUpdateWarn(message)
)
)
{
LogUpdateWarn(
$"Skipping installer download for version {candidate.VersionName}; previous {failureState?.FailureClass ?? "download"} failure is backed off for {remainingBackoff:g}."
);
MarkAttemptFinished("installer download skipped by failure backoff");
return true;
}

RaiseStatus(
CoreTools.Translate(
Expand All @@ -449,17 +488,59 @@ internal static async Task<bool> CheckAndInstallUpdatesAsync(bool autoLaunch = f
isClosable: false);

LogUpdateInfo("Downloading installer...");
await DownloadInstallerAsync(candidate.InstallerDownloadUrl, installerPath, overrides);
string downloadedInstallerPath;
try
{
downloadedInstallerPath = await DownloadInstallerAsync(
candidate.InstallerDownloadUrl,
installerPath,
overrides,
installerIdentity
);
}
catch (Exception ex)
{
UpdaterDownloadEngine.RecordFailure(
installerDownloadFailureStatePath,
installerIdentity,
"download",
DateTime.UtcNow,
message => LogUpdateWarn(message)
);
LogUpdateWarn($"Installer download failed: {ex.Message}");
throw;
}

if (
await CheckInstallerHashAsync(installerPath, candidate.InstallerHash, overrides)
&& CheckInstallerSignerThumbprint(installerPath, overrides)
await CheckInstallerHashAsync(downloadedInstallerPath, candidate.InstallerHash, overrides)
&& CheckInstallerSignerThumbprint(downloadedInstallerPath, overrides)
)
{
UpdaterDownloadEngine.PromotePartialDownload(
installerPath,
message => LogUpdateWarn(message)
);
ClearInstallerValidationFailure(installerValidationFailureMarkerPath);
UpdaterDownloadEngine.ClearFailureState(
installerDownloadFailureStatePath,
message => LogUpdateWarn(message)
);
LogUpdateInfo("Downloaded installer is valid, preparing to launch...");
return await PrepareAndLaunchAsync(installerPath, candidate.VersionName, autoLaunch, manualCheck);
}

UpdaterDownloadEngine.RecordFailure(
installerDownloadFailureStatePath,
installerIdentity,
"validation",
DateTime.UtcNow,
message => LogUpdateWarn(message)
);
UpdaterDownloadEngine.DeletePartialDownload(
installerPath,
message => LogUpdateWarn(message)
);

LogUpdateError("Installer authenticity could not be verified. Aborting update.");
RaiseStatus(
CoreTools.Translate("The installer authenticity could not be verified."),
Expand All @@ -469,7 +550,7 @@ await CheckInstallerHashAsync(installerPath, candidate.InstallerHash, overrides)
actionButtonText: CoreTools.Translate("View log"),
actionButtonAction: OpenUpdateLog);
MarkAttemptFinished("authenticity verification failed");
return false;
return !manualCheck;
}
catch (PlatformArtifactMissingException ex)
{
Expand Down Expand Up @@ -508,6 +589,33 @@ await CheckInstallerHashAsync(installerPath, candidate.InstallerHash, overrides)
}

// ------------------------------------------------------------------ update flow
internal static string GetInstallerValidationFailureMarkerPath(string installerPath)
{
return installerPath + INSTALLER_VALIDATION_FAILURE_MARKER_SUFFIX;
}

private static void ClearInstallerValidationFailure(string markerPath)
{
DeleteFileIfExists(markerPath, "installer validation failure marker");
}

private static void DeleteFileIfExists(string path, string description)
{
if (!File.Exists(path))
{
return;
}

try
{
File.Delete(path);
}
catch (Exception ex)
{
LogUpdateWarn($"Could not delete {description} at '{path}': {ex.Message}");
}
}

private static async Task<bool> PrepareAndLaunchAsync(
string installerPath,
string versionName,
Expand Down Expand Up @@ -1329,10 +1437,11 @@ private static bool VerifyMacAppSignature(string appBundlePath)
}

// ------------------------------------------------------------------ download
private static async Task DownloadInstallerAsync(
private static async Task<string> DownloadInstallerAsync(
string url,
string destination,
UpdaterOverrides overrides)
UpdaterOverrides overrides,
UpdaterDownloadIdentity identity)
{
if (!IsSourceUrlAllowed(url, overrides.AllowUnsafeUrls))
{
Expand All @@ -1343,14 +1452,18 @@ private static async Task DownloadInstallerAsync(
using HttpClient client = new(CreateHttpClientHandler(overrides));
client.Timeout = TimeSpan.FromSeconds(600);
client.DefaultRequestHeaders.UserAgent.ParseAdd(CoreData.UserAgentString);

HttpResponseMessage response = await client.GetAsync(url);
response.EnsureSuccessStatusCode();

using FileStream fs = new(destination, FileMode.OpenOrCreate);
await response.Content.CopyToAsync(fs);
using CancellationTokenSource downloadTimeout = new(TimeSpan.FromSeconds(600));

UpdaterDownloadResult result = await UpdaterDownloadEngine.DownloadInstallerPartAsync(
client,
identity,
destination,
message => LogUpdateDebug(message),
downloadTimeout.Token
);

LogUpdateDebug("Installer download complete.");
return result.PartialPath;
}

// ------------------------------------------------------------------ HTTP client
Expand Down
24 changes: 24 additions & 0 deletions src/UniGetUI.Core.IconEngine.Tests/IconDatabaseTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,29 @@ public async Task TestGetNonExistingIconAndImagesAsync()
);
Assert.Empty(nonexistent_screenshots);
}

[Fact]
public void TestCachedDatabaseFreshness()
{
string path = Path.Join(Path.GetTempPath(), Guid.NewGuid().ToString("N"));
try
{
File.WriteAllText(path, "{}");
DateTime now = DateTime.UtcNow;

File.SetLastWriteTimeUtc(path, now.AddHours(-23));
Assert.True(IconDatabase.IsCachedDatabaseFresh(path, now));

File.SetLastWriteTimeUtc(path, now.AddHours(-25));
Assert.False(IconDatabase.IsCachedDatabaseFresh(path, now));
}
finally
{
if (File.Exists(path))
{
File.Delete(path);
}
}
}
}
}
58 changes: 48 additions & 10 deletions src/UniGetUI.Core.IconStore/IconDatabase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ namespace UniGetUI.Core.IconEngine
/// </summary>
public class IconDatabase
{
private const string ICON_DATABASE_FILE_NAME = "Icon Database.json";
private static readonly TimeSpan ICON_DATABASE_REFRESH_INTERVAL = TimeSpan.FromDays(1);

public struct IconCount
{
public int PackagesWithIconCount = 0;
Expand Down Expand Up @@ -40,16 +43,31 @@ private Dictionary<
/// </summary>
public async Task LoadIconAndScreenshotsDatabaseAsync()
{
try
string IconsAndScreenshotsFile = GetIconsAndScreenshotsFile();
bool hasCustomDownloadUrl = Settings.Get(Settings.K.IconDataBaseURL);
if (
!hasCustomDownloadUrl
&& IsCachedDatabaseFresh(IconsAndScreenshotsFile, DateTime.UtcNow)
)
{
string IconsAndScreenshotsFile = Path.Join(
CoreData.UniGetUICacheDirectory_Data,
"Icon Database.json"
Logger.Debug("Using cached icons and screenshots database; refresh is not due yet");
await LoadFromCacheAsync();
if (__icon_count.PackagesWithIconCount > 0)
{
return;
}

Logger.Warn(
"Cached icons and screenshots database could not be loaded; refreshing it"
);
}

try
{
Uri DownloadUrl = new(
"https://github.com/Devolutions/UniGetUI/raw/refs/heads/main/WebBasedData/screenshot-database-v2.json"
);
if (Settings.Get(Settings.K.IconDataBaseURL))
if (hasCustomDownloadUrl)
{
DownloadUrl = new Uri(Settings.GetValue(Settings.K.IconDataBaseURL));
}
Expand All @@ -58,7 +76,7 @@ public async Task LoadIconAndScreenshotsDatabaseAsync()
{
client.DefaultRequestHeaders.UserAgent.ParseAdd(CoreData.UserAgentString);
string fileContents = await client.GetStringAsync(DownloadUrl);
await File.WriteAllTextAsync(IconsAndScreenshotsFile, fileContents);
await WriteCacheFileAsync(IconsAndScreenshotsFile, fileContents);
}

Logger.ImportantInfo("Downloaded new icons and screenshots successfully!");
Expand All @@ -79,14 +97,34 @@ public async Task LoadIconAndScreenshotsDatabaseAsync()
await LoadFromCacheAsync();
}

internal static bool IsCachedDatabaseFresh(string path, DateTime utcNow)
{
if (!File.Exists(path))
{
return false;
}

DateTime lastWriteTimeUtc = File.GetLastWriteTimeUtc(path);
return utcNow - lastWriteTimeUtc < ICON_DATABASE_REFRESH_INTERVAL;
}

private static string GetIconsAndScreenshotsFile()
{
return Path.Join(CoreData.UniGetUICacheDirectory_Data, ICON_DATABASE_FILE_NAME);
}

private static async Task WriteCacheFileAsync(string path, string contents)
{
string temporaryPath = path + ".tmp";
await File.WriteAllTextAsync(temporaryPath, contents);
File.Move(temporaryPath, path, overwrite: true);
}

public async Task LoadFromCacheAsync()
{
try
{
string IconsAndScreenshotsFile = Path.Join(
CoreData.UniGetUICacheDirectory_Data,
"Icon Database.json"
);
string IconsAndScreenshotsFile = GetIconsAndScreenshotsFile();
IconScreenshotDatabase_v2 JsonData =
IconStoreJson.DeserializeIconDatabase(
await File.ReadAllTextAsync(IconsAndScreenshotsFile)
Expand Down
3 changes: 3 additions & 0 deletions src/UniGetUI.Core.IconStore/InternalsVisibleTo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;

[assembly: InternalsVisibleTo("UniGetUI.Core.IconEngine.Tests")]
Loading
Loading