Spread rebalanced parts across same-filesystem JBOD disks

least_used picked the disk with the largest free space from one
system.disks snapshot. JBOD disks on the same filesystem report almost
equal free space, and the difference is statvfs noise from concurrent
writes, so all parts of a download could land on a single disk.
TestForceRebalance flaked on CI this way (25.3, no files on hdd2).

Local disks whose free space is within 1% of total space of the least
used one are now treated as equally used and get parts round-robin.
Disks with really different free space keep least_used behavior.

Also surface find output in TestForceRebalance assertions, the CI log
showed only "exit status 1".

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: Slach

pkg/backup/download.go

@@ -497,6 +497,7 @@ func (b *Backuper) saveLocalBackupMetadata(backupName string, backupMetadata met
 
 func (b *Backuper) reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDownload ListOfTables, disks []clickhouse.Disk, remoteBackup storage.Backup) error {
 	var disksByStoragePolicyAndType map[string]map[string][]clickhouse.Disk
+	assignedParts := map[string]int{}
 	filterDisksByTypeAndStoragePolicies := func(disk string, diskType string, disks []clickhouse.Disk, remoteBackup storage.Backup, t metadata.TableMetadata) (string, []clickhouse.Disk, error) {
 		_, ok := remoteBackup.DiskTypes[disk]
 		if !ok {
@@ -568,6 +569,7 @@ func (b *Backuper) reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDown
 					if t.Parts[disk][j].RebalancedDisk != "" {
 						existingDisk := t.Parts[disk][j].RebalancedDisk
 						rebalancedDisks[existingDisk] = struct{}{}
+						assignedParts[existingDisk]++
 						isRebalanced = true
 						// Rebuild RebalancedFiles map for already-rebalanced parts
 						if t.Files != nil && len(t.Files[disk]) > 0 {
@@ -582,7 +584,7 @@ func (b *Backuper) reBalanceTablesMetadataIfDiskNotExists(tableMetadataAfterDown
 						}
 						continue
 					}
-					isObjectDisk, downloadDisk, newFreeSpace, reBalanceErr := b.getDownloadDiskForNonExistsDisk(diskType, filteredDisks, partSize)
+					isObjectDisk, downloadDisk, newFreeSpace, reBalanceErr := b.getDownloadDiskForNonExistsDisk(diskType, filteredDisks, partSize, assignedParts)
 					if reBalanceErr != nil {
 						return errors.Wrap(reBalanceErr, "getDownloadDiskForNonExistsDisk")
 					}
@@ -2417,23 +2419,38 @@ func (b *Backuper) splitDisksByTypeAndStoragePolicy(disks []clickhouse.Disk) map
 // getDownloadDiskForNonExistsDisk - https://github.com/Altinity/clickhouse-backup/issues/561
 // allow to Restore to new server with different storage policy, different disk count,
 // implements `least_used` for normal disk and `random` for Object disks
-func (b *Backuper) getDownloadDiskForNonExistsDisk(notExistsDiskType string, filteredDisks []clickhouse.Disk, partSize uint64) (bool, string, uint64, error) {
+// local disks whose free space differs from the least used one by less than 1% of total space are treated as
+// equally used (JBOD disks on the same filesystem, the difference is statvfs noise from concurrent writes),
+// parts are distributed across them round-robin by assignedParts
+func (b *Backuper) getDownloadDiskForNonExistsDisk(notExistsDiskType string, filteredDisks []clickhouse.Disk, partSize uint64, assignedParts map[string]int) (bool, string, uint64, error) {
 	// random for non-local disks
 	if notExistsDiskType != "local" {
 		roundRobinIdx := rand.Intn(len(filteredDisks))
 		return true, filteredDisks[roundRobinIdx].Name, 0, nil
 	}
 	// least_used for local
-	freeSpace := partSize
+	maxFreeSpace := uint64(0)
+	maxTotalSpace := uint64(0)
+	for _, d := range filteredDisks {
+		if d.FreeSpace > partSize && d.FreeSpace > maxFreeSpace {
+			maxFreeSpace = d.FreeSpace
+		}
+		maxTotalSpace = max(maxTotalSpace, d.TotalSpace)
+	}
+	if maxFreeSpace == 0 {
+		return false, "", 0, errors.Errorf("%s free space, not found in system.disks with `local` type", utils.FormatBytes(partSize))
+	}
+	tolerance := maxTotalSpace / 100
 	leastUsedIdx := -1
 	for idx, d := range filteredDisks {
-		if filteredDisks[idx].FreeSpace > freeSpace {
-			freeSpace = d.FreeSpace
+		if d.FreeSpace <= partSize || d.FreeSpace+tolerance < maxFreeSpace {
+			continue
+		}
+		if leastUsedIdx < 0 || assignedParts[d.Name] < assignedParts[filteredDisks[leastUsedIdx].Name] ||
+			(assignedParts[d.Name] == assignedParts[filteredDisks[leastUsedIdx].Name] && d.FreeSpace > filteredDisks[leastUsedIdx].FreeSpace) {
 			leastUsedIdx = idx
 		}
 	}
-	if leastUsedIdx < 0 {
-		return false, "", 0, errors.Errorf("%s free space, not found in system.disks with `local` type", utils.FormatBytes(partSize))
-	}
+	assignedParts[filteredDisks[leastUsedIdx].Name]++
 	return false, filteredDisks[leastUsedIdx].Name, filteredDisks[leastUsedIdx].FreeSpace - partSize, nil
 }

pkg/backup/download_test.go

@@ -234,6 +234,50 @@ func TestReBalanceTablesMetadataIfDiskNotExists_Parts_NoErrors(t *testing.T) {
 	}
 }
 
+// JBOD disks on the same filesystem report almost the same free space, the difference is statvfs noise
+// from concurrent writes, parts shall be distributed across all of them instead of landing on one disk
+func TestReBalanceTablesMetadataIfDiskNotExists_SameFilesystemDisks(t *testing.T) {
+	fb := Backuper{
+		DiskToPathMap: map[string]string{"default": "/var/lib/clickhouse"},
+		cfg:           &config.Config{ClickHouse: config.ClickHouseConfig{ForceRebalance: true}},
+	}
+	const totalSpace = 100 * 1024 * 1024 * 1024
+	sameFSDisks := []clickhouse.Disk{
+		{Name: "hdd1", Path: "/hdd1_data", Type: "local", FreeSpace: 50*1024*1024*1024 + 10*1024*1024, TotalSpace: totalSpace, StoragePolicies: []string{"default"}},
+		{Name: "hdd2", Path: "/hdd2_data", Type: "local", FreeSpace: 50 * 1024 * 1024 * 1024, TotalSpace: totalSpace, StoragePolicies: []string{"default"}},
+	}
+	table := metadata.TableMetadata{
+		Parts: map[string][]metadata.Part{
+			"default": {{Name: "202401_1_1_0"}, {Name: "202402_2_2_0"}, {Name: "202403_3_3_0"}, {Name: "202404_4_4_0"}},
+		},
+		Database:   "default",
+		Table:      "test",
+		Query:      "CREATE TABLE default.test(id UInt64) ENGINE=MergeTree() ORDER BY id",
+		TotalBytes: 4 * 8192,
+		LocalFile:  "/dev/null",
+	}
+	backup := storage.Backup{BackupMetadata: metadata.BackupMetadata{BackupName: "test", DataFormat: "directory", DiskTypes: map[string]string{"default": "local"}}}
+	require.NoError(t, fb.reBalanceTablesMetadataIfDiskNotExists(ListOfTables{&table}, sameFSDisks, backup))
+	partsByDisk := map[string]int{}
+	for _, p := range table.Parts["default"] {
+		partsByDisk[p.RebalancedDisk]++
+	}
+	assert.Equal(t, map[string]int{"hdd1": 2, "hdd2": 2}, partsByDisk)
+
+	// disks with really different free space keep `least_used` behavior
+	differentDisks := []clickhouse.Disk{
+		{Name: "hdd1", Path: "/hdd1_data", Type: "local", FreeSpace: 80 * 1024 * 1024 * 1024, TotalSpace: totalSpace, StoragePolicies: []string{"default"}},
+		{Name: "hdd2", Path: "/hdd2_data", Type: "local", FreeSpace: 20 * 1024 * 1024 * 1024, TotalSpace: totalSpace, StoragePolicies: []string{"default"}},
+	}
+	for j := range table.Parts["default"] {
+		table.Parts["default"][j].RebalancedDisk = ""
+	}
+	require.NoError(t, fb.reBalanceTablesMetadataIfDiskNotExists(ListOfTables{&table}, differentDisks, backup))
+	for _, p := range table.Parts["default"] {
+		assert.Equal(t, "hdd1", p.RebalancedDisk)
+	}
+}
+
 func TestReBalanceTablesMetadataIfDiskNotExists_CheckErrors(t *testing.T) {
 	invalidRemoteBackup := remoteBackup
 	invalidRemoteBackup.DataFormat = DirectoryFormat

test/integration/forceRebalance_test.go

@@ -107,9 +107,9 @@ XML
 
 	// Step 6: Verify backup data landed on both hdd1 and hdd2
 	hdd1Out, err := env.DockerExecOut("clickhouse-backup", "find", "/hdd1_data/backup/"+backupName+"/shadow", "-type", "f")
-	r.NoError(err)
+	r.NoError(err, hdd1Out)
 	hdd2Out, err := env.DockerExecOut("clickhouse-backup", "find", "/hdd2_data/backup/"+backupName+"/shadow", "-type", "f")
-	r.NoError(err)
+	r.NoError(err, hdd2Out)
 	log.Debug().Msgf("hdd1 files:\n%s", hdd1Out)
 	log.Debug().Msgf("hdd2 files:\n%s", hdd2Out)
 	r.NotEmpty(hdd1Out, "hdd1 should contain some backup data")