harden pgext catalog workflows

Author: Vonng

cli/cc_cate.go

@@ -474,12 +474,17 @@ func (g *CCCateGenerator) GenerateAllCategoryPages(ctx context.Context) error {
 		return fmt.Errorf("failed to generate category index: %w", err)
 	}
 
+	var failed []string
 	for _, cat := range g.Cache.Categories {
 		if err := g.GenerateCategoryPage(ctx, cat); err != nil {
 			logrus.Errorf("Failed to generate category page for %s: %v", cat.Name, err)
+			failed = append(failed, cat.Name)
 		}
 	}
 
-	logrus.Infof("Generated %d category pages", len(g.Cache.Categories))
+	logrus.Infof("Generated %d/%d category pages", len(g.Cache.Categories)-len(failed), len(g.Cache.Categories))
+	if len(failed) > 0 {
+		return fmt.Errorf("failed to generate %d category pages: %s", len(failed), strings.Join(failed, ", "))
+	}
 	return nil
 }

cli/cc_list.go

@@ -354,7 +354,7 @@ icon: fas fa-code
 			anchor = "cpp"
 		}
 
-		b.WriteString(fmt.Sprintf("\n--------\n\n"))
+		b.WriteString("\n--------\n\n")
 		b.WriteString(fmt.Sprintf("## %s {#%s}\n\n", lang, anchor))
 		b.WriteString(fmt.Sprintf("%s 语言编写的扩展（%d 个）\n\n", lang, len(exts)))
 

cli/cc_os.go

@@ -276,7 +276,6 @@ func (g *CCOSGenerator) generateOSMatrixCell(ospkg *OSPackageInfo, pg int, osNam
 	return fmt.Sprintf("%s %s %s %d", state, org, version, count)
 }
 
-
 // GenerateAllOSPages generates OS pages for all active OS distributions
 func (g *CCOSGenerator) GenerateAllOSPages(ctx context.Context) error {
 	osList, err := GetActiveOS(ctx)
@@ -337,10 +336,7 @@ func (g *CCOSGenerator) GenerateAllOSPages(ctx context.Context) error {
 
 	if len(failedOS) > 0 {
 		logrus.Warnf("Failed to generate pages for: %v", failedOS)
-	}
-
-	if successCount == 0 {
-		return fmt.Errorf("failed to generate any OS pages")
+		return fmt.Errorf("failed to generate %d/%d OS pages: %s", len(failedOS), len(osList), strings.Join(failedOS, ", "))
 	}
 	return nil
 }

cli/cc_page.go

@@ -33,7 +33,10 @@ func NewCCPageGenerator(cache *ExtensionCache, outputDir, stubDir string) *CCPag
 
 // GenerateExtensionPage generates a single extension page (Chinese only)
 func (g *CCPageGenerator) GenerateExtensionPage(ctx context.Context, ext *Extension) error {
-	content := g.generateExtensionContent(ctx, ext)
+	content, err := g.generateExtensionContent(ctx, ext)
+	if err != nil {
+		return err
+	}
 
 	// Append stub content if exists
 	stubPath := filepath.Join(g.StubDir, ext.Name+".md")
@@ -45,11 +48,17 @@ func (g *CCPageGenerator) GenerateExtensionPage(ctx context.Context, ext *Extens
 }
 
 // generateExtensionContent generates the markdown content
-func (g *CCPageGenerator) generateExtensionContent(ctx context.Context, ext *Extension) string {
+func (g *CCPageGenerator) generateExtensionContent(ctx context.Context, ext *Extension) (string, error) {
 	var b strings.Builder
 
-	packages, _ := LoadPackages(ctx, ext.Pkg)
-	binaries, _ := LoadBinaries(ctx, ext.Name)
+	packages, err := LoadPackages(ctx, ext.Pkg)
+	if err != nil {
+		return "", fmt.Errorf("load packages for %s: %w", ext.Name, err)
+	}
+	binaries, err := LoadBinaries(ctx, ext.Name)
+	if err != nil {
+		return "", fmt.Errorf("load binaries for %s: %w", ext.Name, err)
+	}
 	siblings := g.Cache.GetSiblingExtensions(ext.Pkg, ext.Name)
 	allExts := g.getAllPackageExtensions(ext, siblings)
 
@@ -71,7 +80,7 @@ func (g *CCPageGenerator) generateExtensionContent(ctx context.Context, ext *Ext
 
 	b.WriteString(g.generateInstall(ext))
 
-	return b.String()
+	return b.String(), nil
 }
 
 // getAllPackageExtensions returns all extensions for the package (self + siblings), sorted by ID

cli/confgen.go

@@ -35,6 +35,7 @@ func GeneratePigstyConfig(osName string, outputDir string, dryRun bool, verbose
 	if err != nil {
 		return fmt.Errorf("failed to get SQL DB: %w", err)
 	}
+	defer sqlDB.Close()
 
 	// Create generator
 	generator := NewPigstyConfigGenerator(sqlDB, osName, verbose)
@@ -886,6 +887,7 @@ func GetConfigConstants() *ConfigConstants {
 			{"pg_exporter", "pg_exporter", "pg-exporter"},
 			{"pgbackrest_exporter", "pgbackrest_exporter", "pgbackrest-exporter"},
 			{"vip-manager", "vip-manager", "vip-manager"},
+			{"pg-hardstorage", "pg-hardstorage", "pg-hardstorage"},
 			{"pg_hardstorage", "pg-hardstorage", "pg-hardstorage"},
 			{"pgbadger", "pgbadger", "pgbadger"},
 			{"pg_activity", "pg_activity", "pg-activity"},
@@ -1132,7 +1134,7 @@ repo_upstream_default:
   - { name: docker-ce      ,description: 'Docker'             ,module: infra   ,releases: [11,12,13,22,24,26] ,arch: [x86_64, aarch64] ,baseurl: { default: 'https://download.docker.com/linux/${distro_name} ${distro_codename} stable'                               ,china: 'https://mirrors.aliyun.com/docker-ce/linux/${distro_name} ${distro_codename} stable' }}
   - { name: base           ,description: 'Debian Basic'       ,module: node    ,releases: [11,12,13         ] ,arch: [x86_64, aarch64] ,baseurl: { default: 'http://deb.debian.org/debian/ ${distro_codename} main non-free-firmware'                                  ,china: 'https://mirrors.aliyun.com/debian/ ${distro_codename} main non-free-firmware' }}
   - { name: updates        ,description: 'Debian Updates'     ,module: node    ,releases: [11,12,13         ] ,arch: [x86_64, aarch64] ,baseurl: { default: 'http://deb.debian.org/debian/ ${distro_codename}-updates main non-free-firmware'                          ,china: 'https://mirrors.aliyun.com/debian/ ${distro_codename}-updates main non-free-firmware' }}
-  - { name: security       ,description: 'Debian Securitcy'   ,module: node    ,releases: [11,12,13         ] ,arch: [x86_64, aarch64] ,baseurl: { default: 'http://security.debian.org/debian-security ${distro_codename}-security main non-free-firmware'            ,china: 'https://mirrors.aliyun.com/debian-security/ ${distro_codename}-security main non-free-firmware' }}
+  - { name: security       ,description: 'Debian Security'    ,module: node    ,releases: [11,12,13         ] ,arch: [x86_64, aarch64] ,baseurl: { default: 'http://security.debian.org/debian-security ${distro_codename}-security main non-free-firmware'            ,china: 'https://mirrors.aliyun.com/debian-security/ ${distro_codename}-security main non-free-firmware' }}
   - { name: base           ,description: 'Ubuntu Basic'       ,module: node    ,releases: [         22,24,26] ,arch: [x86_64         ] ,baseurl: { default: 'https://mirrors.edge.kernel.org/ubuntu/ ${distro_codename}           main universe multiverse restricted' ,china: 'https://mirrors.aliyun.com/ubuntu/ ${distro_codename}           main restricted universe multiverse' }}
   - { name: updates        ,description: 'Ubuntu Updates'     ,module: node    ,releases: [         22,24,26] ,arch: [x86_64         ] ,baseurl: { default: 'https://mirrors.edge.kernel.org/ubuntu/ ${distro_codename}-updates   main restricted universe multiverse' ,china: 'https://mirrors.aliyun.com/ubuntu/ ${distro_codename}-updates   main restricted universe multiverse' }}
   - { name: backports      ,description: 'Ubuntu Backports'   ,module: node    ,releases: [         22,24,26] ,arch: [x86_64         ] ,baseurl: { default: 'https://mirrors.edge.kernel.org/ubuntu/ ${distro_codename}-backports main restricted universe multiverse' ,china: 'https://mirrors.aliyun.com/ubuntu/ ${distro_codename}-backports main restricted universe multiverse' }}

cli/confgen_test.go

@@ -1038,8 +1038,7 @@ func TestDebTemplateRepoReleaseCompatibility(t *testing.T) {
 
 func TestRPMTemplateEL10RepoReleaseCompatibility(t *testing.T) {
 	required := []string{
-		"name: timescaledb    ,description: 'TimescaleDB'        ,module: extra   ,releases: [8,9   ] ,arch: [x86_64, aarch64]",
-		"name: timescaledb    ,description: 'TimescaleDB'        ,module: extra   ,releases: [   10] ,arch: [x86_64         ]",
+		"name: timescaledb    ,description: 'TimescaleDB'        ,module: extra   ,releases: [8,9,10] ,arch: [x86_64, aarch64]",
 		"name: mysql          ,description: 'MySQL'              ,module: mysql   ,releases: [8,9,10] ,arch: [x86_64, aarch64]",
 		"name: mongo          ,description: 'MongoDB'            ,module: mongo   ,releases: [8,9,10] ,arch: [x86_64, aarch64]",
 		"name: gitlab-ee      ,description: 'Gitlab EE'          ,module: gitlab  ,releases: [8,9,10] ,arch: [x86_64, aarch64]",
@@ -1052,7 +1051,8 @@ func TestRPMTemplateEL10RepoReleaseCompatibility(t *testing.T) {
 	}
 
 	forbidden := []string{
-		"name: timescaledb    ,description: 'TimescaleDB'        ,module: extra   ,releases: [8,9,10] ,arch: [x86_64, aarch64]",
+		"name: timescaledb    ,description: 'TimescaleDB'        ,module: extra   ,releases: [8,9   ] ,arch: [x86_64, aarch64]",
+		"name: timescaledb    ,description: 'TimescaleDB'        ,module: extra   ,releases: [   10] ,arch: [x86_64         ]",
 	}
 	for _, fragment := range forbidden {
 		if strings.Contains(rpmTemplate, fragment) {

cli/fetcher.go

@@ -49,10 +49,11 @@ type FetchResult struct {
 
 // FetchOptions contains options for fetch operation
 type FetchOptions struct {
-	Force    bool   // Force re-download all repos
-	Region   string // Region for mirror selection (default|china)
-	Parallel int    // Number of parallel workers
-	Retry    int    // Number of retry attempts
+	Force      bool   // Force re-download all repos
+	Region     string // Region for mirror selection (default|china)
+	Parallel   int    // Number of parallel workers
+	Retry      int    // Number of retry attempts
+	BestEffort bool   // Allow a partial result when at least one repository succeeds
 }
 
 // Fetcher handles repository metadata fetching
@@ -62,6 +63,7 @@ type Fetcher struct {
 	force      bool
 	maxWorkers int
 	retries    int
+	bestEffort bool
 }
 
 // NewFetcher creates a new repository fetcher
@@ -93,6 +95,7 @@ func NewFetcher(opts FetchOptions) *Fetcher {
 		force:      opts.Force,
 		maxWorkers: opts.Parallel,
 		retries:    opts.Retry,
+		bestEffort: opts.BestEffort,
 	}
 }
 
@@ -116,48 +119,49 @@ func (f *Fetcher) FetchAll(ctx context.Context) error {
 	results := f.fetchConcurrent(ctx, repos)
 
 	// Process results
-	var updated, skipped, failed int
-	var errors []string
+	report := RunReport{Operation: "fetch", Total: len(repos)}
 
 	for _, result := range results {
 		if result.Error != nil {
-			failed++
-			errors = append(errors, fmt.Sprintf("%s: %v", result.Repository.ID, result.Error))
+			report.AddFailure(result.Repository.ID, result.Error)
 			continue
 		}
 
 		if result.Updated {
 			if err := f.saveMetadata(ctx, result); err != nil {
-				failed++
-				errors = append(errors, fmt.Sprintf("%s: save failed: %v", result.Repository.ID, err))
+				report.AddFailure(result.Repository.ID, fmt.Errorf("save metadata: %w", err))
 			} else {
-				updated++
+				report.Succeeded++
 				logrus.Debugf("Updated %s (%s)", result.Repository.ID, formatBytes(int64(len(result.Data))))
 			}
 		} else {
-			skipped++
+			report.Skipped++
 			logrus.Debugf("Skipped %s (not modified)", result.Repository.ID)
 		}
 	}
 
-	// Update status timestamp
-	if updated > 0 {
-		if err := f.updateFetchTime(ctx); err != nil {
-			logrus.Warnf("Failed to update fetch time: %v", err)
-		}
-	}
-
 	// Log summary
-	logrus.Infof("Fetch complete: %d updated, %d skipped, %d failed", updated, skipped, failed)
+	logrus.Infof("Fetch complete: %d updated, %d skipped, %d failed",
+		report.Succeeded, report.Skipped, report.Failed())
 
 	// Report errors
-	for _, errMsg := range errors {
-		logrus.Error(errMsg)
+	for _, failure := range report.Failures {
+		logrus.Errorf("%s: %v", failure.Item, failure.Err)
 	}
 
-	// Fail if all repositories failed
-	if failed == len(repos) {
-		return fmt.Errorf("all repositories failed to fetch")
+	if err := report.Err(f.bestEffort); err != nil {
+		return err
+	}
+	if report.Failed() > 0 {
+		logrus.Warnf("best-effort fetch accepted %d failed repositories", report.Failed())
+	}
+
+	// Advance the accepted-run timestamp only after strict/best-effort policy
+	// has approved the result.
+	if report.Succeeded > 0 {
+		if err := f.updateFetchTime(ctx); err != nil {
+			logrus.Warnf("Failed to update fetch time: %v", err)
+		}
 	}
 
 	return nil
@@ -195,8 +199,7 @@ func (f *Fetcher) loadRepositories(ctx context.Context) ([]*RepoMetadata, error)
 			&repo.CachedSize,
 		)
 		if err != nil {
-			logrus.Warnf("Failed to scan repository: %v", err)
-			continue
+			return nil, fmt.Errorf("scan repository metadata: %w", err)
 		}
 		repos = append(repos, repo)
 	}

cli/gen_list.go

@@ -787,12 +787,7 @@ func (g *ListGenerator) GenerateLicenseList(locale, outputPath string) error {
 		licenses = append(licenses, licenseItem{name, exts, order})
 	}
 
-	sort.Slice(licenses, func(i, j int) bool {
-		if len(licenses[i].exts) != len(licenses[j].exts) {
-			return len(licenses[i].exts) > len(licenses[j].exts)
-		}
-		return licenses[i].order < licenses[j].order
-	})
+	sortLicenseItems(licenses)
 
 	var b strings.Builder
 
@@ -841,6 +836,24 @@ func (g *ListGenerator) GenerateLicenseList(locale, outputPath string) error {
 	return os.WriteFile(outputPath, []byte(b.String()), 0644)
 }
 
+func sortLicenseItems(licenses []licenseItem) {
+	sort.Slice(licenses, func(i, j int) bool {
+		if len(licenses[i].exts) != len(licenses[j].exts) {
+			return len(licenses[i].exts) > len(licenses[j].exts)
+		}
+		if licenses[i].order != licenses[j].order {
+			return licenses[i].order < licenses[j].order
+		}
+
+		left := strings.ToLower(licenses[i].name)
+		right := strings.ToLower(licenses[j].name)
+		if left != right {
+			return left < right
+		}
+		return licenses[i].name < licenses[j].name
+	})
+}
+
 func (g *ListGenerator) generateLicenseSection(license string, extensions []*Extension, isZh bool) string {
 	var b strings.Builder
 

cli/gen_list_test.go

@@ -0,0 +1,71 @@
+package cli
+
+import (
+	"bytes"
+	"database/sql"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestSortLicenseItemsUsesStableNameTieBreak(t *testing.T) {
+	oneExtension := []*Extension{{}}
+	items := []licenseItem{
+		{name: "Zulu", exts: oneExtension, order: 999},
+		{name: "beta", exts: oneExtension, order: 999},
+		{name: "Alpha", exts: oneExtension, order: 999},
+		{name: "alpha", exts: oneExtension, order: 999},
+	}
+
+	sortLicenseItems(items)
+
+	got := make([]string, len(items))
+	for i, item := range items {
+		got[i] = item.name
+	}
+	if want := "Alpha,alpha,beta,Zulu"; strings.Join(got, ",") != want {
+		t.Fatalf("license order = %q, want %q", strings.Join(got, ","), want)
+	}
+}
+
+func TestGenerateLicenseListIsDeterministicForEqualRankLicenses(t *testing.T) {
+	extensions := []*Extension{
+		{ID: 4, Name: "zulu_ext", Pkg: "zulu_ext", License: sql.NullString{String: "Zulu", Valid: true}},
+		{ID: 3, Name: "beta_ext", Pkg: "beta_ext", License: sql.NullString{String: "beta", Valid: true}},
+		{ID: 2, Name: "alpha_upper_ext", Pkg: "alpha_upper_ext", License: sql.NullString{String: "Alpha", Valid: true}},
+		{ID: 1, Name: "alpha_lower_ext", Pkg: "alpha_lower_ext", License: sql.NullString{String: "alpha", Valid: true}},
+	}
+
+	generate := func(name string, exts []*Extension) []byte {
+		t.Helper()
+		path := filepath.Join(t.TempDir(), name+".md")
+		generator := NewListGenerator(&ExtensionCache{Extensions: exts}, filepath.Dir(path))
+		if err := generator.GenerateLicenseList("en", path); err != nil {
+			t.Fatalf("generate %s: %v", name, err)
+		}
+		content, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatalf("read %s: %v", name, err)
+		}
+		return content
+	}
+
+	forward := generate("forward", extensions)
+	reverse := generate("reverse", []*Extension{extensions[3], extensions[2], extensions[1], extensions[0]})
+	if !bytes.Equal(forward, reverse) {
+		t.Fatal("license output changed when extension input order changed")
+	}
+
+	previous := -1
+	for _, section := range []string{"Alpha", "alpha", "beta", "Zulu"} {
+		index := bytes.Index(forward, []byte("\n## "+section+"\n"))
+		if index < 0 {
+			t.Fatalf("generated output is missing %s section", section)
+		}
+		if index <= previous {
+			t.Fatalf("%s section appears out of deterministic name order", section)
+		}
+		previous = index
+	}
+}

cli/gen_page.go

@@ -108,7 +108,8 @@ func (g *ExtensionGenerator) generateExtensionContent(ext *Extension, packages [
 func (g *ExtensionGenerator) generateFrontmatter(ext *Extension) string {
 	categoryTitle := "Unknown"
 	if ext.Category.Valid {
-		categoryTitle = strings.Title(ext.Category.String)
+		// Category codes are canonical display labels (for example RAG, GIS).
+		categoryTitle = ext.Category.String
 	}
 
 	desc := ext.EnDesc.String