fix: preserve non-BMP unicode in templated config field rendering (SC-135815) (#5909)

Fields containing both repl{{}} template expressions and non-BMP unicode
characters (codepoints > U+FFFF such as emoji) previously crashed the
text/template parser with `unexpected "\\" in operand`. The root cause
is that gopkg.in/yaml.v2 considers non-BMP codepoints non-printable, so
it force-emits them as \UXXXXXXXX escapes inside double-quoted strings,
and double-quoting in turn escapes any " in the value as \" - which
breaks template parsing when \" lands inside a {{ ... }} delimiter.

Fix MarshalConfig with two passes:

1. normalizeTemplateStyles: re-parse the yaml.v2 output with yaml.v3,
   walk scalar nodes whose value contains repl{{ or {{repl, swap any
   non-BMP rune for an ASCII placeholder so the encoder picks single-
   quoted or literal-block style instead of double-quoted, re-emit, then
   restore the original UTF-8 runes.

2. decodeUnicodeEscapes: post-process the final string to decode any
   remaining \UXXXXXXXX (non-BMP), \uXXXX\uXXXX (UTF-16 surrogate pair),
   or \uXXXX (BMP) escapes back to their UTF-8 equivalents.

Tests cover:

  - The exact ticket repro: multi-line |- block scalar + repl{{}} +
    non-BMP emoji - asserts the new marshaller succeeds AND the old
    marshaller still fails (regression guard).
  - The customer-supplied workaround pattern (printf "\UXXXXXXXX" inside
    a repl template, ASCII-only source) - asserts both old and new
    marshallers produce the same correct rendered output, so the fix
    does not disturb the workaround.
  - decodeUnicodeEscapes unit tests covering 8-digit escapes, UTF-16
    surrogate pairs, BMP escapes, lone surrogates, and invalid codepoints.

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: jmboby

pkg/config/config.go

@@ -3,7 +3,12 @@ package config
 import (
 	"bytes"
 	"fmt"
+	"regexp"
+	"strconv"
 	"strings"
+	"unicode"
+	"unicode/utf16"
+	"unicode/utf8"
 
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
@@ -13,10 +18,21 @@ import (
 	kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1"
 	"github.com/replicatedhq/kotskinds/multitype"
 	"github.com/replicatedhq/kotskinds/pkg/licensewrapper"
+	goyaml "go.yaml.in/yaml/v3"
 	"k8s.io/apimachinery/pkg/runtime/serializer/json"
 	"k8s.io/client-go/kubernetes/scheme"
 )
 
+// Regexps for decoding Unicode escape sequences introduced by yaml.v2 marshalling.
+// yaml.v2 escapes non-BMP characters (codepoints > U+FFFF) as \UXXXXXXXX because its
+// isPrintable() excludes the 0x10000-0x10FFFF range. These escape sequences break
+// Go's text/template parser when they appear in config fields alongside repl{{}} expressions.
+var (
+	reUnicodeEscape8 = regexp.MustCompile(`\\U([0-9A-Fa-f]{8})`)
+	reSurrogatePair  = regexp.MustCompile(`\\u([Dd][89ABab][0-9A-Fa-f]{2})\\u([Dd][C-Fc-f][0-9A-Fa-f]{2})`)
+	reUnicodeEscape4 = regexp.MustCompile(`\\u([0-9A-Fa-f]{4})`)
+)
+
 func TemplateConfigObjects(configSpec *kotsv1beta1.Config, configValues map[string]template.ItemValue, license *licensewrapper.LicenseWrapper, app *kotsv1beta1.Application, localRegistry registrytypes.RegistrySettings, versionInfo *template.VersionInfo, appInfo *template.ApplicationInfo, identityconfig *kotsv1beta1.IdentityConfig, namespace string, decryptValues bool) (*kotsv1beta1.Config, error) {
 	templatedString, err := templateConfigObjects(configSpec, configValues, license, app, localRegistry, versionInfo, appInfo, identityconfig, namespace, decryptValues, MarshalConfig)
 	if err != nil {
@@ -156,7 +172,123 @@ func MarshalConfig(config *kotsv1beta1.Config) (string, error) {
 		return "", errors.Wrap(err, "failed to fix up yaml")
 	}
 
-	return string(b), nil
+	normalized, err := normalizeTemplateStyles(b)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to normalize template styles")
+	}
+
+	return decodeUnicodeEscapes(string(normalized)), nil
+}
+
+// decodeUnicodeEscapes replaces YAML/JSON Unicode escape sequences with their UTF-8 equivalents.
+// This is needed because the K8s serialiser routes through gopkg.in/yaml.v2, which escapes
+// non-BMP Unicode characters (codepoints > U+FFFF) as \UXXXXXXXX in double-quoted strings.
+// Processing order matters: \UXXXXXXXX first, then surrogate pairs, then standalone \uXXXX.
+func decodeUnicodeEscapes(s string) string {
+	// Decode \UXXXXXXXX (8-digit YAML escapes for non-BMP codepoints)
+	s = reUnicodeEscape8.ReplaceAllStringFunc(s, func(match string) string {
+		codepoint, err := strconv.ParseUint(match[2:], 16, 32)
+		if err != nil || !utf8.ValidRune(rune(codepoint)) {
+			return match
+		}
+		return string(rune(codepoint))
+	})
+
+	// Decode \uXXXX\uXXXX (JSON-style UTF-16 surrogate pairs)
+	s = reSurrogatePair.ReplaceAllStringFunc(s, func(match string) string {
+		high, err1 := strconv.ParseUint(match[2:6], 16, 16)
+		low, err2 := strconv.ParseUint(match[8:12], 16, 16)
+		if err1 != nil || err2 != nil {
+			return match
+		}
+		r := utf16.DecodeRune(rune(high), rune(low))
+		if r == unicode.ReplacementChar {
+			return match
+		}
+		return string(r)
+	})
+
+	// Decode standalone \uXXXX (BMP codepoints, skip surrogates)
+	s = reUnicodeEscape4.ReplaceAllStringFunc(s, func(match string) string {
+		codepoint, err := strconv.ParseUint(match[2:], 16, 16)
+		if err != nil || (codepoint >= 0xD800 && codepoint <= 0xDFFF) {
+			return match
+		}
+		if !utf8.ValidRune(rune(codepoint)) {
+			return match
+		}
+		return string(rune(codepoint))
+	})
+
+	return s
+}
+
+// reNonBMP matches any Unicode character above U+FFFF (non-BMP codepoints).
+var reNonBMP = regexp.MustCompile(`[\x{10000}-\x{10FFFF}]`)
+
+// normalizeTemplateStyles ensures YAML strings containing template expressions
+// (repl{{ or {{repl) are not double-quoted. The go.yaml.in/yaml/v3 library considers
+// non-BMP Unicode characters (> U+FFFF) non-printable, forcing double-quoted style
+// with \UXXXXXXXX escapes. This also introduces \" for any " in the value, which
+// breaks Go's text/template parser when \" appears inside template delimiters.
+//
+// The approach: temporarily replace non-BMP characters with ASCII placeholders so
+// the YAML encoder chooses single-quoted (or literal block) style, then restore
+// the original UTF-8 characters in the encoded output.
+func normalizeTemplateStyles(yamlBytes []byte) ([]byte, error) {
+	var doc goyaml.Node
+	if err := goyaml.Unmarshal(yamlBytes, &doc); err != nil {
+		return nil, err
+	}
+
+	// Track placeholder→original mappings
+	placeholders := map[string]string{}
+	walkYAMLNodes(&doc, func(node *goyaml.Node) {
+		if node.Kind != goyaml.ScalarNode {
+			return
+		}
+		if !strings.Contains(node.Value, "repl{{") && !strings.Contains(node.Value, "{{repl") {
+			return
+		}
+		// Replace non-BMP characters with ASCII placeholders
+		node.Value = reNonBMP.ReplaceAllStringFunc(node.Value, func(ch string) string {
+			r := []rune(ch)[0]
+			placeholder := fmt.Sprintf("__KOTS_U%08X__", r)
+			placeholders[placeholder] = ch
+			return placeholder
+		})
+		// Now the value is BMP-only, so the encoder can use non-double-quoted styles
+		if strings.Contains(node.Value, "\n") {
+			node.Style = goyaml.LiteralStyle
+		} else {
+			node.Style = goyaml.SingleQuotedStyle
+		}
+	})
+
+	var buf bytes.Buffer
+	enc := goyaml.NewEncoder(&buf)
+	enc.SetIndent(2)
+	if err := enc.Encode(&doc); err != nil {
+		return nil, err
+	}
+
+	// Restore original UTF-8 characters
+	result := buf.String()
+	for placeholder, original := range placeholders {
+		result = strings.ReplaceAll(result, placeholder, original)
+	}
+
+	return []byte(result), nil
+}
+
+func walkYAMLNodes(node *goyaml.Node, fn func(*goyaml.Node)) {
+	if node == nil {
+		return
+	}
+	fn(node)
+	for _, child := range node.Content {
+		walkYAMLNodes(child, fn)
+	}
 }
 
 func UnmarshalConfigValuesContent(content []byte) (map[string]template.ItemValue, error) {

pkg/config/config_test.go

@@ -341,6 +341,163 @@ spec:
           secretName-1: "123"
           secretName-2: "456"
           secretName-3: "789"
+`,
+			expectOldFail: false,
+		},
+		{
+			name: "non-BMP unicode in help_text with template function",
+			configSpecData: `
+apiVersion: kots.io/v1beta1
+kind: Config
+metadata:
+  name: test-app
+spec:
+  groups:
+    - name: example_settings
+      title: My Example Config
+      items:
+        - name: other_field
+          title: Other Field
+          type: text
+          default: "hello"
+        - name: a_field
+          title: A Field
+          type: bool
+          default: "1"
+          help_text: "🔏 This field is locked repl{{ ConfigOption \"other_field\" }}"`,
+			configValuesData: map[string]template.ItemValue{
+				"other_field": {
+					Value: "world",
+				},
+			},
+			useAppSpec: false,
+			want: `apiVersion: kots.io/v1beta1
+kind: Config
+metadata:
+  name: test-app
+spec:
+  groups:
+  - items:
+    - default: "hello"
+      name: other_field
+      title: Other Field
+      type: text
+      value: world
+    - default: "1"
+      help_text: "🔏 This field is locked world"
+      name: a_field
+      title: A Field
+      type: bool
+      value: ""
+    name: example_settings
+    title: My Example Config
+status: {}
+`,
+			expectOldFail: true,
+		},
+		{
+			name: "non-BMP unicode in multi-line block-scalar help_text with repl template (SC-135815)",
+			configSpecData: `
+apiVersion: kots.io/v1beta1
+kind: Config
+metadata:
+  name: test-app
+spec:
+  groups:
+    - name: test_group
+      title: Test
+      items:
+        - name: other_field
+          title: Other Field
+          type: bool
+          default: "1"
+        - name: test_field
+          title: Test Field
+          type: bool
+          default: "1"
+          help_text: |-
+            Some description text
+
+            🔏 **This field may not be edited once set**
+
+            repl{{ if (ne Sequence 0) }}
+            repl{{- if ConfigOptionEquals "other_field" "1" }}
+            ℹ️ A conditional note
+            repl{{- end }}
+            repl{{- end }}`,
+			configValuesData: map[string]template.ItemValue{
+				"other_field": {
+					Value: "1",
+				},
+			},
+			useAppSpec: false,
+			want: `apiVersion: kots.io/v1beta1
+kind: Config
+metadata:
+  name: test-app
+spec:
+  groups:
+  - items:
+    - default: "1"
+      name: other_field
+      title: Other Field
+      type: bool
+      value: "1"
+    - default: "1"
+      help_text: |-
+        Some description text
+
+        🔏 **This field may not be edited once set**
+      name: test_field
+      title: Test Field
+      type: bool
+      value: ""
+    name: test_group
+    title: Test
+status: {}
+`,
+			expectOldFail: true,
+		},
+		{
+			name: "customer workaround: printf escape in repl template renders to non-BMP emoji",
+			configSpecData: `
+apiVersion: kots.io/v1beta1
+kind: Config
+metadata:
+  name: sample-app
+spec:
+  groups:
+    - name: sample-group
+      title: Sample Title
+      description: |
+        sample description
+      items:
+        - name: dns_support_choice
+          title: DNS Support
+          type: text
+          required: true
+          help_text: |
+             repl{{printf "\U0001F512"}}`,
+			configValuesData: map[string]template.ItemValue{},
+			useAppSpec:       false,
+			want: `apiVersion: kots.io/v1beta1
+kind: Config
+metadata:
+  name: sample-app
+spec:
+  groups:
+  - description: |
+      sample description
+    items:
+    - help_text: "🔒"
+      name: dns_support_choice
+      required: true
+      title: DNS Support
+      type: text
+      value: ""
+    name: sample-group
+    title: Sample Title
+status: {}
 `,
 			expectOldFail: false,
 		},
@@ -493,3 +650,58 @@ func TestApplyValuesToConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestDecodeUnicodeEscapes(t *testing.T) {
+	tests := []struct {
+		name  string
+		input string
+		want  string
+	}{
+		{
+			name:  "8-digit non-BMP escape",
+			input: `\U0001F510`,
+			want:  "\U0001F510",
+		},
+		{
+			name:  "surrogate pair",
+			input: `\uD83D\uDD10`,
+			want:  "\U0001F510",
+		},
+		{
+			name:  "4-digit BMP escape",
+			input: `\u00E9`,
+			want:  "\u00E9",
+		},
+		{
+			name:  "no escapes unchanged",
+			input: "hello world",
+			want:  "hello world",
+		},
+		{
+			name:  "invalid codepoint left unchanged",
+			input: `\UFFFFFFFF`,
+			want:  `\UFFFFFFFF`,
+		},
+		{
+			name:  "lone surrogate left unchanged",
+			input: `\uD800`,
+			want:  `\uD800`,
+		},
+		{
+			name:  "mixed content preserved",
+			input: "text before \\U0001F510 text after",
+			want:  "text before \U0001F510 text after",
+		},
+		{
+			name:  "multiple escapes decoded",
+			input: `\U0001F510 and \U0001F512`,
+			want:  "\U0001F510 and \U0001F512",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := decodeUnicodeEscapes(tt.input)
+			require.Equal(t, tt.want, got)
+		})
+	}
+}