fix: mirror upstream toPOSTag for the accepted tag set

ParsePOSType is the Go counterpart of upstream's toPOSTag, so derive the
accepted strings from that function instead of inventing them.

The previous commit mapped POS_PA, POS_PV_I and POS_PA_I to "@". That
string is not something upstream ever converts to or from: tagToString
guards its table with assert(t < POSTag::max) and pa == max, so reading
"@" only happens in an NDEBUG build past the assertion, and toPOSTag has
no entry for "@" at all. Drop the three constants rather than exposing a
sentinel as a tag.

Restore POS_V and add the remaining strings toPOSTag accepts but Kiwi
never emits: A (POSTag::p), NF, NV, NA and UNK (POSTag::unknown), and ^
(POSTag::unknown, spelled POS_CARET). POS_V had been removed on the
grounds that Kiwi never returns "V", which is true of tagToString but
irrelevant to a parser -- toPOSTag accepts it.

kiwigo now accepts exactly toPOSTag's 75 strings plus "P", which
tagToString can return for POSTag::p even though toPOSTag does not
accept it. Emit the two groups as separate blocks so the generated
alignment matches gofmt.

Author: kkweon

postype.go

@@ -45,12 +45,27 @@ func TestParsePOSType(t *testing.T) {
 			want:    POS_VV_I,
 			wantErr: false,
 		},
+		// toPOSTag accepts these as input even though Kiwi never emits them.
 		{
-			name:    "@ is a POSType",
-			arg:     "@",
-			want:    POS_PA,
+			name:    "V is a POSType",
+			arg:     "V",
+			want:    POS_V,
 			wantErr: false,
 		},
+		{
+			name:    "UNK is a POSType",
+			arg:     "UNK",
+			want:    POS_UNK,
+			wantErr: false,
+		},
+		// toPOSTag has no entry for "@"; it is the sentinel tagToString falls
+		// back to for values it is never meant to be called with.
+		{
+			name:    "@ is not a valid POSType",
+			arg:     "@",
+			want:    POS_UNKNOWN,
+			wantErr: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

postype_test.go

@@ -33,6 +33,7 @@
 # GitHub raw content URL template
 KIWI_TYPES_H_URL = "https://raw.githubusercontent.com/bab2min/Kiwi/{version}/include/kiwi/Types.h"
 KIWI_UTILS_CPP_URL = "https://raw.githubusercontent.com/bab2min/Kiwi/{version}/src/Utils.cpp"
+KIWI_STRUTILS_H_URL = "https://raw.githubusercontent.com/bab2min/Kiwi/{version}/src/StrUtils.h"
 
 # Enumerators that are not POS tags themselves.
 SKIPPED_ENUMERATORS = {
@@ -44,6 +45,12 @@
 # Bases whose `<base>i` spelling maps to a POS_<BASE>_I constant.
 IRREGULAR_BASES = ("vv", "va", "vx", "xsa", "pv", "pa")
 
+# Go constant names for tag strings that toPOSTag accepts but that are not
+# spelled like an identifier.
+PUNCTUATION_TAG_NAMES = {
+    "^": "POS_CARET",
+}
+
 # Backwards-compatible aliases kept for downstream code. These cannot be derived
 # from the C++ source; they exist because kiwigo used to spell them this way.
 COMPAT_ALIASES = [
@@ -157,18 +164,37 @@ def extract_case_returns(body: str) -> dict[str, str]:
     return dict(re.findall(pattern, body))
 
 
+def extract_accepted_tags(body: str) -> list[str]:
+    """Extract the tag strings that `toPOSTag` accepts, in source order.
+
+    toPOSTag is the upstream counterpart of ParsePOSType, so it defines which
+    strings are valid input. Anything it does not recognise falls through to
+    `return POSTag::max`, which is upstream's way of rejecting the string.
+    """
+    pattern = r'tagStr\s*==\s*u"([^"]*)"\s*\)\s*return'
+    seen: list[str] = []
+    for text in re.findall(pattern, body):
+        if text not in seen:
+            seen.append(text)
+    return seen
+
+
 def make_tag_to_string(tag_strings: list[str], irregular_cases: dict[str, str],
-                       values: dict[str, int], irregular_flag: int):
-    """Build a Python transcription of the C++ `tagToString`."""
+                       values: dict[str, int], irregular_flag: int, max_value: int):
+    """Build a Python transcription of the C++ `tagToString`.
+
+    Returns None where upstream has no tag string for the value. tagToString
+    guards its table lookup with `assert(t < POSTag::max)` and its irregular
+    branch falls through to a "@" sentinel, so those values are not tags that
+    upstream ever converts; `toPOSTag` rejects "@" as well.
+    """
     # Map the numeric value of each irregular case label to its return string.
     irregular_by_value = {values[name]: text for name, text in irregular_cases.items()}
-    default_irregular = "@"
 
     def tag_to_string(value: int) -> str | None:
         if value & irregular_flag:
-            cleared = value & ~irregular_flag
-            return irregular_by_value.get(cleared, default_irregular)
-        if value >= len(tag_strings):
+            return irregular_by_value.get(value & ~irregular_flag)
+        if value >= max_value:
             return None
         return tag_strings[value]
 
@@ -187,10 +213,23 @@ def go_constant_name(cpp_name: str) -> str:
 
 
 def build_tag_entries(tags: list[dict], values: dict[str, int], tag_to_string,
-                      regular_cases: dict[str, str]) -> list[tuple[str, str]]:
-    """Build the ordered (go_name, go_value) list for the generated constants."""
-    entries: list[tuple[str, str]] = []
+                      regular_cases: dict[str, str],
+                      accepted: list[str]) -> tuple[list[tuple[str, str]], list[tuple[str, str]]]:
+    """Build the (go_name, go_value) lists for the generated constants.
+
+    Returns the tags Kiwi can emit and, separately, the input-only aliases that
+    upstream's toPOSTag accepts but never produces.
+    """
+    emitted: list[tuple[str, str]] = []
     seen_names: set[str] = set()
+    seen_values: set[str] = set()
+
+    def add(go_name: str, go_value: str, target: list[tuple[str, str]]) -> None:
+        if go_name in seen_names:
+            return
+        seen_names.add(go_name)
+        seen_values.add(go_value)
+        target.append((go_name, go_value))
 
     for tag in tags:
         cpp_name = tag["name"]
@@ -201,27 +240,29 @@ def build_tag_entries(tags: list[dict], values: dict[str, int], tag_to_string,
         if go_value is None:
             continue
 
-        go_name = go_constant_name(cpp_name)
-        if go_name in seen_names:
-            continue
-
-        seen_names.add(go_name)
-        entries.append((go_name, go_value))
+        add(go_constant_name(cpp_name), go_value, emitted)
 
     # `-R` variants are only reachable through tagRToString, so they are not
     # derivable from the enum alone.
     for cpp_name, text in regular_cases.items():
-        go_name = f"POS_{cpp_name.upper()}_R"
-        if go_name in seen_names:
+        add(f"POS_{cpp_name.upper()}_R", text, emitted)
+
+    # Strings toPOSTag accepts as input but tagToString never returns, such as
+    # "V" and "A" for POSTag::p or "NF"/"NV"/"NA"/"UNK" for POSTag::unknown.
+    aliases: list[tuple[str, str]] = []
+    for text in accepted:
+        if text in seen_values:
             continue
-        seen_names.add(go_name)
-        entries.append((go_name, text))
+        go_name = PUNCTUATION_TAG_NAMES.get(text, f"POS_{text.upper()}")
+        add(go_name, text, aliases)
 
-    return entries
+    return emitted, aliases
 
 
-def generate_go_file(entries: list[tuple[str, str]], version: str) -> str:
+def generate_go_file(emitted: list[tuple[str, str]], aliases: list[tuple[str, str]],
+                     version: str) -> str:
     """Generate Go source file with POS type definitions."""
+    entries = emitted + aliases
     lines = []
     lines.append("// Code generated by scripts/extract_postags.py; DO NOT EDIT.")
     lines.append(f"// Source: Kiwi {version}")
@@ -236,11 +277,19 @@ def generate_go_file(entries: list[tuple[str, str]], version: str) -> str:
     lines.append("")
     lines.append("const (")
 
-    max_name_len = max(len(name) for name, _ in entries)
+    # gofmt aligns each run of declarations separated by a blank line on its
+    # own, so each group is padded to its own longest name.
+    def declare_group(group: list[tuple[str, str]]) -> list[str]:
+        width = max(len(name) for name, _ in group)
+        return [
+            f'\t{go_name}{" " * (width - len(go_name) + 1)}POSType = "{go_value}"'
+            for go_name, go_value in group
+        ]
 
-    for go_name, go_value in entries:
-        padding = " " * (max_name_len - len(go_name) + 1)
-        lines.append(f'\t{go_name}{padding}POSType = "{go_value}"')
+    lines.extend(declare_group(emitted))
+    lines.append("")
+    lines.append("\t// Accepted by toPOSTag but never produced by Kiwi itself.")
+    lines.extend(declare_group(aliases))
 
     lines.append(")")
     lines.append("")
@@ -255,8 +304,10 @@ def generate_go_file(entries: list[tuple[str, str]], version: str) -> str:
     lines.append(")")
     lines.append("")
 
-    # Generate isValid function. A Go switch requires distinct case values, and
-    # several constants are aliases sharing one string, so deduplicate by value.
+    # Generate isValid function. This mirrors which strings upstream's toPOSTag
+    # accepts, plus the strings tagToString/tagRToString can return. A Go switch
+    # requires distinct case values, and several constants are aliases sharing
+    # one string, so deduplicate by value.
     lines.append("func (p POSType) isValid() bool {")
     lines.append("\tswitch p {")
     lines.append("\tcase")
@@ -305,6 +356,10 @@ def main():
     print(f"Fetching {utils_url}")
     utils_source = fetch_file(utils_url)
 
+    strutils_url = KIWI_STRUTILS_H_URL.format(version=version)
+    print(f"Fetching {strutils_url}")
+    strutils_source = fetch_file(strutils_url)
+
     # Extract enum values
     print("Parsing POSTag enum...")
     tags = extract_enum_values(types_source, "POSTag")
@@ -328,11 +383,21 @@ def main():
     regular_cases = extract_case_returns(tag_r_to_string_body)
     print(f"Found {len(regular_cases)} regular-conjugation tags")
 
-    tag_to_string = make_tag_to_string(tag_strings, irregular_cases, values, values["irregular"])
-    entries = build_tag_entries(tags, values, tag_to_string, regular_cases)
+    # toPOSTag defines which strings upstream accepts as input, which is the
+    # role ParsePOSType plays on the Go side.
+    to_pos_tag_body = extract_function_body(strutils_source, r"inline\s+POSTag\s+toPOSTag\s*\(\s*std::u16string_view")
+    accepted = extract_accepted_tags(to_pos_tag_body)
+    if not accepted:
+        raise SystemExit("toPOSTag not found; cannot determine accepted tag strings")
+    print(f"Found {len(accepted)} accepted tag strings")
+
+    tag_to_string = make_tag_to_string(
+        tag_strings, irregular_cases, values, values["irregular"], values["max"]
+    )
+    emitted, aliases = build_tag_entries(tags, values, tag_to_string, regular_cases, accepted)
 
     # Generate Go file
-    go_content = generate_go_file(entries, version)
+    go_content = generate_go_file(emitted, aliases, version)
 
     # Write output
     output_path = Path("postype_generated.go")
@@ -343,13 +408,15 @@ def main():
     print("\nExtracted tags:")
     for tag in tags:
         cpp_name = tag["name"]
-        if cpp_name in SKIPPED_ENUMERATORS:
-            print(f"  {cpp_name:20s} -> (skipped)")
+        text = None if cpp_name in SKIPPED_ENUMERATORS else tag_to_string(values[cpp_name])
+        if text is None:
+            print(f"  {cpp_name:20s} -> (no tag string; skipped)")
             continue
-        text = tag_to_string(values[cpp_name])
         print(f'  {cpp_name:20s} -> {go_constant_name(cpp_name):20s} = "{text}"')
     for cpp_name, text in regular_cases.items():
         print(f'  {cpp_name + " (R)":20s} -> {"POS_" + cpp_name.upper() + "_R":20s} = "{text}"')
+    for go_name, text in aliases:
+        print(f'  {"(toPOSTag only)":20s} -> {go_name:20s} = "{text}"')
 
 
 if __name__ == "__main__":