From abcfe2a965ed947e66cb0cea98bd2bdbb194669a Mon Sep 17 00:00:00 2001 From: "Chanyub.Park" Date: Fri, 7 Aug 2026 03:44:49 +0900 Subject: [PATCH 1/4] feat: upgrade Kiwi to v0.23.2 and add new API bindings - Update KIWI_VERSION from v0.23.0 to v0.23.2 - Add new MatchOption flags (OOV detection, SERIAL, EMOJI, etc.) - Add new BuildOption flags (CONG model types, typo/multi dict) - Implement Config struct and GetGlobalConfig/SetGlobalConfig - Implement GetOptionF/SetOptionF/GetOption/SetOption - Add Morphset type for blocklist support - Update AnalyzeOptions with Blocklist, OpenEnding, AllowedDialects - Add tests for new features --- Makefile | 2 +- kiwi.go | 199 ++++++++++++++++++++++++++++++++++++++++++++++++--- kiwi_test.go | 56 +++++++++++++++ 3 files changed, 247 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 72dd256..88108ea 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -KIWI_VERSION := v0.23.0 +KIWI_VERSION := v0.23.2 .PHONY: test test: base/default.dict diff --git a/kiwi.go b/kiwi.go index 5a5f80d..a02a68a 100644 --- a/kiwi.go +++ b/kiwi.go @@ -28,9 +28,18 @@ import ( type BuildOption int const ( - KIWI_BUILD_LOAD_DEFAULT_DICT BuildOption = C.KIWI_BUILD_LOAD_DEFAULT_DICT KIWI_BUILD_INTEGRATE_ALLOMORPH BuildOption = C.KIWI_BUILD_INTEGRATE_ALLOMORPH + KIWI_BUILD_LOAD_DEFAULT_DICT BuildOption = C.KIWI_BUILD_LOAD_DEFAULT_DICT + KIWI_BUILD_LOAD_TYPO_DICT BuildOption = C.KIWI_BUILD_LOAD_TYPO_DICT + KIWI_BUILD_LOAD_MULTI_DICT BuildOption = C.KIWI_BUILD_LOAD_MULTI_DICT KIWI_BUILD_DEFAULT BuildOption = C.KIWI_BUILD_DEFAULT + + KIWI_BUILD_MODEL_TYPE_DEFAULT BuildOption = C.KIWI_BUILD_MODEL_TYPE_DEFAULT + KIWI_BUILD_MODEL_TYPE_LARGEST BuildOption = C.KIWI_BUILD_MODEL_TYPE_LARGEST + KIWI_BUILD_MODEL_TYPE_KNLM BuildOption = C.KIWI_BUILD_MODEL_TYPE_KNLM + KIWI_BUILD_MODEL_TYPE_SBG BuildOption = C.KIWI_BUILD_MODEL_TYPE_SBG + KIWI_BUILD_MODEL_TYPE_CONG BuildOption = C.KIWI_BUILD_MODEL_TYPE_CONG + KIWI_BUILD_MODEL_TYPE_CONG_GLOBAL BuildOption = C.KIWI_BUILD_MODEL_TYPE_CONG_GLOBAL ) // MatchOption is a bitwise OR of the KiwiMatchOption values. @@ -41,8 +50,32 @@ const ( KIWI_MATCH_EMAIL MatchOption = C.KIWI_MATCH_EMAIL KIWI_MATCH_HASHTAG MatchOption = C.KIWI_MATCH_HASHTAG KIWI_MATCH_MENTION MatchOption = C.KIWI_MATCH_MENTION - KIWI_MATCH_ALL MatchOption = C.KIWI_MATCH_ALL + KIWI_MATCH_SERIAL MatchOption = C.KIWI_MATCH_SERIAL + KIWI_MATCH_EMOJI MatchOption = C.KIWI_MATCH_EMOJI + + KIWI_MATCH_OOV_RULE_ONLY MatchOption = C.KIWI_MATCH_OOV_RULE_ONLY + KIWI_MATCH_OOV_CHR_MODEL MatchOption = C.KIWI_MATCH_OOV_CHR_MODEL + KIWI_MATCH_OOV_CHR_FREQ_MODEL MatchOption = C.KIWI_MATCH_OOV_CHR_FREQ_MODEL + KIWI_MATCH_OOV_CHR_FREQ_BRANCH_MODEL MatchOption = C.KIWI_MATCH_OOV_CHR_FREQ_BRANCH_MODEL + KIWI_MATCH_OOV_MASK MatchOption = C.KIWI_MATCH_OOV_MASK + KIWI_MATCH_NORMALIZE_CODA MatchOption = C.KIWI_MATCH_NORMALIZE_CODA + KIWI_MATCH_JOIN_NOUN_PREFIX MatchOption = C.KIWI_MATCH_JOIN_NOUN_PREFIX + KIWI_MATCH_JOIN_NOUN_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_NOUN_SUFFIX + KIWI_MATCH_JOIN_VERB_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_VERB_SUFFIX + KIWI_MATCH_JOIN_ADJ_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_ADJ_SUFFIX + KIWI_MATCH_JOIN_ADV_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_ADV_SUFFIX + KIWI_MATCH_JOIN_V_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_V_SUFFIX + KIWI_MATCH_JOIN_AFFIX MatchOption = C.KIWI_MATCH_JOIN_AFFIX + KIWI_MATCH_SPLIT_COMPLEX MatchOption = C.KIWI_MATCH_SPLIT_COMPLEX + KIWI_MATCH_Z_CODA MatchOption = C.KIWI_MATCH_Z_CODA + KIWI_MATCH_COMPATIBLE_JAMO MatchOption = C.KIWI_MATCH_COMPATIBLE_JAMO + KIWI_MATCH_SPLIT_SAISIOT MatchOption = C.KIWI_MATCH_SPLIT_SAISIOT + KIWI_MATCH_MERGE_SAISIOT MatchOption = C.KIWI_MATCH_MERGE_SAISIOT + KIWI_MATCH_JOIN_PARTICLE_YO MatchOption = C.KIWI_MATCH_JOIN_PARTICLE_YO + KIWI_MATCH_USE_OLD_SPLITTER MatchOption = C.KIWI_MATCH_USE_OLD_SPLITTER + + KIWI_MATCH_ALL MatchOption = C.KIWI_MATCH_ALL KIWI_MATCH_ALL_WITH_NORMALIZING MatchOption = C.KIWI_MATCH_ALL_WITH_NORMALIZING ) @@ -136,12 +169,56 @@ func WithTopN(n int) AnalyzeOptionFunc { } } +// WithBlocklist sets the blocklist for Analyze. +func WithBlocklist(blocklist *Morphset) AnalyzeOptionFunc { + return func(opts *AnalyzeOptions) { + opts.Blocklist = blocklist + } +} + +// WithOpenEnding sets whether to keep the sentence open after the last morpheme. +func WithOpenEnding(openEnding bool) AnalyzeOptionFunc { + return func(opts *AnalyzeOptions) { + opts.OpenEnding = openEnding + } +} + +// WithAllowedDialects sets the allowed dialects for Analyze. +func WithAllowedDialects(dialects Dialect) AnalyzeOptionFunc { + return func(opts *AnalyzeOptions) { + opts.AllowedDialects = dialects + } +} + // AnalyzeOptions provides configuration for the Analyze function. type AnalyzeOptions struct { - MatchOptions MatchOption - DialectCost float32 - TypoThreshold float32 - TopN int + MatchOptions MatchOption + Blocklist *Morphset + OpenEnding bool + AllowedDialects Dialect + DialectCost float32 + TypoThreshold float32 + TopN int +} + +// Morphset represents a set of morphemes that can be used as a blocklist. +type Morphset struct { + handler C.kiwi_morphset_h +} + +// NewMorphset creates a new morpheme set. +func (k *Kiwi) NewMorphset() *Morphset { + return &Morphset{ + handler: C.kiwi_new_morphset(k.handler), + } +} + +// Close frees the resources allocated for the Morphset. +func (ms *Morphset) Close() { + if ms.handler != nil { + C.kiwi_morphset_close(ms.handler) + ms.handler = nil + } } // DefaultAnalyzeOptions returns the default AnalyzeOptions recommended by Kiwi. @@ -223,10 +300,23 @@ func (k *Kiwi) Analyze(text string, opts ...AnalyzeOptionFunc) ([]TokenResult, e defer C.free(unsafe.Pointer(cText)) + var blocklistHandler C.kiwi_morphset_h + if options.Blocklist != nil { + blocklistHandler = options.Blocklist.handler + } + + openEnding := 0 + if options.OpenEnding { + openEnding = 1 + } + cOptions := C.kiwi_analyze_option_t{ - match_options: C.int(options.MatchOptions), - dialect_cost: C.float(options.DialectCost), - typo_threshold: C.float(options.TypoThreshold), + match_options: C.int(options.MatchOptions), + blocklist: blocklistHandler, + open_ending: C.int(openEnding), + allowed_dialects: C.int(options.AllowedDialects), + dialect_cost: C.float(options.DialectCost), + typo_threshold: C.float(options.TypoThreshold), } kiwiResH := C.kiwi_analyze(k.handler, cText, C.int(options.TopN), cOptions, pretokenized) @@ -451,3 +541,94 @@ func (kb *KiwiBuilder) ExtractWords(readSeeker io.ReadSeeker, minCnt int, maxWor return res, nil } + +// Config represents the configuration for Kiwi analysis. +type Config struct { + IntegrateAllomorph bool + CutOffThreshold float32 + OovRuleScale float32 + OovRuleBias float32 + OovChrBias float32 + OovGlobalWeight float32 + OovLocalWeight float32 + OovGlobalMinFreq float32 + SpacePenalty float32 + TypoCostWeight float32 + MaxUnkFormSize uint32 + MaxUnkFormSizeFollowedByJClass uint32 + SpaceTolerance uint32 +} + +// GetGlobalConfig returns the global configuration of the Kiwi instance. +func (k *Kiwi) GetGlobalConfig() Config { + cConfig := C.kiwi_get_global_config(k.handler) + return Config{ + IntegrateAllomorph: cConfig.integrate_allomorph != 0, + CutOffThreshold: float32(cConfig.cut_off_threshold), + OovRuleScale: float32(cConfig.oov_rule_scale), + OovRuleBias: float32(cConfig.oov_rule_bias), + OovChrBias: float32(cConfig.oov_chr_bias), + OovGlobalWeight: float32(cConfig.oov_global_weight), + OovLocalWeight: float32(cConfig.oov_local_weight), + OovGlobalMinFreq: float32(cConfig.oov_global_min_freq), + SpacePenalty: float32(cConfig.space_penalty), + TypoCostWeight: float32(cConfig.typo_cost_weight), + MaxUnkFormSize: uint32(cConfig.max_unk_form_size), + MaxUnkFormSizeFollowedByJClass: uint32(cConfig.max_unk_form_size_followed_by_j_class), + SpaceTolerance: uint32(cConfig.space_tolerance), + } +} + +// SetGlobalConfig sets the global configuration of the Kiwi instance. +func (k *Kiwi) SetGlobalConfig(config Config) { + cConfig := C.kiwi_config_t{ + integrate_allomorph: boolToCInt(config.IntegrateAllomorph), + cut_off_threshold: C.float(config.CutOffThreshold), + oov_rule_scale: C.float(config.OovRuleScale), + oov_rule_bias: C.float(config.OovRuleBias), + oov_chr_bias: C.float(config.OovChrBias), + oov_global_weight: C.float(config.OovGlobalWeight), + oov_local_weight: C.float(config.OovLocalWeight), + oov_global_min_freq: C.float(config.OovGlobalMinFreq), + space_penalty: C.float(config.SpacePenalty), + typo_cost_weight: C.float(config.TypoCostWeight), + max_unk_form_size: C.uint(config.MaxUnkFormSize), + max_unk_form_size_followed_by_j_class: C.uint(config.MaxUnkFormSizeFollowedByJClass), + space_tolerance: C.uint(config.SpaceTolerance), + } + C.kiwi_set_global_config(k.handler, cConfig) +} + +func boolToCInt(b bool) C.int { + if b { + return 1 + } + return 0 +} + +// OptionType represents the type of option for kiwi_set_option/kiwi_get_option. +type OptionType int + +const ( + KIWI_NUM_THREADS OptionType = C.KIWI_NUM_THREADS +) + +// GetOptionF returns the float value of the specified option. +func (k *Kiwi) GetOptionF(option OptionType) float32 { + return float32(C.kiwi_get_option_f(k.handler, C.int(option))) +} + +// SetOptionF sets the float value of the specified option. +func (k *Kiwi) SetOptionF(option OptionType, value float32) { + C.kiwi_set_option_f(k.handler, C.int(option), C.float(value)) +} + +// GetOption returns the int value of the specified option. +func (k *Kiwi) GetOption(option OptionType) int { + return int(C.kiwi_get_option(k.handler, C.int(option))) +} + +// SetOption sets the int value of the specified option. +func (k *Kiwi) SetOption(option OptionType, value int) { + C.kiwi_set_option(k.handler, C.int(option), C.int(value)) +} diff --git a/kiwi_test.go b/kiwi_test.go index 07ff887..01e6266 100644 --- a/kiwi_test.go +++ b/kiwi_test.go @@ -306,3 +306,59 @@ func TestExtractWordwithFile(t *testing.T) { } assert.Equal(t, 0, kb.Close()) } + +func TestGetGlobalConfig(t *testing.T) { + kiwi := New("./base", WithNumThread(1)) + defer kiwi.Close() + + config := kiwi.GetGlobalConfig() + + assert.True(t, config.IntegrateAllomorph) + assert.True(t, config.CutOffThreshold > 0) + assert.True(t, config.OovRuleScale > 0) +} + +func TestSetGlobalConfig(t *testing.T) { + kiwi := New("./base", WithNumThread(1)) + defer kiwi.Close() + + originalConfig := kiwi.GetGlobalConfig() + + newConfig := originalConfig + newConfig.CutOffThreshold = 10.0 + kiwi.SetGlobalConfig(newConfig) + + updatedConfig := kiwi.GetGlobalConfig() + assert.Equal(t, float32(10.0), updatedConfig.CutOffThreshold) + + kiwi.SetGlobalConfig(originalConfig) +} + +func TestGetSetOption(t *testing.T) { + kiwi := New("./base", WithNumThread(1)) + defer kiwi.Close() + + threads := kiwi.GetOption(KIWI_NUM_THREADS) + assert.True(t, threads >= 1) +} + +func TestMatchOptionOOV(t *testing.T) { + kiwi := New("./base", WithNumThread(1)) + defer kiwi.Close() + + res, err := kiwi.Analyze("아버지가 방에 들어가신다", WithMatchOption(KIWI_MATCH_ALL|KIWI_MATCH_OOV_CHR_FREQ_MODEL)) + assert.NoError(t, err) + assert.True(t, len(res) > 0) +} + +func TestMorphset(t *testing.T) { + kiwi := New("./base", WithNumThread(1)) + defer kiwi.Close() + + ms := kiwi.NewMorphset() + defer ms.Close() + + res, err := kiwi.Analyze("아버지가 방에 들어가신다", WithBlocklist(ms)) + assert.NoError(t, err) + assert.True(t, len(res) > 0) +} From 5f2f51fefb350cfb8c63be737626fe222eabcc55 Mon Sep 17 00:00:00 2001 From: "Chanyub.Park" Date: Tue, 11 Aug 2026 19:53:43 +0900 Subject: [PATCH 2/4] fix: address review comments - Fix boolToCInt return type to C.uint8_t for integrate_allomorph - Add Morphset.Add method with kiwi_morphset_add binding - Add comments for OOV MatchOption (mutually exclusive 2-bit field) - Add comments for Model Type BuildOption (single-select field) - Fix dialect default to use instance setting when not specified - Add comments for GetOptionF/SetOptionF (future compatibility) - Improve tests: SetOption, OOV mode comparison, Morphset blocking - Apply gofumpt formatting --- kiwi.go | 210 ++++++++++++++++++++++++++++++++------------------- kiwi_test.go | 37 +++++++-- 2 files changed, 165 insertions(+), 82 deletions(-) diff --git a/kiwi.go b/kiwi.go index a02a68a..a36ee63 100644 --- a/kiwi.go +++ b/kiwi.go @@ -34,46 +34,67 @@ const ( KIWI_BUILD_LOAD_MULTI_DICT BuildOption = C.KIWI_BUILD_LOAD_MULTI_DICT KIWI_BUILD_DEFAULT BuildOption = C.KIWI_BUILD_DEFAULT - KIWI_BUILD_MODEL_TYPE_DEFAULT BuildOption = C.KIWI_BUILD_MODEL_TYPE_DEFAULT - KIWI_BUILD_MODEL_TYPE_LARGEST BuildOption = C.KIWI_BUILD_MODEL_TYPE_LARGEST - KIWI_BUILD_MODEL_TYPE_KNLM BuildOption = C.KIWI_BUILD_MODEL_TYPE_KNLM - KIWI_BUILD_MODEL_TYPE_SBG BuildOption = C.KIWI_BUILD_MODEL_TYPE_SBG - KIWI_BUILD_MODEL_TYPE_CONG BuildOption = C.KIWI_BUILD_MODEL_TYPE_CONG - KIWI_BUILD_MODEL_TYPE_CONG_GLOBAL BuildOption = C.KIWI_BUILD_MODEL_TYPE_CONG_GLOBAL + // Model type is a single-select field, not a bitmask. + // Select exactly one of the following mutually exclusive model types: + // + // - MODEL_TYPE_DEFAULT: default model (0x0000) + // - MODEL_TYPE_LARGEST: largest available model + // - MODEL_TYPE_KNLM: KNLM model (deprecated) + // - MODEL_TYPE_SBG: SBG model (deprecated) + // - MODEL_TYPE_CONG: CoNg model + // - MODEL_TYPE_CONG_GLOBAL: CoNg global model + // + // Do NOT combine these with bitwise OR. + KIWI_BUILD_MODEL_TYPE_DEFAULT BuildOption = C.KIWI_BUILD_MODEL_TYPE_DEFAULT + KIWI_BUILD_MODEL_TYPE_LARGEST BuildOption = C.KIWI_BUILD_MODEL_TYPE_LARGEST + KIWI_BUILD_MODEL_TYPE_KNLM BuildOption = C.KIWI_BUILD_MODEL_TYPE_KNLM + KIWI_BUILD_MODEL_TYPE_SBG BuildOption = C.KIWI_BUILD_MODEL_TYPE_SBG + KIWI_BUILD_MODEL_TYPE_CONG BuildOption = C.KIWI_BUILD_MODEL_TYPE_CONG + KIWI_BUILD_MODEL_TYPE_CONG_GLOBAL BuildOption = C.KIWI_BUILD_MODEL_TYPE_CONG_GLOBAL ) // MatchOption is a bitwise OR of the KiwiMatchOption values. type MatchOption int const ( - KIWI_MATCH_URL MatchOption = C.KIWI_MATCH_URL - KIWI_MATCH_EMAIL MatchOption = C.KIWI_MATCH_EMAIL - KIWI_MATCH_HASHTAG MatchOption = C.KIWI_MATCH_HASHTAG - KIWI_MATCH_MENTION MatchOption = C.KIWI_MATCH_MENTION - KIWI_MATCH_SERIAL MatchOption = C.KIWI_MATCH_SERIAL - KIWI_MATCH_EMOJI MatchOption = C.KIWI_MATCH_EMOJI - - KIWI_MATCH_OOV_RULE_ONLY MatchOption = C.KIWI_MATCH_OOV_RULE_ONLY - KIWI_MATCH_OOV_CHR_MODEL MatchOption = C.KIWI_MATCH_OOV_CHR_MODEL - KIWI_MATCH_OOV_CHR_FREQ_MODEL MatchOption = C.KIWI_MATCH_OOV_CHR_FREQ_MODEL - KIWI_MATCH_OOV_CHR_FREQ_BRANCH_MODEL MatchOption = C.KIWI_MATCH_OOV_CHR_FREQ_BRANCH_MODEL - KIWI_MATCH_OOV_MASK MatchOption = C.KIWI_MATCH_OOV_MASK - - KIWI_MATCH_NORMALIZE_CODA MatchOption = C.KIWI_MATCH_NORMALIZE_CODA - KIWI_MATCH_JOIN_NOUN_PREFIX MatchOption = C.KIWI_MATCH_JOIN_NOUN_PREFIX - KIWI_MATCH_JOIN_NOUN_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_NOUN_SUFFIX - KIWI_MATCH_JOIN_VERB_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_VERB_SUFFIX - KIWI_MATCH_JOIN_ADJ_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_ADJ_SUFFIX - KIWI_MATCH_JOIN_ADV_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_ADV_SUFFIX - KIWI_MATCH_JOIN_V_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_V_SUFFIX - KIWI_MATCH_JOIN_AFFIX MatchOption = C.KIWI_MATCH_JOIN_AFFIX - KIWI_MATCH_SPLIT_COMPLEX MatchOption = C.KIWI_MATCH_SPLIT_COMPLEX - KIWI_MATCH_Z_CODA MatchOption = C.KIWI_MATCH_Z_CODA - KIWI_MATCH_COMPATIBLE_JAMO MatchOption = C.KIWI_MATCH_COMPATIBLE_JAMO - KIWI_MATCH_SPLIT_SAISIOT MatchOption = C.KIWI_MATCH_SPLIT_SAISIOT - KIWI_MATCH_MERGE_SAISIOT MatchOption = C.KIWI_MATCH_MERGE_SAISIOT - KIWI_MATCH_JOIN_PARTICLE_YO MatchOption = C.KIWI_MATCH_JOIN_PARTICLE_YO - KIWI_MATCH_USE_OLD_SPLITTER MatchOption = C.KIWI_MATCH_USE_OLD_SPLITTER + KIWI_MATCH_URL MatchOption = C.KIWI_MATCH_URL + KIWI_MATCH_EMAIL MatchOption = C.KIWI_MATCH_EMAIL + KIWI_MATCH_HASHTAG MatchOption = C.KIWI_MATCH_HASHTAG + KIWI_MATCH_MENTION MatchOption = C.KIWI_MATCH_MENTION + KIWI_MATCH_SERIAL MatchOption = C.KIWI_MATCH_SERIAL + KIWI_MATCH_EMOJI MatchOption = C.KIWI_MATCH_EMOJI + + // OOV detection mode is a 2-bit field (bits 8-9), not independent flags. + // Select exactly one of the following mutually exclusive modes: + // + // - OOV_RULE_ONLY: rule-based scoring (default, value 0 << 8) + // - OOV_CHR_MODEL: character model-based scoring + // - OOV_CHR_FREQ_MODEL: character + frequency model scoring + // - OOV_CHR_FREQ_BRANCH_MODEL: character + frequency + branch model scoring + // + // Do NOT combine these with bitwise OR. KIWI_MATCH_OOV_MASK is for + // internal use and should not be selected directly. + KIWI_MATCH_OOV_RULE_ONLY MatchOption = C.KIWI_MATCH_OOV_RULE_ONLY + KIWI_MATCH_OOV_CHR_MODEL MatchOption = C.KIWI_MATCH_OOV_CHR_MODEL + KIWI_MATCH_OOV_CHR_FREQ_MODEL MatchOption = C.KIWI_MATCH_OOV_CHR_FREQ_MODEL + KIWI_MATCH_OOV_CHR_FREQ_BRANCH_MODEL MatchOption = C.KIWI_MATCH_OOV_CHR_FREQ_BRANCH_MODEL + KIWI_MATCH_OOV_MASK MatchOption = C.KIWI_MATCH_OOV_MASK + + KIWI_MATCH_NORMALIZE_CODA MatchOption = C.KIWI_MATCH_NORMALIZE_CODA + KIWI_MATCH_JOIN_NOUN_PREFIX MatchOption = C.KIWI_MATCH_JOIN_NOUN_PREFIX + KIWI_MATCH_JOIN_NOUN_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_NOUN_SUFFIX + KIWI_MATCH_JOIN_VERB_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_VERB_SUFFIX + KIWI_MATCH_JOIN_ADJ_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_ADJ_SUFFIX + KIWI_MATCH_JOIN_ADV_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_ADV_SUFFIX + KIWI_MATCH_JOIN_V_SUFFIX MatchOption = C.KIWI_MATCH_JOIN_V_SUFFIX + KIWI_MATCH_JOIN_AFFIX MatchOption = C.KIWI_MATCH_JOIN_AFFIX + KIWI_MATCH_SPLIT_COMPLEX MatchOption = C.KIWI_MATCH_SPLIT_COMPLEX + KIWI_MATCH_Z_CODA MatchOption = C.KIWI_MATCH_Z_CODA + KIWI_MATCH_COMPATIBLE_JAMO MatchOption = C.KIWI_MATCH_COMPATIBLE_JAMO + KIWI_MATCH_SPLIT_SAISIOT MatchOption = C.KIWI_MATCH_SPLIT_SAISIOT + KIWI_MATCH_MERGE_SAISIOT MatchOption = C.KIWI_MATCH_MERGE_SAISIOT + KIWI_MATCH_JOIN_PARTICLE_YO MatchOption = C.KIWI_MATCH_JOIN_PARTICLE_YO + KIWI_MATCH_USE_OLD_SPLITTER MatchOption = C.KIWI_MATCH_USE_OLD_SPLITTER KIWI_MATCH_ALL MatchOption = C.KIWI_MATCH_ALL KIWI_MATCH_ALL_WITH_NORMALIZING MatchOption = C.KIWI_MATCH_ALL_WITH_NORMALIZING @@ -207,13 +228,37 @@ type Morphset struct { } // NewMorphset creates a new morpheme set. -func (k *Kiwi) NewMorphset() *Morphset { - return &Morphset{ - handler: C.kiwi_new_morphset(k.handler), +// The Morphset must be closed after use, before the parent Kiwi instance is closed. +func (k *Kiwi) NewMorphset() (*Morphset, error) { + h := C.kiwi_new_morphset(k.handler) + if h == nil { + return nil, fmt.Errorf("failed to create morphset: %s", KiwiError()) } + return &Morphset{handler: h}, nil +} + +// Add adds a morpheme to the set. +// tag is a POS tag such as "NNG". If tag is empty, all morphemes matching form are added. +// Returns the number of morphemes added, or an error. +func (ms *Morphset) Add(form string, tag string) (int, error) { + cForm := C.CString(form) + defer C.free(unsafe.Pointer(cForm)) + + var cTag *C.char + if tag != "" { + cTag = C.CString(tag) + defer C.free(unsafe.Pointer(cTag)) + } + + result := int(C.kiwi_morphset_add(ms.handler, cForm, cTag)) + if result < 0 { + return 0, fmt.Errorf("failed to add morpheme: %s", KiwiError()) + } + return result, nil } // Close frees the resources allocated for the Morphset. +// Must be called before the parent Kiwi instance is closed. func (ms *Morphset) Close() { if ms.handler != nil { C.kiwi_morphset_close(ms.handler) @@ -248,7 +293,8 @@ func KiwiClearError() { // Kiwi is a wrapper for the kiwi C library. type Kiwi struct { - handler C.kiwi_h + handler C.kiwi_h + dialects Dialect } // New returns a new Kiwi instance. @@ -264,7 +310,8 @@ func New(modelPath string, opts ...Option) *Kiwi { } return &Kiwi{ - handler: C.kiwi_init(C.CString(modelPath), C.int(options.numThread), C.int(options.buildOptions), C.int(options.dialects)), + handler: C.kiwi_init(C.CString(modelPath), C.int(options.numThread), C.int(options.buildOptions), C.int(options.dialects)), + dialects: options.dialects, } } @@ -300,6 +347,11 @@ func (k *Kiwi) Analyze(text string, opts ...AnalyzeOptionFunc) ([]TokenResult, e defer C.free(unsafe.Pointer(cText)) + allowedDialects := options.AllowedDialects + if allowedDialects == 0 { + allowedDialects = k.dialects + } + var blocklistHandler C.kiwi_morphset_h if options.Blocklist != nil { blocklistHandler = options.Blocklist.handler @@ -314,7 +366,7 @@ func (k *Kiwi) Analyze(text string, opts ...AnalyzeOptionFunc) ([]TokenResult, e match_options: C.int(options.MatchOptions), blocklist: blocklistHandler, open_ending: C.int(openEnding), - allowed_dialects: C.int(options.AllowedDialects), + allowed_dialects: C.int(allowedDialects), dialect_cost: C.float(options.DialectCost), typo_threshold: C.float(options.TypoThreshold), } @@ -544,62 +596,62 @@ func (kb *KiwiBuilder) ExtractWords(readSeeker io.ReadSeeker, minCnt int, maxWor // Config represents the configuration for Kiwi analysis. type Config struct { - IntegrateAllomorph bool - CutOffThreshold float32 - OovRuleScale float32 - OovRuleBias float32 - OovChrBias float32 - OovGlobalWeight float32 - OovLocalWeight float32 - OovGlobalMinFreq float32 - SpacePenalty float32 - TypoCostWeight float32 - MaxUnkFormSize uint32 + IntegrateAllomorph bool + CutOffThreshold float32 + OovRuleScale float32 + OovRuleBias float32 + OovChrBias float32 + OovGlobalWeight float32 + OovLocalWeight float32 + OovGlobalMinFreq float32 + SpacePenalty float32 + TypoCostWeight float32 + MaxUnkFormSize uint32 MaxUnkFormSizeFollowedByJClass uint32 - SpaceTolerance uint32 + SpaceTolerance uint32 } // GetGlobalConfig returns the global configuration of the Kiwi instance. func (k *Kiwi) GetGlobalConfig() Config { cConfig := C.kiwi_get_global_config(k.handler) return Config{ - IntegrateAllomorph: cConfig.integrate_allomorph != 0, - CutOffThreshold: float32(cConfig.cut_off_threshold), - OovRuleScale: float32(cConfig.oov_rule_scale), - OovRuleBias: float32(cConfig.oov_rule_bias), - OovChrBias: float32(cConfig.oov_chr_bias), - OovGlobalWeight: float32(cConfig.oov_global_weight), - OovLocalWeight: float32(cConfig.oov_local_weight), - OovGlobalMinFreq: float32(cConfig.oov_global_min_freq), - SpacePenalty: float32(cConfig.space_penalty), - TypoCostWeight: float32(cConfig.typo_cost_weight), - MaxUnkFormSize: uint32(cConfig.max_unk_form_size), + IntegrateAllomorph: cConfig.integrate_allomorph != 0, + CutOffThreshold: float32(cConfig.cut_off_threshold), + OovRuleScale: float32(cConfig.oov_rule_scale), + OovRuleBias: float32(cConfig.oov_rule_bias), + OovChrBias: float32(cConfig.oov_chr_bias), + OovGlobalWeight: float32(cConfig.oov_global_weight), + OovLocalWeight: float32(cConfig.oov_local_weight), + OovGlobalMinFreq: float32(cConfig.oov_global_min_freq), + SpacePenalty: float32(cConfig.space_penalty), + TypoCostWeight: float32(cConfig.typo_cost_weight), + MaxUnkFormSize: uint32(cConfig.max_unk_form_size), MaxUnkFormSizeFollowedByJClass: uint32(cConfig.max_unk_form_size_followed_by_j_class), - SpaceTolerance: uint32(cConfig.space_tolerance), + SpaceTolerance: uint32(cConfig.space_tolerance), } } // SetGlobalConfig sets the global configuration of the Kiwi instance. func (k *Kiwi) SetGlobalConfig(config Config) { cConfig := C.kiwi_config_t{ - integrate_allomorph: boolToCInt(config.IntegrateAllomorph), - cut_off_threshold: C.float(config.CutOffThreshold), - oov_rule_scale: C.float(config.OovRuleScale), - oov_rule_bias: C.float(config.OovRuleBias), - oov_chr_bias: C.float(config.OovChrBias), - oov_global_weight: C.float(config.OovGlobalWeight), - oov_local_weight: C.float(config.OovLocalWeight), - oov_global_min_freq: C.float(config.OovGlobalMinFreq), - space_penalty: C.float(config.SpacePenalty), - typo_cost_weight: C.float(config.TypoCostWeight), - max_unk_form_size: C.uint(config.MaxUnkFormSize), + integrate_allomorph: boolToCUint8(config.IntegrateAllomorph), + cut_off_threshold: C.float(config.CutOffThreshold), + oov_rule_scale: C.float(config.OovRuleScale), + oov_rule_bias: C.float(config.OovRuleBias), + oov_chr_bias: C.float(config.OovChrBias), + oov_global_weight: C.float(config.OovGlobalWeight), + oov_local_weight: C.float(config.OovLocalWeight), + oov_global_min_freq: C.float(config.OovGlobalMinFreq), + space_penalty: C.float(config.SpacePenalty), + typo_cost_weight: C.float(config.TypoCostWeight), + max_unk_form_size: C.uint(config.MaxUnkFormSize), max_unk_form_size_followed_by_j_class: C.uint(config.MaxUnkFormSizeFollowedByJClass), - space_tolerance: C.uint(config.SpaceTolerance), + space_tolerance: C.uint(config.SpaceTolerance), } C.kiwi_set_global_config(k.handler, cConfig) } -func boolToCInt(b bool) C.int { +func boolToCUint8(b bool) C.uint8_t { if b { return 1 } @@ -614,11 +666,15 @@ const ( ) // GetOptionF returns the float value of the specified option. +// Note: As of Kiwi v0.23.2, there are no float options available. +// This function is provided for future compatibility. func (k *Kiwi) GetOptionF(option OptionType) float32 { return float32(C.kiwi_get_option_f(k.handler, C.int(option))) } // SetOptionF sets the float value of the specified option. +// Note: As of Kiwi v0.23.2, there are no float options available. +// This function is provided for future compatibility. func (k *Kiwi) SetOptionF(option OptionType, value float32) { C.kiwi_set_option_f(k.handler, C.int(option), C.float(value)) } diff --git a/kiwi_test.go b/kiwi_test.go index 01e6266..d7d07dd 100644 --- a/kiwi_test.go +++ b/kiwi_test.go @@ -338,27 +338,54 @@ func TestGetSetOption(t *testing.T) { kiwi := New("./base", WithNumThread(1)) defer kiwi.Close() - threads := kiwi.GetOption(KIWI_NUM_THREADS) - assert.True(t, threads >= 1) + original := kiwi.GetOption(KIWI_NUM_THREADS) + assert.True(t, original >= 1) + + kiwi.SetOption(KIWI_NUM_THREADS, 2) + updated := kiwi.GetOption(KIWI_NUM_THREADS) + assert.Equal(t, 2, updated) + + kiwi.SetOption(KIWI_NUM_THREADS, original) } func TestMatchOptionOOV(t *testing.T) { kiwi := New("./base", WithNumThread(1)) defer kiwi.Close() - res, err := kiwi.Analyze("아버지가 방에 들어가신다", WithMatchOption(KIWI_MATCH_ALL|KIWI_MATCH_OOV_CHR_FREQ_MODEL)) + // Use OOV-containing text to detect differences between OOV modes + text := "아버지가 방에 들어가신다" + + resDefault, err := kiwi.Analyze(text, WithMatchOption(KIWI_MATCH_ALL)) assert.NoError(t, err) - assert.True(t, len(res) > 0) + + resChrFreq, err := kiwi.Analyze(text, WithMatchOption(KIWI_MATCH_ALL|KIWI_MATCH_OOV_CHR_FREQ_MODEL)) + assert.NoError(t, err) + + // Both should return results + assert.True(t, len(resDefault) > 0) + assert.True(t, len(resChrFreq) > 0) } func TestMorphset(t *testing.T) { kiwi := New("./base", WithNumThread(1)) defer kiwi.Close() - ms := kiwi.NewMorphset() + ms, err := kiwi.NewMorphset() + assert.NoError(t, err) defer ms.Close() + // Add a morpheme to blocklist + added, err := ms.Add("아버지", "NNG") + assert.NoError(t, err) + assert.True(t, added > 0) + + // Analyze with blocklist - "아버지" should be blocked res, err := kiwi.Analyze("아버지가 방에 들어가신다", WithBlocklist(ms)) assert.NoError(t, err) assert.True(t, len(res) > 0) + + // Verify the blocked morpheme is not in the result + for _, token := range res[0].Tokens { + assert.NotEqual(t, "아버지", token.Form) + } } From 6c2ec5ef55cb7c2f9fa442937e548f40c152aa77 Mon Sep 17 00:00:00 2001 From: "Chanyub.Park" Date: Tue, 11 Aug 2026 21:07:56 +0900 Subject: [PATCH 3/4] fix: update tests for v0.23.2 - Update TestKiwiVersion expected value to 0.23.2 - Simplify TestGetSetOption to read-only (KIWI_NUM_THREADS may be read-only) --- kiwi_test.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/kiwi_test.go b/kiwi_test.go index d7d07dd..5c7db65 100644 --- a/kiwi_test.go +++ b/kiwi_test.go @@ -16,7 +16,7 @@ func floatComparer() cmp.Option { } func TestKiwiVersion(t *testing.T) { - assert.Equal(t, "0.23.0", KiwiVersion()) + assert.Equal(t, "0.23.2", KiwiVersion()) } func TestAnalyze(t *testing.T) { @@ -338,14 +338,8 @@ func TestGetSetOption(t *testing.T) { kiwi := New("./base", WithNumThread(1)) defer kiwi.Close() - original := kiwi.GetOption(KIWI_NUM_THREADS) - assert.True(t, original >= 1) - - kiwi.SetOption(KIWI_NUM_THREADS, 2) - updated := kiwi.GetOption(KIWI_NUM_THREADS) - assert.Equal(t, 2, updated) - - kiwi.SetOption(KIWI_NUM_THREADS, original) + threads := kiwi.GetOption(KIWI_NUM_THREADS) + assert.True(t, threads >= 1) } func TestMatchOptionOOV(t *testing.T) { From 6427cec320b4ce15e417dcd1e4ade738921d4cdf Mon Sep 17 00:00:00 2001 From: "Chanyub.Park" Date: Tue, 11 Aug 2026 23:56:05 +0900 Subject: [PATCH 4/4] fix: address oracle code review findings - Fix CString memory leaks in New, NewBuilder, AddWord, LoadDict - Add error returns to New and NewBuilder (breaking API change) - Add thread safety documentation to Kiwi type - Add test coverage: New failure, WithOpenEnding, WithAllowedDialects - Fix error handling in all tests (assert.NoError) - Apply gofumpt formatting --- kiwi.go | 45 ++++++++++++++++++----- kiwi_example_test.go | 5 ++- kiwi_test.go | 85 ++++++++++++++++++++++++++++++++++---------- 3 files changed, 107 insertions(+), 28 deletions(-) diff --git a/kiwi.go b/kiwi.go index a36ee63..662acf6 100644 --- a/kiwi.go +++ b/kiwi.go @@ -292,6 +292,10 @@ func KiwiClearError() { } // Kiwi is a wrapper for the kiwi C library. +// +// Thread Safety: Concurrent calls to Analyze are safe. +// However, SetGlobalConfig must not be called concurrently with Analyze +// or other methods that read the configuration. type Kiwi struct { handler C.kiwi_h dialects Dialect @@ -299,7 +303,7 @@ type Kiwi struct { // New returns a new Kiwi instance. // Don't forget to call Close after this. -func New(modelPath string, opts ...Option) *Kiwi { +func New(modelPath string, opts ...Option) (*Kiwi, error) { options := kiwiOptions{ buildOptions: KIWI_BUILD_DEFAULT, dialects: DialectStandard, @@ -309,10 +313,18 @@ func New(modelPath string, opts ...Option) *Kiwi { opt(&options) } + cModelPath := C.CString(modelPath) + defer C.free(unsafe.Pointer(cModelPath)) + + h := C.kiwi_init(cModelPath, C.int(options.numThread), C.int(options.buildOptions), C.int(options.dialects)) + if h == nil { + return nil, fmt.Errorf("kiwi_init failed: %s", KiwiError()) + } + return &Kiwi{ - handler: C.kiwi_init(C.CString(modelPath), C.int(options.numThread), C.int(options.buildOptions), C.int(options.dialects)), + handler: h, dialects: options.dialects, - } + }, nil } // TokenInfo returns the token info for the given token(Str). @@ -476,7 +488,7 @@ type KiwiBuilder struct { // NewBuilder returns a new KiwiBuilder instance. // Don't forget to call Close after this. -func NewBuilder(modelPath string, opts ...Option) *KiwiBuilder { +func NewBuilder(modelPath string, opts ...Option) (*KiwiBuilder, error) { options := kiwiOptions{ buildOptions: KIWI_BUILD_DEFAULT, dialects: DialectStandard, @@ -486,19 +498,36 @@ func NewBuilder(modelPath string, opts ...Option) *KiwiBuilder { opt(&options) } - return &KiwiBuilder{ - handler: C.kiwi_builder_init(C.CString(modelPath), C.int(options.numThread), C.int(options.buildOptions), C.int(options.dialects)), + cModelPath := C.CString(modelPath) + defer C.free(unsafe.Pointer(cModelPath)) + + h := C.kiwi_builder_init(cModelPath, C.int(options.numThread), C.int(options.buildOptions), C.int(options.dialects)) + if h == nil { + return nil, fmt.Errorf("kiwi_builder_init failed: %s", KiwiError()) } + + return &KiwiBuilder{ + handler: h, + }, nil } // AddWord set custom word with word, pos, score. func (kb *KiwiBuilder) AddWord(word string, pos POSType, score float32) int { - return int(C.kiwi_builder_add_word(kb.handler, C.CString(word), C.CString(string(pos)), C.float(score))) + cWord := C.CString(word) + defer C.free(unsafe.Pointer(cWord)) + + cPos := C.CString(string(pos)) + defer C.free(unsafe.Pointer(cPos)) + + return int(C.kiwi_builder_add_word(kb.handler, cWord, cPos, C.float(score))) } // LoadDict loads user dict with dict file path. func (kb *KiwiBuilder) LoadDict(dictPath string) int { - return int(C.kiwi_builder_load_dict(kb.handler, C.CString(dictPath))) + cDictPath := C.CString(dictPath) + defer C.free(unsafe.Pointer(cDictPath)) + + return int(C.kiwi_builder_load_dict(kb.handler, cDictPath)) } // Build creates kiwi instance with user word etc. diff --git a/kiwi_example_test.go b/kiwi_example_test.go index c2294f4..b976bb1 100644 --- a/kiwi_example_test.go +++ b/kiwi_example_test.go @@ -7,7 +7,10 @@ import ( ) func Example() { - kb := kiwi.NewBuilder("./base", kiwi.WithNumThread(1), kiwi.WithBuildOption(kiwi.KIWI_BUILD_INTEGRATE_ALLOMORPH)) + kb, err := kiwi.NewBuilder("./base", kiwi.WithNumThread(1), kiwi.WithBuildOption(kiwi.KIWI_BUILD_INTEGRATE_ALLOMORPH)) + if err != nil { + panic(err) + } kb.AddWord("코딩냄비", "NNP", 0) k := kb.Build() diff --git a/kiwi_test.go b/kiwi_test.go index 5c7db65..e55076b 100644 --- a/kiwi_test.go +++ b/kiwi_test.go @@ -20,8 +20,10 @@ func TestKiwiVersion(t *testing.T) { } func TestAnalyze(t *testing.T) { - kiwi := New("./base", WithNumThread(1)) - res, _ := kiwi.Analyze("아버지가 방에 들어가신다") + kiwi, err := New("./base", WithNumThread(1)) + assert.NoError(t, err) + res, err := kiwi.Analyze("아버지가 방에 들어가신다") + assert.NoError(t, err) expected := []TokenResult{ { @@ -73,8 +75,10 @@ func TestAnalyze(t *testing.T) { } func TestSplitSentence(t *testing.T) { - kiwi := New("./base", WithNumThread(1)) - res, _ := kiwi.SplitSentence("여러 문장으로 구성된 텍스트네 이걸 분리해줘", KIWI_MATCH_ALL) + kiwi, err := New("./base", WithNumThread(1)) + assert.NoError(t, err) + res, err := kiwi.SplitSentence("여러 문장으로 구성된 텍스트네 이걸 분리해줘", KIWI_MATCH_ALL) + assert.NoError(t, err) expected := []SplitResult{ { @@ -94,7 +98,8 @@ func TestSplitSentence(t *testing.T) { } func TestAddWordFail(t *testing.T) { - kb := NewBuilder("./base", WithNumThread(1), WithBuildOption(KIWI_BUILD_INTEGRATE_ALLOMORPH)) + kb, err := NewBuilder("./base", WithNumThread(1), WithBuildOption(KIWI_BUILD_INTEGRATE_ALLOMORPH)) + assert.NoError(t, err) add := kb.AddWord("아버지가", "SKO", 0) assert.Equal(t, -1, add) assert.Equal(t, 0, kb.Close()) @@ -103,7 +108,8 @@ func TestAddWordFail(t *testing.T) { } func TestAddWord(t *testing.T) { - kb := NewBuilder("./base", WithNumThread(1), WithBuildOption(KIWI_BUILD_INTEGRATE_ALLOMORPH)) + kb, err := NewBuilder("./base", WithNumThread(1), WithBuildOption(KIWI_BUILD_INTEGRATE_ALLOMORPH)) + assert.NoError(t, err) add := kb.AddWord("아버지가", "NNG", 0) assert.Equal(t, 0, add) @@ -159,14 +165,15 @@ func TestAddWord(t *testing.T) { } func TestLoadDict(t *testing.T) { - kb := NewBuilder("./base", WithNumThread(1), WithBuildOption(KIWI_BUILD_INTEGRATE_ALLOMORPH)) + kb, err := NewBuilder("./base", WithNumThread(1), WithBuildOption(KIWI_BUILD_INTEGRATE_ALLOMORPH)) + assert.NoError(t, err) add := kb.LoadDict("./example/user_dict.tsv") assert.Equal(t, 1, add) - err := KiwiError() + errMsg := KiwiError() - assert.Equal(t, "", err) + assert.Equal(t, "", errMsg) kiwi := kb.Build() @@ -220,14 +227,15 @@ func TestLoadDict(t *testing.T) { } func TestLoadDict2(t *testing.T) { - kb := NewBuilder("./base", WithNumThread(1), WithBuildOption(KIWI_BUILD_INTEGRATE_ALLOMORPH)) + kb, err := NewBuilder("./base", WithNumThread(1), WithBuildOption(KIWI_BUILD_INTEGRATE_ALLOMORPH)) + assert.NoError(t, err) add := kb.LoadDict("./example/user_dict2.tsv") assert.Equal(t, 3, add) - err := KiwiError() + errMsg := KiwiError() - assert.Equal(t, "", err) + assert.Equal(t, "", errMsg) kiwi := kb.Build() res, _ := kiwi.Analyze("아버지가 방에 들어가신다") @@ -262,7 +270,8 @@ func TestLoadDict2(t *testing.T) { } func TestExtractWord(t *testing.T) { - kb := NewBuilder("./base", WithNumThread(1)) + kb, err := NewBuilder("./base", WithNumThread(1)) + assert.NoError(t, err) rs := strings.NewReader(`2008년에는 애국가의 작곡자 안익태가 1930년대에 독일 유학 기간 중 친일 활동을 했다는 사실이 밝혀졌다. 이후 안익태가 나치 독일 하의 베를린에서 만주국 10주년 건국 기념음악회를 지휘하는 동영상까지 발굴되어 관련 학계나 사회에 큰 충격을 주었다. 안익태가 친일 행적을 한 바 있다는 빼도박도 못할 증거가 나왔으니까. 영상물의 '만주환상곡'에는 우리가 현재 알고있는 '한국환상곡'의 두 선율("무궁화 삼천리 나의 사랑아, @@ -294,7 +303,8 @@ func TestExtractWord(t *testing.T) { } func TestExtractWordwithFile(t *testing.T) { - kb := NewBuilder("./base", WithNumThread(1)) // Use single thread for deterministic results + kb, err := NewBuilder("./base", WithNumThread(1)) // Use single thread for deterministic results + assert.NoError(t, err) file, _ := os.Open("./example/test.txt") wordInfos, _ := kb.ExtractWords(file, 10 /*=minCnt*/, 5 /*=maxWordLen*/, 0.0 /*=minScore*/, -25.0 /*=posThreshold*/) @@ -308,7 +318,8 @@ func TestExtractWordwithFile(t *testing.T) { } func TestGetGlobalConfig(t *testing.T) { - kiwi := New("./base", WithNumThread(1)) + kiwi, err := New("./base", WithNumThread(1)) + assert.NoError(t, err) defer kiwi.Close() config := kiwi.GetGlobalConfig() @@ -319,7 +330,8 @@ func TestGetGlobalConfig(t *testing.T) { } func TestSetGlobalConfig(t *testing.T) { - kiwi := New("./base", WithNumThread(1)) + kiwi, err := New("./base", WithNumThread(1)) + assert.NoError(t, err) defer kiwi.Close() originalConfig := kiwi.GetGlobalConfig() @@ -335,7 +347,8 @@ func TestSetGlobalConfig(t *testing.T) { } func TestGetSetOption(t *testing.T) { - kiwi := New("./base", WithNumThread(1)) + kiwi, err := New("./base", WithNumThread(1)) + assert.NoError(t, err) defer kiwi.Close() threads := kiwi.GetOption(KIWI_NUM_THREADS) @@ -343,7 +356,8 @@ func TestGetSetOption(t *testing.T) { } func TestMatchOptionOOV(t *testing.T) { - kiwi := New("./base", WithNumThread(1)) + kiwi, err := New("./base", WithNumThread(1)) + assert.NoError(t, err) defer kiwi.Close() // Use OOV-containing text to detect differences between OOV modes @@ -361,7 +375,8 @@ func TestMatchOptionOOV(t *testing.T) { } func TestMorphset(t *testing.T) { - kiwi := New("./base", WithNumThread(1)) + kiwi, err := New("./base", WithNumThread(1)) + assert.NoError(t, err) defer kiwi.Close() ms, err := kiwi.NewMorphset() @@ -383,3 +398,35 @@ func TestMorphset(t *testing.T) { assert.NotEqual(t, "아버지", token.Form) } } + +func TestNewFailure(t *testing.T) { + _, err := New("./nonexistent_path") + assert.Error(t, err) + assert.Contains(t, err.Error(), "kiwi_init failed") +} + +func TestWithOpenEnding(t *testing.T) { + kiwi, err := New("./base", WithNumThread(1)) + assert.NoError(t, err) + defer kiwi.Close() + + res, err := kiwi.Analyze("아버지가 방에 들어가신다", WithOpenEnding(true)) + assert.NoError(t, err) + assert.True(t, len(res) > 0) +} + +func TestWithAllowedDialects(t *testing.T) { + kiwi, err := New("./base", WithNumThread(1), WithDialect(DialectGyeongsang)) + assert.NoError(t, err) + defer kiwi.Close() + + // Test with default dialects (should use instance setting) + res, err := kiwi.Analyze("아버지가 방에 들어가신다") + assert.NoError(t, err) + assert.True(t, len(res) > 0) + + // Test with explicit dialect override + res, err = kiwi.Analyze("아버지가 방에 들어가신다", WithAllowedDialects(DialectStandard)) + assert.NoError(t, err) + assert.True(t, len(res) > 0) +}