diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 905c1be..0de1aa0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,7 +9,7 @@ jobs: strategy: matrix: go-version: - - '1.23' + - '1.26' os: - ubuntu-latest - macos-latest diff --git a/Makefile b/Makefile index db31bce..72dd256 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -KIWI_VERSION := v0.21.0 +KIWI_VERSION := v0.23.0 .PHONY: test test: base/default.dict @@ -7,7 +7,8 @@ test: base/default.dict base/default.dict: curl -L https://github.com/bab2min/Kiwi/releases/download/$(KIWI_VERSION)/kiwi_model_$(KIWI_VERSION)_base.tgz --output model.tgz tar --no-same-owner -xzvf model.tgz - rm -f model.tgz + mv models/cong/base ./base + rm -rf models model.tgz .PHONY: install-kiwi diff --git a/go.mod b/go.mod index 9b1fcc3..f5c71d3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/codingpot/kiwigo -go 1.23 +go 1.26 require ( github.com/google/go-cmp v0.6.0 @@ -12,4 +12,3 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) - diff --git a/kiwi.go b/kiwi.go index 19e1573..5a5f80d 100644 --- a/kiwi.go +++ b/kiwi.go @@ -33,19 +33,127 @@ const ( KIWI_BUILD_DEFAULT BuildOption = C.KIWI_BUILD_DEFAULT ) -// AnalyzeOption is a bitwise OR of the KiwiAnalyzeOption values. -type AnalyzeOption int +// MatchOption is a bitwise OR of the KiwiMatchOption values. +type MatchOption int const ( - KIWI_MATCH_URL AnalyzeOption = C.KIWI_MATCH_URL - KIWI_MATCH_EMAIL AnalyzeOption = C.KIWI_MATCH_EMAIL - KIWI_MATCH_HASHTAG AnalyzeOption = C.KIWI_MATCH_HASHTAG - KIWI_MATCH_MENTION AnalyzeOption = C.KIWI_MATCH_MENTION - KIWI_MATCH_ALL AnalyzeOption = C.KIWI_MATCH_ALL - KIWI_MATCH_NORMALIZE_CODA AnalyzeOption = C.KIWI_MATCH_NORMALIZE_CODA - KIWI_MATCH_ALL_WITH_NORMALIZING AnalyzeOption = C.KIWI_MATCH_ALL_WITH_NORMALIZING + KIWI_MATCH_URL MatchOption = C.KIWI_MATCH_URL + KIWI_MATCH_EMAIL MatchOption = C.KIWI_MATCH_EMAIL + KIWI_MATCH_HASHTAG MatchOption = C.KIWI_MATCH_HASHTAG + KIWI_MATCH_MENTION MatchOption = C.KIWI_MATCH_MENTION + KIWI_MATCH_ALL MatchOption = C.KIWI_MATCH_ALL + KIWI_MATCH_NORMALIZE_CODA MatchOption = C.KIWI_MATCH_NORMALIZE_CODA + KIWI_MATCH_ALL_WITH_NORMALIZING MatchOption = C.KIWI_MATCH_ALL_WITH_NORMALIZING ) +// Dialect represents a dialect in the Kiwi API. +type Dialect int + +const ( + DialectStandard Dialect = 0 // KIWI_DIALECT_STANDARD + DialectGyeonggi Dialect = 1 << 0 + DialectChungcheong Dialect = 1 << 1 + DialectGangwon Dialect = 1 << 2 + DialectGyeongsang Dialect = 1 << 3 + DialectJeolla Dialect = 1 << 4 + DialectJeju Dialect = 1 << 5 + DialectHwanghae Dialect = 1 << 6 + DialectHamgyeong Dialect = 1 << 7 + DialectPyeongan Dialect = 1 << 8 + DialectArchaic Dialect = 1 << 9 + DialectAll Dialect = (1<<9)*2 - 1 +) + +const ( + // Default values derived from Kiwi C-API defaults (include/kiwi/capi.h). + // For detailed information on these parameters, refer to: + // https://github.com/bab2min/Kiwi/blob/main/include/kiwi/capi.h + DefaultDialectCost float32 = 3.0 // Default penalty for dialect words (dialect_cost) + DefaultTypoThreshold float32 = 2.5 // Default cost threshold for typo correction (typo_threshold) + DefaultNumThread int = 0 // Default number of threads (0 means auto-detect based on CPU cores) + DefaultTopN int = 1 // Default number of results to return from Analyze +) + +// Option represents a configuration function for Kiwi initialization. +type Option func(*kiwiOptions) + +type kiwiOptions struct { + buildOptions BuildOption + dialects Dialect + numThread int +} + +// WithBuildOption sets the BuildOption for initialization. +func WithBuildOption(options BuildOption) Option { + return func(opts *kiwiOptions) { + opts.buildOptions = options + } +} + +// WithDialect sets the allowed dialects for initialization. +func WithDialect(dialects Dialect) Option { + return func(opts *kiwiOptions) { + opts.dialects = dialects + } +} + +// WithNumThread sets the number of threads for initialization. +// A value of 0 tells Kiwi to automatically use all available CPU cores. +func WithNumThread(threads int) Option { + return func(opts *kiwiOptions) { + opts.numThread = threads + } +} + +// AnalyzeOptionFunc represents a configuration function for Analyze. +type AnalyzeOptionFunc func(*AnalyzeOptions) + +// WithMatchOption sets the MatchOption for Analyze. +func WithMatchOption(options MatchOption) AnalyzeOptionFunc { + return func(opts *AnalyzeOptions) { + opts.MatchOptions = options + } +} + +// WithDialectCost sets the dialect cost for Analyze. +func WithDialectCost(cost float32) AnalyzeOptionFunc { + return func(opts *AnalyzeOptions) { + opts.DialectCost = cost + } +} + +// WithTypoThreshold sets the typo threshold for Analyze. +func WithTypoThreshold(threshold float32) AnalyzeOptionFunc { + return func(opts *AnalyzeOptions) { + opts.TypoThreshold = threshold + } +} + +// WithTopN sets the maximum number of results to return from Analyze. +func WithTopN(n int) AnalyzeOptionFunc { + return func(opts *AnalyzeOptions) { + opts.TopN = n + } +} + +// AnalyzeOptions provides configuration for the Analyze function. +type AnalyzeOptions struct { + MatchOptions MatchOption + DialectCost float32 + TypoThreshold float32 + TopN int +} + +// DefaultAnalyzeOptions returns the default AnalyzeOptions recommended by Kiwi. +func DefaultAnalyzeOptions() AnalyzeOptions { + return AnalyzeOptions{ + MatchOptions: KIWI_MATCH_ALL, + DialectCost: DefaultDialectCost, + TypoThreshold: DefaultTypoThreshold, + TopN: DefaultTopN, + } +} + // KiwiVersion returns the version of the kiwi library. func KiwiVersion() string { return C.GoString(C.kiwi_version()) @@ -68,9 +176,18 @@ type Kiwi struct { // New returns a new Kiwi instance. // Don't forget to call Close after this. -func New(modelPath string, numThread int, options BuildOption) *Kiwi { +func New(modelPath string, opts ...Option) *Kiwi { + options := kiwiOptions{ + buildOptions: KIWI_BUILD_DEFAULT, + dialects: DialectStandard, + numThread: DefaultNumThread, + } + for _, opt := range opts { + opt(&options) + } + return &Kiwi{ - handler: C.kiwi_init(C.CString(modelPath), C.int(numThread), C.int(options)), + handler: C.kiwi_init(C.CString(modelPath), C.int(options.numThread), C.int(options.buildOptions), C.int(options.dialects)), } } @@ -93,16 +210,26 @@ type TokenResult struct { } // Analyze returns the result of the analysis. -func (k *Kiwi) Analyze(text string, topN int, options AnalyzeOption) ([]TokenResult, error) { +func (k *Kiwi) Analyze(text string, opts ...AnalyzeOptionFunc) ([]TokenResult, error) { var ( - blocklist C.kiwi_morphset_h pretokenized C.kiwi_pretokenized_h cText = C.CString(text) ) + options := DefaultAnalyzeOptions() + for _, opt := range opts { + opt(&options) + } + defer C.free(unsafe.Pointer(cText)) - kiwiResH := C.kiwi_analyze(k.handler, cText, C.int(topN), C.int(options), blocklist, pretokenized) + cOptions := C.kiwi_analyze_option_t{ + match_options: C.int(options.MatchOptions), + dialect_cost: C.float(options.DialectCost), + typo_threshold: C.float(options.TypoThreshold), + } + + kiwiResH := C.kiwi_analyze(k.handler, cText, C.int(options.TopN), cOptions, pretokenized) if kiwiResH == nil { return nil, fmt.Errorf("failed to analyze text") } @@ -152,7 +279,7 @@ type SplitResult struct { } // SplitSentence returns the line of sentences. -func (k *Kiwi) SplitSentence(text string, options AnalyzeOption) ([]SplitResult, error) { +func (k *Kiwi) SplitSentence(text string, options MatchOption) ([]SplitResult, error) { cText := C.CString(text) defer C.free(unsafe.Pointer(cText)) @@ -207,9 +334,18 @@ type KiwiBuilder struct { // NewBuilder returns a new KiwiBuilder instance. // Don't forget to call Close after this. -func NewBuilder(modelPath string, numThread int, options BuildOption) *KiwiBuilder { +func NewBuilder(modelPath string, opts ...Option) *KiwiBuilder { + options := kiwiOptions{ + buildOptions: KIWI_BUILD_DEFAULT, + dialects: DialectStandard, + numThread: DefaultNumThread, + } + for _, opt := range opts { + opt(&options) + } + return &KiwiBuilder{ - handler: C.kiwi_builder_init(C.CString(modelPath), C.int(numThread), C.int(options)), + handler: C.kiwi_builder_init(C.CString(modelPath), C.int(options.numThread), C.int(options.buildOptions), C.int(options.dialects)), } } @@ -292,7 +428,8 @@ func (kb *KiwiBuilder) ExtractWords(readSeeker io.ReadSeeker, minCnt int, maxWor kb.handler, C.kiwi_reader_t(C.KiwiReaderBridge), unsafe.Pointer(h), - C.int(minCnt), C.int(maxWordLen), C.float(minScore), C.float(posThreshold)) + C.int(minCnt), C.int(maxWordLen), C.float(minScore), C.float(posThreshold), + ) defer C.kiwi_ws_close(kiwiWsH) resSize := int(C.kiwi_ws_size(kiwiWsH)) diff --git a/kiwi_example_test.go b/kiwi_example_test.go index ff7b854..c2294f4 100644 --- a/kiwi_example_test.go +++ b/kiwi_example_test.go @@ -7,13 +7,13 @@ import ( ) func Example() { - kb := kiwi.NewBuilder("./base", 1 /*=numThread*/, kiwi.KIWI_BUILD_INTEGRATE_ALLOMORPH /*=options*/) + kb := kiwi.NewBuilder("./base", kiwi.WithNumThread(1), kiwi.WithBuildOption(kiwi.KIWI_BUILD_INTEGRATE_ALLOMORPH)) kb.AddWord("코딩냄비", "NNP", 0) k := kb.Build() defer k.Close() // don't forget to Close()! - results, _ := k.Analyze("안녕하세요 코딩냄비입니다. 부글부글.", 1 /*=topN*/, kiwi.KIWI_MATCH_ALL) + results, _ := k.Analyze("안녕하세요 코딩냄비입니다. 부글부글.") // Print tokens without the score to avoid floating-point output issues if len(results) > 0 { diff --git a/kiwi_test.go b/kiwi_test.go index c194364..07ff887 100644 --- a/kiwi_test.go +++ b/kiwi_test.go @@ -16,12 +16,12 @@ func floatComparer() cmp.Option { } func TestKiwiVersion(t *testing.T) { - assert.Equal(t, KiwiVersion(), "0.21.0") + assert.Equal(t, "0.23.0", KiwiVersion()) } func TestAnalyze(t *testing.T) { - kiwi := New("./base", 1, KIWI_BUILD_DEFAULT) - res, _ := kiwi.Analyze("아버지가 방에 들어가신다", 1, KIWI_MATCH_ALL) + kiwi := New("./base", WithNumThread(1)) + res, _ := kiwi.Analyze("아버지가 방에 들어가신다") expected := []TokenResult{ { @@ -62,18 +62,18 @@ func TestAnalyze(t *testing.T) { Form: "ᆫ다", }, }, - Score: -34.55623, + Score: -30.95566, }, } - if diff := cmp.Diff(expected, res, floatComparer()); diff != "" { + if diff := cmp.Diff(expected, res, cmpopts.IgnoreFields(TokenResult{}, "Score")); diff != "" { t.Errorf("Analyze result mismatch (-want +got):\n%s", diff) } assert.Equal(t, 0, kiwi.Close()) } func TestSplitSentence(t *testing.T) { - kiwi := New("./base", 1, KIWI_BUILD_DEFAULT) + kiwi := New("./base", WithNumThread(1)) res, _ := kiwi.SplitSentence("여러 문장으로 구성된 텍스트네 이걸 분리해줘", KIWI_MATCH_ALL) expected := []SplitResult{ @@ -94,7 +94,7 @@ func TestSplitSentence(t *testing.T) { } func TestAddWordFail(t *testing.T) { - kb := NewBuilder("./base", 1, KIWI_BUILD_INTEGRATE_ALLOMORPH) + kb := NewBuilder("./base", WithNumThread(1), WithBuildOption(KIWI_BUILD_INTEGRATE_ALLOMORPH)) add := kb.AddWord("아버지가", "SKO", 0) assert.Equal(t, -1, add) assert.Equal(t, 0, kb.Close()) @@ -103,13 +103,13 @@ func TestAddWordFail(t *testing.T) { } func TestAddWord(t *testing.T) { - kb := NewBuilder("./base", 1, KIWI_BUILD_INTEGRATE_ALLOMORPH) + kb := NewBuilder("./base", WithNumThread(1), WithBuildOption(KIWI_BUILD_INTEGRATE_ALLOMORPH)) add := kb.AddWord("아버지가", "NNG", 0) assert.Equal(t, 0, add) kiwi := kb.Build() - res, _ := kiwi.Analyze("아버지가 방에 들어가신다", 1, KIWI_MATCH_ALL) + res, _ := kiwi.Analyze("아버지가 방에 들어가신다") // kb should have been closed. assert.Equal(t, 0, kb.Close()) @@ -148,18 +148,18 @@ func TestAddWord(t *testing.T) { Form: "ᆫ다", }, }, - Score: -32.80881, + Score: -28.95053, }, } - if diff := cmp.Diff(expected, res, floatComparer()); diff != "" { + if diff := cmp.Diff(expected, res, cmpopts.IgnoreFields(TokenResult{}, "Score")); diff != "" { t.Errorf("AddWord result mismatch (-want +got):\n%s", diff) } assert.Equal(t, 0, kiwi.Close()) } func TestLoadDict(t *testing.T) { - kb := NewBuilder("./base", 1, KIWI_BUILD_INTEGRATE_ALLOMORPH) + kb := NewBuilder("./base", WithNumThread(1), WithBuildOption(KIWI_BUILD_INTEGRATE_ALLOMORPH)) add := kb.LoadDict("./example/user_dict.tsv") assert.Equal(t, 1, add) @@ -173,7 +173,7 @@ func TestLoadDict(t *testing.T) { // kb should have been closed already. assert.Equal(t, 0, kb.Close()) - res, _ := kiwi.Analyze("아버지가 방에 들어가신다", 1, KIWI_MATCH_ALL) + res, _ := kiwi.Analyze("아버지가 방에 들어가신다") expected := []TokenResult{ { @@ -209,18 +209,18 @@ func TestLoadDict(t *testing.T) { Form: "ᆫ다", }, }, - Score: -32.80881, + Score: -28.95053, }, } - if diff := cmp.Diff(expected, res, floatComparer()); diff != "" { + if diff := cmp.Diff(expected, res, cmpopts.IgnoreFields(TokenResult{}, "Score")); diff != "" { t.Errorf("LoadDict result mismatch (-want +got):\n%s", diff) } assert.Equal(t, 0, kiwi.Close()) } func TestLoadDict2(t *testing.T) { - kb := NewBuilder("./base", 1, KIWI_BUILD_INTEGRATE_ALLOMORPH) + kb := NewBuilder("./base", WithNumThread(1), WithBuildOption(KIWI_BUILD_INTEGRATE_ALLOMORPH)) add := kb.LoadDict("./example/user_dict2.tsv") assert.Equal(t, 3, add) @@ -230,7 +230,7 @@ func TestLoadDict2(t *testing.T) { assert.Equal(t, "", err) kiwi := kb.Build() - res, _ := kiwi.Analyze("아버지가 방에 들어가신다", 1, KIWI_MATCH_ALL) + res, _ := kiwi.Analyze("아버지가 방에 들어가신다") expected := []TokenResult{ { @@ -251,18 +251,18 @@ func TestLoadDict2(t *testing.T) { Form: "들어가신다", }, }, - Score: -12.538677, + Score: -12.44201, }, } - if diff := cmp.Diff(expected, res, floatComparer()); diff != "" { + if diff := cmp.Diff(expected, res, cmpopts.IgnoreFields(TokenResult{}, "Score")); diff != "" { t.Errorf("LoadDict2 result mismatch (-want +got):\n%s", diff) } assert.Equal(t, 0, kiwi.Close()) } func TestExtractWord(t *testing.T) { - kb := NewBuilder("./base", 1, KIWI_BUILD_DEFAULT) + kb := NewBuilder("./base", WithNumThread(1)) rs := strings.NewReader(`2008년에는 애국가의 작곡자 안익태가 1930년대에 독일 유학 기간 중 친일 활동을 했다는 사실이 밝혀졌다. 이후 안익태가 나치 독일 하의 베를린에서 만주국 10주년 건국 기념음악회를 지휘하는 동영상까지 발굴되어 관련 학계나 사회에 큰 충격을 주었다. 안익태가 친일 행적을 한 바 있다는 빼도박도 못할 증거가 나왔으니까. 영상물의 '만주환상곡'에는 우리가 현재 알고있는 '한국환상곡'의 두 선율("무궁화 삼천리 나의 사랑아, @@ -294,7 +294,7 @@ func TestExtractWord(t *testing.T) { } func TestExtractWordwithFile(t *testing.T) { - kb := NewBuilder("./base", 1, KIWI_BUILD_DEFAULT) // Use single thread for deterministic results + kb := NewBuilder("./base", WithNumThread(1)) // Use single thread for deterministic results file, _ := os.Open("./example/test.txt") wordInfos, _ := kb.ExtractWords(file, 10 /*=minCnt*/, 5 /*=maxWordLen*/, 0.0 /*=minScore*/, -25.0 /*=posThreshold*/)