@@ -87,9 +87,9 @@ static float tokenBestMatch(std::string_view qt, std::string_view lastQ, const s
8787 return (best - MIN_FUZZY_TO_COUNT ) / (1 .F - MIN_FUZZY_TO_COUNT );
8888}
8989
90- static float scoreCandidate (std::string_view query, std::string_view cand, float freq) {
91- CVarList2 qTokens (std::string{query}, 0 , ' s ' , true , false );
92- CVarList2 cTokens (std::string{cand}, 0 , ' s ' , true , false );
90+ static float scoreCandidate (std::string_view query, std::string_view cand, float freq, char tokenBreak ) {
91+ CVarList2 qTokens (std::string{query}, 0 , tokenBreak == ' ' ? ' s ' : tokenBreak , true , false );
92+ CVarList2 cTokens (std::string{cand}, 0 , tokenBreak == ' ' ? ' s ' : tokenBreak , true , false );
9393
9494 std::vector<std::string_view> qTok, cTok;
9595 qTok.reserve (qTokens.size ());
@@ -145,11 +145,11 @@ struct SScoreData {
145145 size_t idx = 0 ;
146146};
147147
148- static void workerFn (std::vector<SScoreData>& scores, const std::vector<SP <IFinderResult>>& in, const std::string& query, size_t start, size_t end) {
148+ static void workerFn (std::vector<SScoreData>& scores, const std::vector<SP <IFinderResult>>& in, const std::string& query, size_t start, size_t end, char tokenBreak ) {
149149 for (size_t i = start; i < end; ++i) {
150150 auto & ref = scores[i];
151151
152- ref.score = scoreCandidate (query, in[i]->fuzzable (), in[i]->frequency ());
152+ ref.score = scoreCandidate (query, in[i]->fuzzable (), in[i]->frequency (), tokenBreak );
153153
154154 ref.result = in[i];
155155 ref.idx = i;
@@ -185,7 +185,7 @@ static std::vector<SP<IFinderResult>> getBestResultsStable(std::vector<SScoreDat
185185static constexpr const decltype (sysconf(0 )) MAX_THREADS = 10;
186186
187187//
188- std::vector<SP <IFinderResult>> Fuzzy::getNResults (const std::vector<SP <IFinderResult>>& in, const std::string& query, size_t results) {
188+ std::vector<SP <IFinderResult>> Fuzzy::getNResults (const std::vector<SP <IFinderResult>>& in, const std::string& query, size_t results, char tokenBreak ) {
189189 std::vector<SScoreData> scores;
190190 scores.resize (in.size ());
191191
@@ -203,10 +203,10 @@ std::vector<SP<IFinderResult>> Fuzzy::getNResults(const std::vector<SP<IFinderRe
203203 size_t workElDone = 0 , workElPerThread = in.size () / THREADS ;
204204 for (long i = 0 ; i < THREADS ; ++i) {
205205 if (i == THREADS - 1 ) {
206- workerThreads[i] = std::thread ([&, begin = workElDone] { workerFn (scores, in, query, begin, in.size ()); });
206+ workerThreads[i] = std::thread ([&, begin = workElDone] { workerFn (scores, in, query, begin, in.size (), tokenBreak ); });
207207 break ;
208208 }
209- workerThreads[i] = std::thread ([&, begin = workElDone, end = workElDone + workElPerThread] { workerFn (scores, in, query, begin, end); });
209+ workerThreads[i] = std::thread ([&, begin = workElDone, end = workElDone + workElPerThread] { workerFn (scores, in, query, begin, end, tokenBreak ); });
210210
211211 workElDone += workElPerThread;
212212 }
@@ -218,7 +218,7 @@ std::vector<SP<IFinderResult>> Fuzzy::getNResults(const std::vector<SP<IFinderRe
218218
219219 workerThreads.clear ();
220220 } else
221- workerFn (scores, in, query, 0 , in.size ());
221+ workerFn (scores, in, query, 0 , in.size (), tokenBreak );
222222
223223 return getBestResultsStable (scores, results);
224224}
0 commit comments