3131#include "foundation/profile.h"
3232
3333#include <ctype.h>
34+ #include <float.h>
3435#include <math.h>
3536#include <stdatomic.h>
3637#include <stddef.h>
@@ -2112,28 +2113,59 @@ static int doc_section_tokens(const cbm_gbuf_node_t *n, const char *body, char *
21122113 return count ;
21132114}
21142115
2115- /* Dot product and shared-term count of two ascending term lists. */
2116- static float doc_terms_dot (const int * ia , const float * wa , int na , const int * ib , const float * wb ,
2117- int nb , int * shared ) {
2116+ /* Dot product and shared-term count of a section and an ascending term list
2117+ * (a function's, a file's). The section is held densely, secw/secmark indexed
2118+ * by term with its terms marked `stamp`, so the cost is b's length alone, not
2119+ * the section's (up to SEM_DOC_MAX_TOKENS terms) added to it. The shared terms
2120+ * are met in ascending order, as a merge of the two lists would meet them. */
2121+ static float doc_terms_dot_dense (const float * secw , const int * secmark , int stamp , const int * ib ,
2122+ const float * wb , int nb , int * shared ) {
21182123 float dot = 0.0F ;
2119- int a = 0 ;
2120- int b = 0 ;
21212124 * shared = 0 ;
2122- while (a < na && b < nb ) {
2123- if (ia [a ] == ib [b ]) {
2124- dot += wa [a ] * wb [b ];
2125+ for (int b = 0 ; b < nb ; b ++ ) {
2126+ int t = ib [b ];
2127+ if (secmark [t ] == stamp ) {
2128+ dot += secw [t ] * wb [b ];
21252129 (* shared )++ ;
2126- a ++ ;
2127- b ++ ;
2128- } else if (ia [a ] < ib [b ]) {
2129- a ++ ;
2130- } else {
2131- b ++ ;
21322130 }
21332131 }
21342132 return dot ;
21352133}
21362134
2135+ /* The candidates a section's results read, moved to the front: those that
2136+ * the full cmp_doc_cand order puts in the top k, and every one scoring floor
2137+ * or more. Everything left out scores below both, so sorting the kept prefix
2138+ * gives the full sort's prefix exactly. Returns how many are kept. */
2139+ static int doc_cand_prefix (doc_cand_t * c , int nc , int k , float floor ) {
2140+ if (nc <= k ) {
2141+ return nc ;
2142+ }
2143+ float top [SEM_DOC_TOP_K ];
2144+ int n = 0 ;
2145+ for (int i = 0 ; i < nc ; i ++ ) { /* the k best scores, descending */
2146+ float v = c [i ].score ;
2147+ if (n == k && v <= top [k - SKIP_ONE ]) {
2148+ continue ;
2149+ }
2150+ int j = n < k ? n ++ : k - SKIP_ONE ;
2151+ while (j > 0 && top [j - SKIP_ONE ] < v ) {
2152+ top [j ] = top [j - SKIP_ONE ];
2153+ j -- ;
2154+ }
2155+ top [j ] = v ;
2156+ }
2157+ float cut = top [k - SKIP_ONE ] < floor ? top [k - SKIP_ONE ] : floor ;
2158+ int kept = 0 ;
2159+ for (int i = 0 ; i < nc ; i ++ ) {
2160+ if (c [i ].score >= cut ) {
2161+ doc_cand_t t = c [kept ];
2162+ c [kept ++ ] = c [i ];
2163+ c [i ] = t ;
2164+ }
2165+ }
2166+ return kept ;
2167+ }
2168+
21372169static float terms_norm (const float * w , int n ) {
21382170 float s = 0.0F ;
21392171 for (int k = 0 ; k < n ; k ++ ) {
@@ -2602,6 +2634,7 @@ typedef struct {
26022634 const char * const * home ; /* per section: its doc's home ("" = the root) */
26032635 const doc_files_t * files ;
26042636 int file_key_df ;
2637+ int nterms ; /* the corpus's terms: the dense section vector's length */
26052638 doc_result_t * res ;
26062639 _Atomic int next ;
26072640} doc_ctx_t ;
@@ -2634,8 +2667,8 @@ static void doc_name_evidence(const doc_ctx_t *dc, const int *sidx, int ns, doc_
26342667
26352668/* The section's best CBM_SEM_DOC_FILE_K whole files, retrieved by the
26362669 * section's key terms over the files' postings, scored by cosine. */
2637- static void doc_section_files (doc_ctx_t * dc , int s , const int * sidx , const float * sw , int ns ,
2638- float snorm , int * fstamp , doc_cand_t * fc ) {
2670+ static void doc_section_files (doc_ctx_t * dc , int s , const int * sidx , int ns , float snorm ,
2671+ const float * secw , const int * secmark , int * fstamp , doc_cand_t * fc ) {
26392672 const doc_files_t * df = dc -> files ;
26402673 doc_result_t * r = & dc -> res [s ];
26412674 int nc = 0 ;
@@ -2655,11 +2688,12 @@ static void doc_section_files(doc_ctx_t *dc, int s, const int *sidx, const float
26552688 for (int c = 0 ; c < nc ; c ++ ) {
26562689 int f = fc [c ].func ;
26572690 int len = df -> off [f + SKIP_ONE ] - df -> off [f ];
2658- float dot = doc_terms_dot ( sidx , sw , ns , df -> idx + df -> off [f ], df -> w + df -> off [f ], len ,
2659- & fc [c ].shared );
2691+ float dot = doc_terms_dot_dense ( secw , secmark , s , df -> idx + df -> off [f ], df -> w + df -> off [f ],
2692+ len , & fc [c ].shared );
26602693 float denom = snorm * df -> norm [f ];
26612694 fc [c ].score = denom > 0.0F ? dot / denom : 0.0F ;
26622695 }
2696+ nc = doc_cand_prefix (fc , nc , CBM_SEM_DOC_FILE_K , FLT_MAX ); /* the best files only */
26632697 if (nc > 1 ) {
26642698 qsort (fc , (size_t )nc , sizeof (fc [0 ]), cmp_doc_cand );
26652699 }
@@ -2694,7 +2728,8 @@ static void doc_section_local(doc_ctx_t *dc, int s, const doc_cand_t *cands, int
26942728}
26952729
26962730static void doc_section_one (doc_ctx_t * dc , int s , int * stamp , doc_cand_t * cands , int * sidx ,
2697- float * sw , char * body , int * fstamp , doc_cand_t * fc ) {
2731+ float * sw , char * body , int * fstamp , doc_cand_t * fc , float * secw ,
2732+ int * secmark ) {
26982733 doc_result_t * r = & dc -> res [s ];
26992734 r -> n = 0 ;
27002735 r -> nlocal = 0 ;
@@ -2709,6 +2744,10 @@ static void doc_section_one(doc_ctx_t *dc, int s, int *stamp, doc_cand_t *cands,
27092744 cbm_free (CBM_MEM_CLASS_SEMANTIC , tokens [t ]);
27102745 }
27112746 float snorm = terms_norm (sw , ns );
2747+ for (int k = 0 ; k < ns ; k ++ ) { /* the section, dense */
2748+ secw [sidx [k ]] = sw [k ];
2749+ secmark [sidx [k ]] = s ;
2750+ }
27122751 int nc = 0 ;
27132752 for (int k = 0 ; k < ns && snorm > 0.0F ; k ++ ) {
27142753 int t = sidx [k ];
@@ -2725,11 +2764,13 @@ static void doc_section_one(doc_ctx_t *dc, int s, int *stamp, doc_cand_t *cands,
27252764 }
27262765 for (int c = 0 ; c < nc ; c ++ ) {
27272766 const cbm_sem_func_t * fn = & dc -> funcs [cands [c ].func ];
2728- float dot = doc_terms_dot ( sidx , sw , ns , fn -> tfidf_indices , fn -> tfidf_weights , fn -> tfidf_len ,
2729- & cands [c ].shared );
2767+ float dot = doc_terms_dot_dense ( secw , secmark , s , fn -> tfidf_indices , fn -> tfidf_weights ,
2768+ fn -> tfidf_len , & cands [c ].shared );
27302769 float denom = snorm * dc -> fnorm [cands [c ].func ];
27312770 cands [c ].score = denom > 0.0F ? dot / denom : 0.0F ;
27322771 }
2772+ /* the top SEM_DOC_TOP_K, and the local ones down to CBM_SEM_DOC_MIN_SCORE */
2773+ nc = doc_cand_prefix (cands , nc , SEM_DOC_TOP_K , CBM_SEM_DOC_MIN_SCORE );
27332774 if (nc > 1 ) {
27342775 qsort (cands , (size_t )nc , sizeof (cands [0 ]), cmp_doc_cand );
27352776 }
@@ -2742,7 +2783,7 @@ static void doc_section_one(doc_ctx_t *dc, int s, int *stamp, doc_cand_t *cands,
27422783 doc_name_evidence (dc , sidx , ns , hit );
27432784 }
27442785 doc_section_local (dc , s , cands , nc , sidx , ns );
2745- doc_section_files (dc , s , sidx , sw , ns , snorm , fstamp , fc );
2786+ doc_section_files (dc , s , sidx , ns , snorm , secw , secmark , fstamp , fc );
27462787}
27472788
27482789static void doc_section_worker (int worker_id , void * ctx_ptr ) {
@@ -2757,7 +2798,13 @@ static void doc_section_worker(int worker_id, void *ctx_ptr) {
27572798 int * sidx = cbm_alloc (CBM_MEM_CLASS_SEMANTIC , (size_t )SEM_DOC_MAX_TOKENS * sizeof (int ));
27582799 float * sw = cbm_alloc (CBM_MEM_CLASS_SEMANTIC , (size_t )SEM_DOC_MAX_TOKENS * sizeof (float ));
27592800 char * body = cbm_alloc (CBM_MEM_CLASS_SEMANTIC , SEM_DOC_TEXT_MAX );
2760- if (stamp && cands && fstamp && fc && sidx && sw && body ) {
2801+ float * secw =
2802+ cbm_alloc (CBM_MEM_CLASS_SEMANTIC , ((size_t )dc -> nterms + SKIP_ONE ) * sizeof (float ));
2803+ int * secmark = cbm_alloc (CBM_MEM_CLASS_SEMANTIC , ((size_t )dc -> nterms + SKIP_ONE ) * sizeof (int ));
2804+ if (stamp && cands && fstamp && fc && sidx && sw && body && secw && secmark ) {
2805+ for (int t = 0 ; t < dc -> nterms ; t ++ ) {
2806+ secmark [t ] = -1 ;
2807+ }
27612808 for (int f = 0 ; f < dc -> func_count ; f ++ ) {
27622809 stamp [f ] = -1 ;
27632810 }
@@ -2769,7 +2816,7 @@ static void doc_section_worker(int worker_id, void *ctx_ptr) {
27692816 if (s >= dc -> nsec ) {
27702817 break ;
27712818 }
2772- doc_section_one (dc , s , stamp , cands , sidx , sw , body , fstamp , fc );
2819+ doc_section_one (dc , s , stamp , cands , sidx , sw , body , fstamp , fc , secw , secmark );
27732820 }
27742821 }
27752822 cbm_free (CBM_MEM_CLASS_SEMANTIC , stamp );
@@ -2779,6 +2826,8 @@ static void doc_section_worker(int worker_id, void *ctx_ptr) {
27792826 cbm_free (CBM_MEM_CLASS_SEMANTIC , sidx );
27802827 cbm_free (CBM_MEM_CLASS_SEMANTIC , sw );
27812828 cbm_free (CBM_MEM_CLASS_SEMANTIC , body );
2829+ cbm_free (CBM_MEM_CLASS_SEMANTIC , secw );
2830+ cbm_free (CBM_MEM_CLASS_SEMANTIC , secmark );
27822831}
27832832
27842833static const char * const DOC_KIND_NAME [CBM_SEM_DOC_KIND_COUNT ] = {"function" , "local" , "file" ,
@@ -3047,6 +3096,7 @@ static void phase6c_doc_sections(cbm_pipeline_ctx_t *ctx, const cbm_sem_func_t *
30473096 .home = (const char * const * )place .home ,
30483097 .files = & files ,
30493098 .file_key_df = file_key_df ,
3099+ .nterms = nterms ,
30503100 .res = res };
30513101 atomic_init (& dc .next , 0 );
30523102 cbm_parallel_for_opts_t opts = {.max_workers = worker_count , .force_pthreads = false};
0 commit comments