build(rabitq): upgrade to RaBitQ 0.7 modular targets (#814)

Author: gouyt13

.gitattributes

@@ -1,6 +1,3 @@
 # Auto-generated files — collapsed in GitHub PR diffs
 src/db/index/column/fts_column/gen/** linguist-generated=true
 src/db/sqlengine/antlr/gen/** linguist-generated=true
-
-# Unified diff context lines use a leading space by definition.
-thirdparty/RaBitQ-Library/*.patch whitespace=-blank-at-eol

.gitmodules

@@ -36,7 +36,7 @@
 	url = https://github.com/Neargye/magic_enum.git
 	ignore = dirty
 [submodule "thirdparty/RaBitQ-Library/RaBitQ-Library-0.1"]
-	path = thirdparty/RaBitQ-Library/RaBitQ-Library-0.3.8
+	path = thirdparty/RaBitQ-Library/RaBitQ-Library-0.7.0
 	url = https://github.com/VectorDB-NTU/RaBitQ-Library.git
 	ignore = dirty
 [submodule "thirdparty/aio/libaio-0.3"]

CMakeLists.txt

@@ -160,14 +160,11 @@ if(NOT ANDROID AND NOT IOS AND CMAKE_SIZEOF_VOID_P EQUAL 8
     set(RABITQ_AVX2_FLAGS "/arch:AVX2")
     set(RABITQ_AVX512_FLAGS "/arch:AVX512")
     set(RABITQ_AVX512_POPCNT_FLAGS "/arch:AVX512")
-    set(RABITQ_HNSW_AVX512_CORE_FLAGS "/arch:AVX512")
   else()
     set(RABITQ_AVX2_FLAGS "-mavx2;-mfma")
     set(RABITQ_AVX512_FLAGS "-mavx512f;-mavx512bw;-mavx512dq;-mfma")
     set(RABITQ_AVX512_POPCNT_FLAGS
         "-mavx512f;-mavx512bw;-mavx512dq;-mavx512vpopcntdq;-mfma")
-    set(RABITQ_HNSW_AVX512_CORE_FLAGS
-        "-mavx512f;-mavx512bw;-mavx512dq;-mavx2;-mfma")
   endif()
 
   # check_cxx_compiler_flag takes one argument, so the semicolon-separated
@@ -186,11 +183,6 @@ if(NOT ANDROID AND NOT IOS AND CMAKE_SIZEOF_VOID_P EQUAL 8
 
   if(RABITQ_SUPPORTED)
     add_definitions(-DRABITQ_SUPPORTED=1)
-    if(MSVC)
-      # rabitqlib's AVX2 rotator calls FHT kernels that only exist as MASM on
-      # MSVC; enable_language() must run in the top-level directory.
-      enable_language(ASM_MASM)
-    endif()
   else()
     add_definitions(-DRABITQ_SUPPORTED=0)
     message(STATUS "RaBitQ support disabled - compiler cannot build all runtime dispatch variants")

src/core/algorithm/hnsw_rabitq/hnsw_rabitq_entity.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 #include "hnsw_rabitq_entity.h"
-#include <rabitqlib/index/query.hpp>
+#include <rabitqlib/core/data_layout.hpp>
 #include "utility/sparse_utility.h"
 #include "zvec/core/framework/index_stats.h"
 

src/core/algorithm/hnsw_rabitq/hnsw_rabitq_query_algorithm.cc

@@ -15,7 +15,7 @@
 #include <chrono>
 #include <iostream>
 #include <ailego/internal/cpu_features.h>
-#include <rabitqlib/index/estimator.hpp>
+#include <rabitqlib/core/estimator.hpp>
 #include "zvec/ailego/internal/platform.h"
 #include "hnsw_rabitq_entity.h"
 #include "hnsw_rabitq_query_entity.h"
@@ -46,7 +46,7 @@ int HnswRabitqQueryAlgorithm::cleanup() {
 int HnswRabitqQueryAlgorithm::search(HnswRabitqQueryEntity *entity,
                                      HnswRabitqContext *ctx) const {
   spin_lock_.lock();
-  auto maxLevel = entity_.cur_max_level();
+  auto max_level = entity_.cur_max_level();
   auto entry_point = entity_.entry_point();
   spin_lock_.unlock();
 
@@ -57,7 +57,7 @@ int HnswRabitqQueryAlgorithm::search(HnswRabitqQueryEntity *entity,
   EstimateRecord curest;
   get_bin_est(entity_.get_vector(entry_point), curest, *entity);
 
-  for (level_t cur_level = maxLevel; cur_level >= 1; --cur_level) {
+  for (level_t cur_level = max_level; cur_level >= 1; --cur_level) {
     select_entry_point(cur_level, &entry_point, &curest, ctx, entity);
   }
 
@@ -170,10 +170,10 @@ void HnswRabitqQueryAlgorithm::search_neighbors(
 
       if (ex_bits_ > 0) {
         // Check preliminary score against current worst full estimate.
-        bool flag_update_KNNs =
+        bool update_knns =
             (!topk.full()) || candest.low_dist < topk[0].second.est_dist;
 
-        if (flag_update_KNNs) {
+        if (update_knns) {
           // Compute the full estimate if promising.
           get_full_est(cand_vector, candest, *query_entity);
         } else {

src/core/algorithm/hnsw_rabitq/hnsw_rabitq_query_entity.h

@@ -16,7 +16,7 @@
 
 #include <memory>
 #include <vector>
-#include <rabitqlib/index/query.hpp>
+#include <rabitqlib/core/query.hpp>
 namespace zvec::core {
 
 struct HnswRabitqQueryEntity {

src/core/algorithm/hnsw_rabitq/rabitq_converter.cc

@@ -15,7 +15,7 @@
 #include "rabitq_converter.h"
 #include <cstring>
 #include <memory>
-#include <rabitqlib/utils/rotator.hpp>
+#include <rabitqlib/core/rotator.hpp>
 #include <zvec/ailego/container/params.h>
 #include <zvec/ailego/parallel/thread_pool.h>
 #include <zvec/ailego/utility/string_helper.h>

src/core/algorithm/hnsw_rabitq/rabitq_converter.h

@@ -15,7 +15,7 @@
 
 #include <memory>
 #include <vector>
-#include <rabitqlib/utils/rotator.hpp>
+#include <rabitqlib/core/rotator.hpp>
 #include "zvec/core/framework/index_cluster.h"
 #include "zvec/core/framework/index_converter.h"
 #include "zvec/core/framework/index_reformer.h"

src/core/algorithm/hnsw_rabitq/rabitq_reformer.cc

@@ -15,10 +15,10 @@
 #include "rabitq_reformer.h"
 #include <string>
 #include <vector>
-#include <rabitqlib/defines.hpp>
-#include <rabitqlib/index/query.hpp>
-#include <rabitqlib/quantization/rabitq.hpp>
-#include <rabitqlib/utils/rotator.hpp>
+#include <rabitqlib/core/encoder.hpp>
+#include <rabitqlib/core/metric.hpp>
+#include <rabitqlib/core/query.hpp>
+#include <rabitqlib/core/rotator.hpp>
 #include <zvec/ailego/logger/logger.h>
 #include <zvec/ailego/utility/string_helper.h>
 #include "core/algorithm/cluster/linear_seeker.h"

src/core/algorithm/hnsw_rabitq/rabitq_utils.h

@@ -15,7 +15,7 @@
 
 #include <memory>
 #include <vector>
-#include <rabitqlib/utils/rotator.hpp>
+#include <rabitqlib/core/rotator.hpp>
 #include "zvec/core/framework/index_dumper.h"
 
 namespace zvec {