Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -184,17 +184,42 @@ public void deleteKnowledge(Long id) {
}

private void doDelVector(BusinessKnowledge knowledge) {
agentVectorStoreService.deleteDocumentsByMetadata(knowledge.getAgentId().toString(), vectorMetadata(knowledge));
Map<String, Object> metadata = vectorMetadata(knowledge);
boolean currentMetadataDeleted = deleteDocumentsByMetadata(knowledge, metadata);

// Compatibility cleanup for documents embedded before businessTermId was
// normalized to String metadata.
Map<String, Object> legacyNumericMetadata = new HashMap<>(metadata);
legacyNumericMetadata.put(DocumentMetadataConstant.DB_BUSINESS_TERM_ID, knowledge.getId());
boolean legacyNumericMetadataDeleted = deleteDocumentsByMetadata(knowledge, legacyNumericMetadata);

if (!currentMetadataDeleted && !legacyNumericMetadataDeleted) {
throw new IllegalStateException(
"Failed to delete business knowledge from vector store: " + knowledge.getId());
}
}

private Map<String, Object> vectorMetadata(BusinessKnowledge knowledge) {
Map<String, Object> metadata = new HashMap<>();
metadata.put(Constant.AGENT_ID, knowledge.getAgentId().toString());
metadata.put(DocumentMetadataConstant.DB_BUSINESS_TERM_ID, knowledge.getId());
metadata.put(DocumentMetadataConstant.DB_BUSINESS_TERM_ID, knowledge.getId().toString());
metadata.put(DocumentMetadataConstant.VECTOR_TYPE, DocumentMetadataConstant.BUSINESS_TERM);
return metadata;
}

private boolean deleteDocumentsByMetadata(BusinessKnowledge knowledge, Map<String, Object> metadata) {
try {
return Boolean.TRUE
.equals(agentVectorStoreService.deleteDocumentsByMetadata(knowledge.getAgentId().toString(), metadata));
}
catch (RuntimeException ex) {
Object businessTermId = metadata.get(DocumentMetadataConstant.DB_BUSINESS_TERM_ID);
log.debug("Vector store rejected businessTermId metadata type {}",
businessTermId.getClass().getSimpleName(), ex);
return false;
}
}

@Override
@Transactional(rollbackFor = Exception.class)
public void recallKnowledge(Long id, Boolean isRecall) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,10 +73,17 @@ public Filter.Expression buildDynamicFilter(String agentId, String vectorType) {
return null;
}
else {
// 添加 ID 过滤
String[] stringIds = recalledBusinessKnowledgeIds.stream()
.map(String::valueOf)
.toArray(String[]::new);
Filter.Expression stringIdFilter = b
.in(DocumentMetadataConstant.DB_BUSINESS_TERM_ID, (Object[]) stringIds)
.build();
Filter.Expression legacyNumericIdFilter = b
.in(DocumentMetadataConstant.DB_BUSINESS_TERM_ID, recalledBusinessKnowledgeIds.toArray())
.build();
conditions
.add(b.in(DocumentMetadataConstant.DB_BUSINESS_TERM_ID, recalledBusinessKnowledgeIds.toArray())
.build());
.add(new Filter.Expression(Filter.ExpressionType.OR, stringIdFilter, legacyNumericIdFilter));
}
break;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ public static Document convertBusinessKnowledgeToDocument(BusinessKnowledge busi
Map<String, Object> metadata = new HashMap<>();
metadata.put(DocumentMetadataConstant.VECTOR_TYPE, DocumentMetadataConstant.BUSINESS_TERM);
metadata.put(Constant.AGENT_ID, businessKnowledge.getAgentId().toString());
metadata.put(DocumentMetadataConstant.DB_BUSINESS_TERM_ID, businessKnowledge.getId());
metadata.put(DocumentMetadataConstant.DB_BUSINESS_TERM_ID, businessKnowledge.getId().toString());

return new Document(content, metadata);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package com.alibaba.cloud.ai.dataagent.service.business;

import com.alibaba.cloud.ai.dataagent.constant.DocumentMetadataConstant;
import com.alibaba.cloud.ai.dataagent.converter.BusinessKnowledgeConverter;
import com.alibaba.cloud.ai.dataagent.dto.knowledge.businessknowledge.CreateBusinessKnowledgeDTO;
import com.alibaba.cloud.ai.dataagent.dto.knowledge.businessknowledge.UpdateBusinessKnowledgeDTO;
Expand Down Expand Up @@ -229,7 +230,10 @@ void deleteKnowledge_found() {
when(businessKnowledgeMapper.logicalDelete(1L, 1)).thenReturn(1);

service.deleteKnowledge(1L);
verify(agentVectorStoreService).deleteDocumentsByMetadata(anyString(), anyMap());
verify(agentVectorStoreService).deleteDocumentsByMetadata(anyString(),
argThat(metadata -> "1".equals(metadata.get(DocumentMetadataConstant.DB_BUSINESS_TERM_ID))));
verify(agentVectorStoreService).deleteDocumentsByMetadata(anyString(),
argThat(metadata -> Long.valueOf(1L).equals(metadata.get(DocumentMetadataConstant.DB_BUSINESS_TERM_ID))));
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
import org.springframework.ai.document.Document;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.vectorstore.SearchRequest;
import org.springframework.ai.vectorstore.filter.Filter;

import java.util.ArrayList;
Expand All @@ -33,8 +36,10 @@
import java.util.Map;

import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;

@ExtendWith(MockitoExtension.class)
Expand Down Expand Up @@ -84,6 +89,25 @@ void buildDynamicFilter_businessTerm_withValidIds_returnsFilterExpression() {
assertNotNull(result);
}

@Test
void buildDynamicFilter_businessTerm_matchesStringMetadataInSimpleVectorStore() {
when(businessKnowledgeMapper.selectRecalledKnowledgeIds(anyLong())).thenReturn(List.of(10L));
EmbeddingModel embeddingModel = mock(EmbeddingModel.class);
when(embeddingModel.dimensions()).thenReturn(2);
when(embeddingModel.embed(any(Document.class))).thenReturn(new float[] { 1.0f, 0.0f });
when(embeddingModel.embed(any(String.class))).thenReturn(new float[] { 1.0f, 0.0f });
MetadataAwareSimpleVectorStore store = new MetadataAwareSimpleVectorStore(embeddingModel);
store.add(List.of(new Document("term", Map.of("agentId", "1", DocumentMetadataConstant.VECTOR_TYPE,
DocumentMetadataConstant.BUSINESS_TERM, DocumentMetadataConstant.DB_BUSINESS_TERM_ID, "10"))));

Filter.Expression filter = dynamicFilterService.buildDynamicFilter("1",
DocumentMetadataConstant.BUSINESS_TERM);
List<Document> result = store.similaritySearch(
SearchRequest.builder().query("default").filterExpression(filter).topK(10).similarityThreshold(0).build());

assertEquals(1, result.size());
}

@Test
void buildDynamicFilter_businessTerm_noValidIds_returnsNull() {
when(businessKnowledgeMapper.selectRecalledKnowledgeIds(anyLong())).thenReturn(new ArrayList<>());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ void testConvertBusinessKnowledgeToDocument() {
assertTrue(doc.getText().contains("Gross Merchandise Value"));
assertEquals(DocumentMetadataConstant.BUSINESS_TERM,
doc.getMetadata().get(DocumentMetadataConstant.VECTOR_TYPE));
assertEquals("1", doc.getMetadata().get(DocumentMetadataConstant.DB_BUSINESS_TERM_ID));
}

@Test
Expand Down
Loading