szz
/
ShiShiYiBan
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

【功能新增】AI：知识库文档的模型变化时，触发重索引（异步）

This commit is contained in:
YunaiV 2025-03-10 09:12:24 +08:00
parent 32e1ef4da8
commit d7e801c438
5 changed files with 60 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java
View File

@ -42,6 +42,11 @@ public interface AiKnowledgeSegmentMapper extends BaseMapperX<AiKnowledgeSegment
.orderByDesc(AiKnowledgeSegmentDO::getId));
}
default List<AiKnowledgeSegmentDO> selectListByKnowledgeIdAndStatus(Long knowledgeId, Integer status) {
return selectList(AiKnowledgeSegmentDO::getKnowledgeId, knowledgeId,
AiKnowledgeSegmentDO::getStatus, status);
}
default List<AiKnowledgeSegmentProcessRespVO> selectProcessList(Collection<Long> documentIds) {
MPJLambdaWrapper<AiKnowledgeSegmentDO> wrapper = new MPJLambdaWrapperX<AiKnowledgeSegmentDO>()
.selectAs(AiKnowledgeSegmentDO::getDocumentId, AiKnowledgeSegmentProcessRespVO::getDocumentId)
@ -54,7 +59,7 @@ public interface AiKnowledgeSegmentMapper extends BaseMapperX<AiKnowledgeSegment
}
default void updateRetrievalCountIncrByIds(List<Long> ids) {
update( new LambdaUpdateWrapper<AiKnowledgeSegmentDO>()
update(new LambdaUpdateWrapper<AiKnowledgeSegmentDO>()
.setSql(" retrieval_count = retrieval_count + 1")
.in(AiKnowledgeSegmentDO::getId, ids));
}

17
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java
View File

@ -98,6 +98,23 @@ public interface AiKnowledgeSegmentService {
*/
void updateKnowledgeSegmentStatus(AiKnowledgeSegmentUpdateStatusReqVO reqVO);
/**
* 重新索引知识库下的所有文档段落
*
* @param knowledgeId 知识库编号
*/
void reindexKnowledgeSegmentByKnowledgeId(Long knowledgeId);
/**
* 【异步】重新索引知识库下的所有文档段落
*
* @param knowledgeId 知识库编号
*/
@Async
default void reindexByKnowledgeIdAsync(Long knowledgeId) {
reindexKnowledgeSegmentByKnowledgeId(knowledgeId);
}
/**
* 根据文档编号删除段落
*

24
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java
View File

@ -158,6 +158,30 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
}
}
@Override
public void reindexKnowledgeSegmentByKnowledgeId(Long knowledgeId) {
// 1.1 校验知识库存在
AiKnowledgeDO knowledge = knowledgeService.validateKnowledgeExists(knowledgeId);
// 1.2 获取知识库向量实例
VectorStore vectorStore = getVectorStoreById(knowledge);
// 2.1 查询知识库下的所有启用状态的段落
List<AiKnowledgeSegmentDO> segments = segmentMapper.selectListByKnowledgeIdAndStatus(
knowledgeId, CommonStatusEnum.ENABLE.getStatus());
if (CollUtil.isEmpty(segments)) {
return;
}
// 2.2 遍历所有段落，重新索引
for (AiKnowledgeSegmentDO segment : segments) {
// 删除旧的向量
deleteVectorStore(vectorStore, segment);
// 重新创建向量
writeVectorStore(vectorStore, segment, new Document(segment.getContent()));
}
log.info("[reindexKnowledgeSegmentByKnowledgeId][知识库({}) 重新索引完成，共处理 {} 个段落]",
knowledgeId, segments.size());
}
private void writeVectorStore(VectorStore vectorStore, AiKnowledgeSegmentDO segmentDO, Document segment) {
// 1. 向量存储
// 为什么要 toString 呢？因为部分 VectorStore 实现，不支持 Long 类型，例如说 QdrantVectorStore

12
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeServiceImpl.java
View File

@ -1,10 +1,13 @@
package cn.iocoder.yudao.module.ai.service.knowledge;
import cn.hutool.core.util.ObjUtil;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgePageReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.knowledge.AiKnowledgeSaveReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO;
import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiModelDO;
import cn.iocoder.yudao.module.ai.dal.mysql.knowledge.AiKnowledgeMapper;
import cn.iocoder.yudao.module.ai.service.model.AiModelService;
@ -31,6 +34,8 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService {
@Resource
private AiModelService modelService;
@Resource
private AiKnowledgeSegmentService knowledgeSegmentService;
@Override
public Long createKnowledge(AiKnowledgeSaveReqVO createReqVO) {
@ -47,7 +52,7 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService {
@Override
public void updateKnowledge(AiKnowledgeSaveReqVO updateReqVO) {
// 1.1 校验知识库存在
validateKnowledgeExists(updateReqVO.getId());
AiKnowledgeDO oldKnowledge = validateKnowledgeExists(updateReqVO.getId());
// 1.2 校验模型配置
AiModelDO model = modelService.validateModel(updateReqVO.getEmbeddingModelId());
@ -56,7 +61,10 @@ public class AiKnowledgeServiceImpl implements AiKnowledgeService {
.setEmbeddingModel(model.getModel());
knowledgeMapper.updateById(updateObj);
// TODO @芋艿：如果模型变化，需要 reindex 所有的文档
// 3. 如果模型变化，需要 reindex 所有的文档
if (ObjUtil.notEqual(oldKnowledge.getEmbeddingModelId(), updateReqVO.getEmbeddingModelId())) {
knowledgeSegmentService.reindexByKnowledgeIdAsync(updateReqVO.getId());
}
}
@Override

6
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/model/AiModelServiceImpl.java
View File

@ -16,8 +16,8 @@ import jakarta.annotation.Resource;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.embedding.EmbeddingModel;
import org.springframework.ai.image.ImageModel;
import org.springframework.ai.vectorstore.SimpleVectorStore;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.vectorstore.milvus.MilvusVectorStore;
import org.springframework.stereotype.Service;
import org.springframework.validation.annotation.Validated;
@ -162,10 +162,10 @@ public class AiModelServiceImpl implements AiModelService {
platform, apiKey.getApiKey(), apiKey.getUrl(), model.getModel());
// 创建或获取 VectorStore 对象
// return modelFactory.getOrCreateVectorStore(SimpleVectorStore.class, embeddingModel, metadataFields);
return modelFactory.getOrCreateVectorStore(SimpleVectorStore.class, embeddingModel, metadataFields);
// return modelFactory.getOrCreateVectorStore(QdrantVectorStore.class, embeddingModel, metadataFields);
// return modelFactory.getOrCreateVectorStore(RedisVectorStore.class, embeddingModel, metadataFields);
return modelFactory.getOrCreateVectorStore(MilvusVectorStore.class, embeddingModel, metadataFields);
// return modelFactory.getOrCreateVectorStore(MilvusVectorStore.class, embeddingModel, metadataFields);
}
}