【功能新增】AI:新增知识库分段的新增

This commit is contained in:
YunaiV 2025-03-02 21:52:31 +08:00
parent 5f5e77a392
commit 3f460dc620
11 changed files with 136 additions and 41 deletions

View File

@ -60,5 +60,6 @@ public interface ErrorCodeConstants {
ErrorCode KNOWLEDGE_DOCUMENT_FILE_READ_FAIL = new ErrorCode(1_022_008_102, "文档加载失败!");
ErrorCode KNOWLEDGE_SEGMENT_NOT_EXISTS = new ErrorCode(1_022_008_202, "段落不存在!");
ErrorCode KNOWLEDGE_SEGMENT_CONTENT_TOO_LONG = new ErrorCode(1_022_008_203, "内容 Token 数为 {},超过最大限制 {}");
}

View File

@ -13,6 +13,7 @@ import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.annotation.Resource;
import jakarta.validation.Valid;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
@ -30,6 +31,7 @@ public class AiKnowledgeController {
@GetMapping("/page")
@Operation(summary = "获取知识库分页")
@PreAuthorize("@ss.hasPermission('ai:knowledge:query')")
public CommonResult<PageResult<AiKnowledgeRespVO>> getKnowledgePage(@Valid AiKnowledgePageReqVO pageReqVO) {
PageResult<AiKnowledgeDO> pageResult = knowledgeService.getKnowledgePage(pageReqVO);
@ -39,6 +41,7 @@ public class AiKnowledgeController {
@GetMapping("/get")
@Operation(summary = "获得知识库")
@Parameter(name = "id", description = "编号", required = true, example = "1024")
@PreAuthorize("@ss.hasPermission('ai:knowledge:query')")
public CommonResult<AiKnowledgeRespVO> getKnowledge(@RequestParam("id") Long id) {
AiKnowledgeDO knowledge = knowledgeService.getKnowledge(id);
return success(BeanUtils.toBean(knowledge, AiKnowledgeRespVO.class));
@ -46,12 +49,14 @@ public class AiKnowledgeController {
@PostMapping("/create")
@Operation(summary = "创建知识库")
@PreAuthorize("@ss.hasPermission('ai:knowledge:create')")
public CommonResult<Long> createKnowledge(@RequestBody @Valid AiKnowledgeSaveReqVO createReqVO) {
return success(knowledgeService.createKnowledge(createReqVO));
}
@PutMapping("/update")
@Operation(summary = "更新知识库")
@PreAuthorize("@ss.hasPermission('ai:knowledge:update')")
public CommonResult<Boolean> updateKnowledge(@RequestBody @Valid AiKnowledgeSaveReqVO updateReqVO) {
knowledgeService.updateKnowledge(updateReqVO);
return success(true);

View File

@ -15,6 +15,7 @@ import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.annotation.Resource;
import jakarta.validation.Valid;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
@ -22,7 +23,6 @@ import java.util.List;
import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success;
// TODO @芋艿增加权限标识
@Tag(name = "管理后台 - AI 知识库文档")
@RestController
@RequestMapping("/ai/knowledge/document")
@ -34,6 +34,7 @@ public class AiKnowledgeDocumentController {
@GetMapping("/page")
@Operation(summary = "获取文档分页")
@PreAuthorize("@ss.hasPermission('ai:knowledge:query')")
public CommonResult<PageResult<AiKnowledgeDocumentRespVO>> getKnowledgeDocumentPage(
@Valid AiKnowledgeDocumentPageReqVO pageReqVO) {
PageResult<AiKnowledgeDocumentDO> pageResult = documentService.getKnowledgeDocumentPage(pageReqVO);
@ -42,6 +43,7 @@ public class AiKnowledgeDocumentController {
@GetMapping("/get")
@Operation(summary = "获取文档详情")
@PreAuthorize("@ss.hasPermission('ai:knowledge:query')")
public CommonResult<AiKnowledgeDocumentRespVO> getKnowledgeDocument(@RequestParam("id") Long id) {
AiKnowledgeDocumentDO document = documentService.getKnowledgeDocument(id);
return success(BeanUtils.toBean(document, AiKnowledgeDocumentRespVO.class));
@ -49,6 +51,7 @@ public class AiKnowledgeDocumentController {
@PostMapping("/create")
@Operation(summary = "新建文档(单个)")
@PreAuthorize("@ss.hasPermission('ai:knowledge:create')")
public CommonResult<Long> createKnowledgeDocument(@RequestBody @Valid AiKnowledgeDocumentCreateReqVO reqVO) {
Long id = documentService.createKnowledgeDocument(reqVO);
return success(id);
@ -56,6 +59,7 @@ public class AiKnowledgeDocumentController {
@PostMapping("/create-list")
@Operation(summary = "新建文档(多个)")
@PreAuthorize("@ss.hasPermission('ai:knowledge:create')")
public CommonResult<List<Long>> createKnowledgeDocumentList(
@RequestBody @Valid AiKnowledgeDocumentCreateListReqVO reqVO) {
List<Long> ids = documentService.createKnowledgeDocumentList(reqVO);
@ -64,6 +68,7 @@ public class AiKnowledgeDocumentController {
@PutMapping("/update")
@Operation(summary = "更新文档")
@PreAuthorize("@ss.hasPermission('ai:knowledge:update')")
public CommonResult<Boolean> updateKnowledgeDocument(@Valid @RequestBody AiKnowledgeDocumentUpdateReqVO reqVO) {
documentService.updateKnowledgeDocument(reqVO);
return success(true);
@ -71,10 +76,19 @@ public class AiKnowledgeDocumentController {
@PutMapping("/update-status")
@Operation(summary = "更新文档状态")
@PreAuthorize("@ss.hasPermission('ai:knowledge:update')")
public CommonResult<Boolean> updateKnowledgeDocumentStatus(
@Valid @RequestBody AiKnowledgeDocumentUpdateStatusReqVO reqVO) {
documentService.updateKnowledgeDocumentStatus(reqVO);
return success(true);
}
@DeleteMapping("/delete")
@Operation(summary = "删除文档")
@PreAuthorize("@ss.hasPermission('ai:knowledge:delete')")
public CommonResult<Boolean> deleteKnowledgeDocument(@RequestParam("id") Long id) {
documentService.deleteKnowledgeDocument(id);
return success(true);
}
}

View File

@ -19,6 +19,7 @@ import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.annotation.Resource;
import jakarta.validation.Valid;
import org.hibernate.validator.constraints.URL;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
@ -29,7 +30,6 @@ import java.util.Map;
import static cn.iocoder.yudao.framework.common.pojo.CommonResult.success;
import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.convertSet;
// TODO @芋艿增加权限标识
@Tag(name = "管理后台 - AI 知识库段落")
@RestController
@RequestMapping("/ai/knowledge/segment")
@ -38,27 +38,45 @@ public class AiKnowledgeSegmentController {
@Resource
private AiKnowledgeSegmentService segmentService;
@Resource
private AiKnowledgeDocumentService documentService;
@GetMapping("/get")
@Operation(summary = "获取段落详情")
@Parameter(name = "id", description = "段落编号", required = true, example = "1024")
@PreAuthorize("@ss.hasPermission('ai:knowledge:query')")
public CommonResult<AiKnowledgeSegmentRespVO> getKnowledgeSegment(@RequestParam("id") Long id) {
AiKnowledgeSegmentDO segment = segmentService.getKnowledgeSegment(id);
return success(BeanUtils.toBean(segment, AiKnowledgeSegmentRespVO.class));
}
@GetMapping("/page")
@Operation(summary = "获取段落分页")
@PreAuthorize("@ss.hasPermission('ai:knowledge:query')")
public CommonResult<PageResult<AiKnowledgeSegmentRespVO>> getKnowledgeSegmentPage(
@Valid AiKnowledgeSegmentPageReqVO pageReqVO) {
PageResult<AiKnowledgeSegmentDO> pageResult = segmentService.getKnowledgeSegmentPage(pageReqVO);
return success(BeanUtils.toBean(pageResult, AiKnowledgeSegmentRespVO.class));
}
@PostMapping("/create")
@Operation(summary = "创建段落")
@PreAuthorize("@ss.hasPermission('ai:knowledge:create')")
public CommonResult<Long> createKnowledgeSegment(@Valid @RequestBody AiKnowledgeSegmentSaveReqVO createReqVO) {
return success(segmentService.createKnowledgeSegment(createReqVO));
}
@PutMapping("/update")
@Operation(summary = "更新段落内容")
public CommonResult<Boolean> updateKnowledgeSegment(@Valid @RequestBody AiKnowledgeSegmentUpdateReqVO reqVO) {
@PreAuthorize("@ss.hasPermission('ai:knowledge:update')")
public CommonResult<Boolean> updateKnowledgeSegment(@Valid @RequestBody AiKnowledgeSegmentSaveReqVO reqVO) {
segmentService.updateKnowledgeSegment(reqVO);
return success(true);
}
@PutMapping("/update-status")
@Operation(summary = "启禁用段落内容")
@PreAuthorize("@ss.hasPermission('ai:knowledge:update')")
public CommonResult<Boolean> updateKnowledgeSegmentStatus(
@Valid @RequestBody AiKnowledgeSegmentUpdateStatusReqVO reqVO) {
segmentService.updateKnowledgeSegmentStatus(reqVO);
@ -71,6 +89,7 @@ public class AiKnowledgeSegmentController {
@Parameter(name = "url", description = "文档 URL", required = true),
@Parameter(name = "segmentMaxTokens", description = "分段的最大 Token 数", required = true)
})
@PreAuthorize("@ss.hasPermission('ai:knowledge:query')")
public CommonResult<List<AiKnowledgeSegmentRespVO>> splitContent(
@RequestParam("url") @URL String url,
@RequestParam(value = "segmentMaxTokens") Integer segmentMaxTokens) {
@ -81,6 +100,7 @@ public class AiKnowledgeSegmentController {
@GetMapping("/get-process-list")
@Operation(summary = "获取文档处理列表")
@Parameter(name = "documentIds", description = "文档编号列表", required = true, example = "1,2,3")
@PreAuthorize("@ss.hasPermission('ai:knowledge:query')")
public CommonResult<List<AiKnowledgeSegmentProcessRespVO>> getKnowledgeSegmentProcessList(
@RequestParam("documentIds") List<Long> documentIds) {
List<AiKnowledgeSegmentProcessRespVO> list = segmentService.getKnowledgeSegmentProcessList(documentIds);
@ -89,6 +109,7 @@ public class AiKnowledgeSegmentController {
@GetMapping("/search")
@Operation(summary = "搜索段落内容")
@PreAuthorize("@ss.hasPermission('ai:knowledge:query')")
public CommonResult<List<AiKnowledgeSegmentSearchRespVO>> searchKnowledgeSegment(
@Valid AiKnowledgeSegmentSearchReqVO reqVO) {
// 1. 搜索段落

View File

@ -1,6 +1,8 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment;
import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import cn.iocoder.yudao.framework.common.validation.InEnum;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@ -8,13 +10,14 @@ import lombok.Data;
@Data
public class AiKnowledgeSegmentPageReqVO extends PageParam {
@Schema(description = "分段状态", example = "1")
private Integer status;
@Schema(description = "文档编号", example = "1")
private Integer documentId;
@Schema(description = "分段内容关键字", example = "Java 开发")
private String keyword;
private String content;
@Schema(description = "分段状态", example = "1")
@InEnum(CommonStatusEnum.class)
private Integer status;
}

View File

@ -1,17 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@Schema(description = "管理后台 - AI 更新 知识库段落 request VO")
@Data
public class AiKnowledgeSegmentUpdateReqVO {
@Schema(description = "编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "24790")
private Long id;
@Schema(description = "切片内容", requiredMode = Schema.RequiredMode.REQUIRED, example = "Java 开发手册")
private String content;
}

View File

@ -25,8 +25,8 @@ public interface AiKnowledgeSegmentMapper extends BaseMapperX<AiKnowledgeSegment
default PageResult<AiKnowledgeSegmentDO> selectPage(AiKnowledgeSegmentPageReqVO reqVO) {
return selectPage(reqVO, new LambdaQueryWrapperX<AiKnowledgeSegmentDO>()
.eq(AiKnowledgeSegmentDO::getDocumentId, reqVO.getDocumentId())
.likeIfPresent(AiKnowledgeSegmentDO::getContent, reqVO.getContent())
.eqIfPresent(AiKnowledgeSegmentDO::getStatus, reqVO.getStatus())
.likeIfPresent(AiKnowledgeSegmentDO::getContent, reqVO.getKeyword())
.orderByDesc(AiKnowledgeSegmentDO::getId));
}

View File

@ -74,6 +74,13 @@ public interface AiKnowledgeDocumentService {
*/
void updateKnowledgeDocumentRetrievalCountIncr(Collection<Long> ids);
/**
* 删除文档
*
* @param id 文档编号
*/
void deleteKnowledgeDocument(Long id);
/**
* 校验文档是否存在
*

View File

@ -149,6 +149,19 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic
}
}
@Override
@Transactional(rollbackFor = Exception.class)
public void deleteKnowledgeDocument(Long id) {
// 1. 校验存在
validateKnowledgeDocumentExists(id);
// 2. 删除
knowledgeDocumentMapper.deleteById(id);
// 3. 删除对应的段落
knowledgeSegmentService.deleteKnowledgeSegmentByDocumentId(id);
}
@Override
public void updateKnowledgeDocumentRetrievalCountIncr(Collection<Long> ids) {
if (CollUtil.isEmpty(ids)) {

View File

@ -3,7 +3,7 @@ package cn.iocoder.yudao.module.ai.service.knowledge;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentProcessRespVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentSaveReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO;
import cn.iocoder.yudao.module.ai.service.knowledge.bo.AiKnowledgeSegmentSearchReqBO;
@ -19,6 +19,14 @@ import java.util.List;
*/
public interface AiKnowledgeSegmentService {
/**
* 获取知识库段落详情
*
* @param id 段落编号
* @return 段落详情
*/
AiKnowledgeSegmentDO getKnowledgeSegment(Long id);
/**
* 获取段落分页
*
@ -46,12 +54,20 @@ public interface AiKnowledgeSegmentService {
createKnowledgeSegmentBySplitContent(documentId, content);
}
/**
* 创建知识库段落
*
* @param createReqVO 创建信息
* @return 段落编号
*/
Long createKnowledgeSegment(AiKnowledgeSegmentSaveReqVO createReqVO);
/**
* 更新段落的内容
*
* @param reqVO 更新内容
*/
void updateKnowledgeSegment(AiKnowledgeSegmentUpdateReqVO reqVO);
void updateKnowledgeSegment(AiKnowledgeSegmentSaveReqVO reqVO);
/**
* 更新段落的状态

View File

@ -34,6 +34,7 @@ import java.util.Objects;
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.convertList;
import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_SEGMENT_NOT_EXISTS;
import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.KNOWLEDGE_SEGMENT_CONTENT_TOO_LONG;
/**
* AI 知识库分片 Service 实现类
@ -98,20 +99,20 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
}
@Override
public void updateKnowledgeSegment(AiKnowledgeSegmentUpdateReqVO reqVO) {
public void updateKnowledgeSegment(AiKnowledgeSegmentSaveReqVO reqVO) {
// 1. 校验
AiKnowledgeSegmentDO segment = validateKnowledgeSegmentExists(reqVO.getId());
AiKnowledgeSegmentDO oldSegment = validateKnowledgeSegmentExists(reqVO.getId());
// 2. 删除向量
VectorStore vectorStore = getVectorStoreById(segment.getKnowledgeId());
deleteVectorStore(vectorStore, segment);
VectorStore vectorStore = getVectorStoreById(oldSegment.getKnowledgeId());
deleteVectorStore(vectorStore, oldSegment);
// 3.1 更新切片
AiKnowledgeSegmentDO segmentDO = BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class);
segmentMapper.updateById(segmentDO);
AiKnowledgeSegmentDO newSegment = BeanUtils.toBean(reqVO, AiKnowledgeSegmentDO.class);
segmentMapper.updateById(newSegment);
// 3.2 重新向量化必须开启状态
if (CommonStatusEnum.isEnable(segmentDO.getStatus())) {
writeVectorStore(vectorStore, segmentDO, new Document(segmentDO.getContent()));
if (CommonStatusEnum.isEnable(oldSegment.getStatus())) {
writeVectorStore(vectorStore, newSegment, new Document(newSegment.getContent()));
}
}
@ -143,7 +144,7 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
segmentMapper.updateById(new AiKnowledgeSegmentDO().setId(reqVO.getId()).setStatus(reqVO.getStatus()));
// 4. 更新向量
if (Objects.equals(reqVO.getStatus(), CommonStatusEnum.ENABLE.getStatus())) {
if (CommonStatusEnum.isEnable(reqVO.getStatus())) {
writeVectorStore(vectorStore, segment, new Document(segment.getContent()));
} else {
deleteVectorStore(vectorStore, segment);
@ -183,7 +184,8 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
List<Document> documents = vectorStore.similaritySearch(SearchRequest.builder()
.query(reqBO.getContent())
.topK(ObjUtil.defaultIfNull(reqBO.getTopK(), knowledge.getTopK()))
.similarityThreshold(ObjUtil.defaultIfNull(reqBO.getSimilarityThreshold(), knowledge.getSimilarityThreshold()))
.similarityThreshold(
ObjUtil.defaultIfNull(reqBO.getSimilarityThreshold(), knowledge.getSimilarityThreshold()))
.filterExpression(new FilterExpressionBuilder()
.eq(VECTOR_STORE_METADATA_KNOWLEDGE_ID, reqBO.getKnowledgeId()).build())
.build());
@ -202,7 +204,7 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
// 4. 构建结果
List<AiKnowledgeSegmentSearchRespBO> result = convertList(segments, segment -> {
Document document = CollUtil.findOne(documents, // 找到对应的文档
Document document = CollUtil.findOne(documents, // 找到对应的文档
doc -> Objects.equals(doc.getId(), segment.getVectorId()));
if (document == null) {
return null;
@ -210,8 +212,7 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
return BeanUtils.toBean(segment, AiKnowledgeSegmentSearchRespBO.class)
.setScore(document.getScore());
});
result.sort((o1, o2)
-> Double.compare(o2.getScore(), o1.getScore())); // 按照分数降序排序
result.sort((o1, o2) -> Double.compare(o2.getScore(), o1.getScore())); // 按照分数降序排序
return result;
}
@ -280,4 +281,35 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
return segmentMapper.selectProcessList(documentIds);
}
@Override
public Long createKnowledgeSegment(AiKnowledgeSegmentSaveReqVO createReqVO) {
// 1.1 校验文档是否存在
AiKnowledgeDocumentDO document = knowledgeDocumentService
.validateKnowledgeDocumentExists(createReqVO.getDocumentId());
// 1.2 获取知识库信息
AiKnowledgeDO knowledge = knowledgeService.validateKnowledgeExists(document.getKnowledgeId());
// 1.3 校验 token 熟练
Integer tokens = tokenCountEstimator.estimate(createReqVO.getContent());
if (tokens > document.getSegmentMaxTokens()) {
throw exception(KNOWLEDGE_SEGMENT_CONTENT_TOO_LONG, tokens, document.getSegmentMaxTokens());
}
// 2. 保存段落
AiKnowledgeSegmentDO segment = BeanUtils.toBean(createReqVO, AiKnowledgeSegmentDO.class)
.setKnowledgeId(knowledge.getId()).setDocumentId(document.getId())
.setContentLength(createReqVO.getContent().length()).setTokens(tokens)
.setVectorId(AiKnowledgeSegmentDO.VECTOR_ID_EMPTY)
.setRetrievalCount(0).setStatus(CommonStatusEnum.ENABLE.getStatus());
segmentMapper.insert(segment);
// 3. 向量化
writeVectorStore(getVectorStoreById(knowledge), segment, new Document(segment.getContent()));
return segment.getId();
}
@Override
public AiKnowledgeSegmentDO getKnowledgeSegment(Long id) {
return validateKnowledgeSegmentExists(id);
}
}