From ebd93514b3cbb722dbb0a4f5b7c7490ffce0b417 Mon Sep 17 00:00:00 2001 From: YunaiV Date: Sat, 1 Mar 2025 13:35:00 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90=E5=8A=9F=E8=83=BD=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E3=80=91AI=EF=BC=9A=E6=96=B0=E5=A2=9E=20document=20=E5=90=91?= =?UTF-8?q?=E9=87=8F=E7=9A=84=E8=BF=9B=E5=BA=A6=E6=9F=A5=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AiKnowledgeSegmentController.http | 9 ++++++++- .../AiKnowledgeSegmentController.java | 10 ++++++++++ .../AiKnowledgeSegmentProcessRespVO.java | 19 +++++++++++++++++++ .../AiKnowledgeSegmentUpdateReqVO.java | 2 +- .../knowledge/AiKnowledgeSegmentMapper.java | 15 +++++++++++++++ .../AiKnowledgeDocumentServiceImpl.java | 4 ++-- .../knowledge/AiKnowledgeSegmentService.java | 9 +++++++++ .../AiKnowledgeSegmentServiceImpl.java | 13 +++++++++---- 8 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentProcessRespVO.java diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeSegmentController.http b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeSegmentController.http index 25d622d4c1..eaeee5704f 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeSegmentController.http +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeSegmentController.http @@ -2,4 +2,11 @@ GET {{baseUrl}}/ai/knowledge/segment/split?url=https://static.iocoder.cn/README_yudao.md&segmentMaxTokens=800 Content-Type: application/json Authorization: Bearer {{token}} -tenant-id: {{adminTenantId}} \ No newline at end of file +tenant-id: {{adminTenantId}} + +### 获取文档处理列表 +GET {{baseUrl}}/ai/knowledge/segment/get-process-list?documentIds=1,2,3 +Content-Type: application/json +Authorization: Bearer {{token}} +tenant-id: {{adminTenantId}} + diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeSegmentController.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeSegmentController.java index 2b21e0d1e4..d683c32d6a 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeSegmentController.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/AiKnowledgeSegmentController.java @@ -7,6 +7,7 @@ import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowle import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentRespVO; import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO; import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentProcessRespVO; import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; import cn.iocoder.yudao.module.ai.service.knowledge.AiKnowledgeSegmentService; import io.swagger.v3.oas.annotations.Operation; @@ -69,4 +70,13 @@ public class AiKnowledgeSegmentController { return success(BeanUtils.toBean(segments, AiKnowledgeSegmentRespVO.class)); } + @GetMapping("/get-process-list") + @Operation(summary = "获取文档处理列表") + @Parameter(name = "documentIds", description = "文档编号列表", required = true, example = "1,2,3") + public CommonResult> getKnowledgeSegmentProcessList( + @RequestParam("documentIds") List documentIds) { + List list = segmentService.getKnowledgeSegmentProcessList(documentIds); + return success(list); + } + } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentProcessRespVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentProcessRespVO.java new file mode 100644 index 0000000000..a6b95265b7 --- /dev/null +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentProcessRespVO.java @@ -0,0 +1,19 @@ +package cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Schema(description = "管理后台 - AI 知识库段落向量进度 Response VO") +@Data +public class AiKnowledgeSegmentProcessRespVO { + + @Schema(description = "文档编号", requiredMode = Schema.RequiredMode.REQUIRED, example = "1") + private Long documentId; + + @Schema(description = "总段落数量", requiredMode = Schema.RequiredMode.REQUIRED, example = "10") + private Long count; + + @Schema(description = "已向量化段落数量", requiredMode = Schema.RequiredMode.REQUIRED, example = "5") + private Long embeddingCount; + +} \ No newline at end of file diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentUpdateReqVO.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentUpdateReqVO.java index 23b1461e2d..2fc7a3141f 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentUpdateReqVO.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/knowledge/vo/segment/AiKnowledgeSegmentUpdateReqVO.java @@ -4,7 +4,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; -@Schema(description = "管理后台 - AI 更新 知识库-段落 request VO") +@Schema(description = "管理后台 - AI 更新 知识库段落 request VO") @Data public class AiKnowledgeSegmentUpdateReqVO { diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java index 8381c84300..6f731811fe 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/knowledge/AiKnowledgeSegmentMapper.java @@ -3,11 +3,15 @@ package cn.iocoder.yudao.module.ai.dal.mysql.knowledge; import cn.iocoder.yudao.framework.common.pojo.PageResult; import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX; import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX; +import cn.iocoder.yudao.framework.mybatis.core.query.MPJLambdaWrapperX; import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentProcessRespVO; import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; +import com.github.yulichang.wrapper.MPJLambdaWrapper; import org.apache.ibatis.annotations.Mapper; import java.util.List; +import java.util.Collection; /** * AI 知识库分片 Mapper @@ -37,4 +41,15 @@ public interface AiKnowledgeSegmentMapper extends BaseMapperX selectProcessList(Collection documentIds) { + MPJLambdaWrapper wrapper = new MPJLambdaWrapperX() + .selectAs(AiKnowledgeSegmentDO::getDocumentId, AiKnowledgeSegmentProcessRespVO::getDocumentId) + .selectCount(AiKnowledgeSegmentDO::getId, "count") + .select("COUNT(CASE WHEN vector_id > '" + AiKnowledgeSegmentDO.VECTOR_ID_EMPTY + + "' THEN 1 ELSE NULL END) AS embeddingCount") + .in(AiKnowledgeSegmentDO::getDocumentId, documentIds) + .groupBy(AiKnowledgeSegmentDO::getDocumentId); + return selectJoinList(AiKnowledgeSegmentProcessRespVO.class, wrapper); + } + } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java index 4cff8324c6..466b440cb7 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java @@ -67,8 +67,8 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic .setStatus(CommonStatusEnum.ENABLE.getStatus()); knowledgeDocumentMapper.insert(documentDO); - // 4. 文档切片入库（同步） - knowledgeSegmentService.createKnowledgeSegmentBySplitContent(documentDO.getId(), content); + // 4. 文档切片入库（异步） + knowledgeSegmentService.createKnowledgeSegmentBySplitContentAsync(documentDO.getId(), content); return documentDO.getId(); } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java index 137887af77..a1a0b2d455 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentService.java @@ -5,6 +5,7 @@ import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowle import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentSearchReqVO; import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO; import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentProcessRespVO; import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; import org.springframework.scheduling.annotation.Async; @@ -82,4 +83,12 @@ public interface AiKnowledgeSegmentService { */ List splitContent(String url, Integer segmentMaxTokens); + /** + * 获取文档处理进度（多个） + * + * @param documentIds 文档编号列表 + * @return 文档处理列表 + */ + List getKnowledgeSegmentProcessList(List documentIds); + } diff --git a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java index a20a18e8c3..0dce49f2be 100644 --- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java +++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java @@ -6,10 +6,7 @@ import cn.hutool.core.util.StrUtil; import cn.iocoder.yudao.framework.common.enums.CommonStatusEnum; import cn.iocoder.yudao.framework.common.pojo.PageResult; import cn.iocoder.yudao.framework.common.util.object.BeanUtils; -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentPageReqVO; -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentSearchReqVO; -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateReqVO; -import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.AiKnowledgeSegmentUpdateStatusReqVO; +import cn.iocoder.yudao.module.ai.controller.admin.knowledge.vo.segment.*; import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDO; import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeDocumentDO; import cn.iocoder.yudao.module.ai.dal.dataobject.knowledge.AiKnowledgeSegmentDO; @@ -252,4 +249,12 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService .build(); } + @Override + public List getKnowledgeSegmentProcessList(List documentIds) { + if (CollUtil.isEmpty(documentIds)) { + return Collections.emptyList(); + } + return segmentMapper.selectProcessList(documentIds); + } + }