【功能新增】AI：增加 QdrantVectorStore 向量库的接入

2025-03-06 22:22:22 +08:00 · 2025-03-06 22:22:22 +08:00 · 44bcc9476d
parent 6ccd0ca61e
commit 44bcc9476d
9 changed files with 78 additions and 12 deletions
--- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java
+++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeDocumentServiceImpl.java
@ -54,7 +54,6 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic
    private AiKnowledgeService knowledgeService;

    @Override
-    @Transactional(rollbackFor = Exception.class)
    public Long createKnowledgeDocument(AiKnowledgeDocumentCreateReqVO createReqVO) {
        // 1. 校验参数
        knowledgeService.validateKnowledgeExists(createReqVO.getKnowledgeId());
@ -74,7 +73,6 @@ public class AiKnowledgeDocumentServiceImpl implements AiKnowledgeDocumentServic
    }

    @Override
-    @Transactional(rollbackFor = Exception.class)
    public List<Long> createKnowledgeDocumentList(AiKnowledgeDocumentCreateListReqVO createListReqVO) {
        // 1. 校验参数
        knowledgeService.validateKnowledgeExists(createListReqVO.getKnowledgeId());
--- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java
+++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/knowledge/AiKnowledgeSegmentServiceImpl.java
@ -115,6 +115,7 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
        segmentMapper.updateById(newSegment);
        // 3.2 重新向量化，必须开启状态
        if (CommonStatusEnum.isEnable(oldSegment.getStatus())) {
+            newSegment.setKnowledgeId(oldSegment.getKnowledgeId()).setDocumentId(oldSegment.getDocumentId());
            writeVectorStore(vectorStore, newSegment, new Document(newSegment.getContent()));
        }
    }
@ -156,9 +157,10 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService

    private void writeVectorStore(VectorStore vectorStore, AiKnowledgeSegmentDO segmentDO, Document segment) {
        // 1. 向量存储
-        segment.getMetadata().put(VECTOR_STORE_METADATA_KNOWLEDGE_ID, segmentDO.getKnowledgeId());
-        segment.getMetadata().put(VECTOR_STORE_METADATA_DOCUMENT_ID, segmentDO.getDocumentId());
-        segment.getMetadata().put(VECTOR_STORE_METADATA_SEGMENT_ID, segmentDO.getId());
+        // 为什么要 toString 呢？因为部分 VectorStore 实现，不支持 Long 类型，例如说 QdrantVectorStore
+        segment.getMetadata().put(VECTOR_STORE_METADATA_KNOWLEDGE_ID, segmentDO.getKnowledgeId().toString());
+        segment.getMetadata().put(VECTOR_STORE_METADATA_DOCUMENT_ID, segmentDO.getDocumentId().toString());
+        segment.getMetadata().put(VECTOR_STORE_METADATA_SEGMENT_ID, segmentDO.getId().toString());
        vectorStore.add(List.of(segment));

        // 2. 更新向量 ID
@ -190,7 +192,8 @@ public class AiKnowledgeSegmentServiceImpl implements AiKnowledgeSegmentService
                .similarityThreshold(
                        ObjUtil.defaultIfNull(reqBO.getSimilarityThreshold(), knowledge.getSimilarityThreshold()))
                .filterExpression(new FilterExpressionBuilder()
-                        .eq(VECTOR_STORE_METADATA_KNOWLEDGE_ID, reqBO.getKnowledgeId()).build())
+                        .eq(VECTOR_STORE_METADATA_KNOWLEDGE_ID, reqBO.getKnowledgeId().toString())
+                        .build())
                .build());
        if (CollUtil.isEmpty(documents)) {
            return ListUtil.empty();
--- a/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/model/AiModelServiceImpl.java
+++ b/yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/model/AiModelServiceImpl.java
@ -16,8 +16,8 @@ import jakarta.annotation.Resource;
 import org.springframework.ai.chat.model.ChatModel;
 import org.springframework.ai.embedding.EmbeddingModel;
 import org.springframework.ai.image.ImageModel;
-import org.springframework.ai.vectorstore.SimpleVectorStore;
 import org.springframework.ai.vectorstore.VectorStore;
+import org.springframework.ai.vectorstore.qdrant.QdrantVectorStore;
 import org.springframework.stereotype.Service;
 import org.springframework.validation.annotation.Validated;

@ -162,7 +162,8 @@ public class AiModelServiceImpl implements AiModelService {
                platform, apiKey.getApiKey(), apiKey.getUrl(), model.getModel());

        // 创建或获取 VectorStore 对象
-        return modelFactory.getOrCreateVectorStore(SimpleVectorStore.class, embeddingModel);
+//        return modelFactory.getOrCreateVectorStore(SimpleVectorStore.class, embeddingModel);
+        return modelFactory.getOrCreateVectorStore(QdrantVectorStore.class, embeddingModel);
    }

 }
--- a/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml
+++ b/yudao-module-ai/yudao-spring-boot-starter-ai/pom.xml
@ -70,6 +70,7 @@
            <groupId>${spring-ai.groupId}</groupId>
            <artifactId>spring-ai-qdrant-store</artifactId>
            <version>${spring-ai.version}</version>
+<!--            <optional>true</optional>-->
        </dependency>

        <dependency>
--- a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java
+++ b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/config/YudaoAiAutoConfiguration.java
@ -11,6 +11,7 @@ import cn.iocoder.yudao.framework.ai.core.model.siliconflow.SiliconFlowChatModel
 import cn.iocoder.yudao.framework.ai.core.model.suno.api.SunoApi;
 import cn.iocoder.yudao.framework.ai.core.model.xinghuo.XingHuoChatModel;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.ai.autoconfigure.vectorstore.qdrant.QdrantVectorStoreProperties;
 import org.springframework.ai.openai.OpenAiChatModel;
 import org.springframework.ai.openai.OpenAiChatOptions;
 import org.springframework.ai.openai.api.OpenAiApi;
@ -29,7 +30,9 @@ import org.springframework.context.annotation.Lazy;
 * @author fansili
 */
@AutoConfiguration
-@EnableConfigurationProperties(YudaoAiProperties.class)
+@EnableConfigurationProperties({YudaoAiProperties.class,
+        QdrantVectorStoreProperties.class // 解析 Qdrant 配置
+})
@Slf4j
 public class YudaoAiAutoConfiguration {

--- a/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactoryImpl.java
+++ b/yudao-module-ai/yudao-spring-boot-starter-ai/src/main/java/cn/iocoder/yudao/framework/ai/core/factory/AiModelFactoryImpl.java
@ -5,6 +5,7 @@ import cn.hutool.core.lang.Assert;
 import cn.hutool.core.lang.Singleton;
 import cn.hutool.core.lang.func.Func0;
 import cn.hutool.core.util.ArrayUtil;
+import cn.hutool.core.util.ReflectUtil;
 import cn.hutool.core.util.RuntimeUtil;
 import cn.hutool.core.util.StrUtil;
 import cn.hutool.extra.spring.SpringUtil;
@ -26,6 +27,9 @@ import com.alibaba.cloud.ai.dashscope.embedding.DashScopeEmbeddingModel;
 import com.alibaba.cloud.ai.dashscope.embedding.DashScopeEmbeddingOptions;
 import com.alibaba.cloud.ai.dashscope.image.DashScopeImageModel;
 import com.azure.ai.openai.OpenAIClientBuilder;
+import io.micrometer.observation.ObservationRegistry;
+import io.qdrant.client.QdrantClient;
+import io.qdrant.client.QdrantGrpcClient;
 import lombok.SneakyThrows;
 import org.springframework.ai.autoconfigure.azure.openai.AzureOpenAiAutoConfiguration;
 import org.springframework.ai.autoconfigure.azure.openai.AzureOpenAiChatProperties;
@ -33,11 +37,14 @@ import org.springframework.ai.autoconfigure.azure.openai.AzureOpenAiConnectionPr
 import org.springframework.ai.autoconfigure.ollama.OllamaAutoConfiguration;
 import org.springframework.ai.autoconfigure.openai.OpenAiAutoConfiguration;
 import org.springframework.ai.autoconfigure.qianfan.QianFanAutoConfiguration;
+import org.springframework.ai.autoconfigure.vectorstore.qdrant.QdrantVectorStoreAutoConfiguration;
+import org.springframework.ai.autoconfigure.vectorstore.qdrant.QdrantVectorStoreProperties;
 import org.springframework.ai.autoconfigure.zhipuai.ZhiPuAiAutoConfiguration;
 import org.springframework.ai.autoconfigure.zhipuai.ZhiPuAiConnectionProperties;
 import org.springframework.ai.azure.openai.AzureOpenAiChatModel;
 import org.springframework.ai.chat.model.ChatModel;
 import org.springframework.ai.document.MetadataMode;
+import org.springframework.ai.embedding.BatchingStrategy;
 import org.springframework.ai.embedding.EmbeddingModel;
 import org.springframework.ai.image.ImageModel;
 import org.springframework.ai.ollama.OllamaChatModel;
@ -57,10 +64,15 @@ import org.springframework.ai.stabilityai.StabilityAiImageModel;
 import org.springframework.ai.stabilityai.api.StabilityAiApi;
 import org.springframework.ai.vectorstore.SimpleVectorStore;
 import org.springframework.ai.vectorstore.VectorStore;
+import org.springframework.ai.vectorstore.observation.DefaultVectorStoreObservationConvention;
+import org.springframework.ai.vectorstore.observation.VectorStoreObservationConvention;
+import org.springframework.ai.vectorstore.qdrant.QdrantVectorStore;
 import org.springframework.ai.zhipuai.ZhiPuAiChatModel;
 import org.springframework.ai.zhipuai.ZhiPuAiImageModel;
 import org.springframework.ai.zhipuai.api.ZhiPuAiApi;
 import org.springframework.ai.zhipuai.api.ZhiPuAiImageApi;
+import org.springframework.beans.BeansException;
+import org.springframework.beans.factory.ObjectProvider;
 import org.springframework.web.client.RestClient;

 import java.io.File;
@ -214,13 +226,14 @@ public class AiModelFactoryImpl implements AiModelFactory {

    @Override
    public VectorStore getOrCreateVectorStore(Class<? extends VectorStore> type, EmbeddingModel embeddingModel) {
-        // String cacheKey = buildClientCacheKey(VectorStore.class, platform, apiKey,
-        // url);
        String cacheKey = buildClientCacheKey(VectorStore.class, embeddingModel, type);
        return Singleton.get(cacheKey, (Func0<VectorStore>) () -> {
            if (type == SimpleVectorStore.class) {
                return buildSimpleVectorStore(embeddingModel);
            }
+            if (type == QdrantVectorStore.class) {
+                return buildQdrantVectorStore(embeddingModel);
+            }
            throw new IllegalArgumentException(StrUtil.format("未知类型({})", type));
            // TODO @芋艿：先临时使用 store
            // TODO @芋艿：@xin：后续看看，是不是切到阿里云之类的
@ -456,4 +469,38 @@ public class AiModelFactoryImpl implements AiModelFactory {
        return vectorStore;
    }

+    private QdrantVectorStore buildQdrantVectorStore(EmbeddingModel embeddingModel) {
+        QdrantVectorStoreAutoConfiguration configuration = new QdrantVectorStoreAutoConfiguration();
+        QdrantVectorStoreProperties vectorStoreProperties = SpringUtil.getBean(QdrantVectorStoreProperties.class);
+        // 参考 QdrantVectorStoreAutoConfiguration 实现，创建 QdrantClient 对象
+        QdrantGrpcClient.Builder grpcClientBuilder = QdrantGrpcClient.newBuilder(
+                vectorStoreProperties.getHost(), vectorStoreProperties.getPort(), vectorStoreProperties.isUseTls());
+        if (StrUtil.isNotEmpty(vectorStoreProperties.getApiKey())) {
+            grpcClientBuilder.withApiKey(vectorStoreProperties.getApiKey());
+        }
+        QdrantClient qdrantClient = new QdrantClient(grpcClientBuilder.build());
+        // 参考 QdrantVectorStoreAutoConfiguration 实现，实现 batchingStrategy
+        BatchingStrategy batchingStrategy = ReflectUtil.invoke(configuration, "batchingStrategy");
+
+        // 创建 QdrantVectorStore 对象
+        ObjectProvider<ObservationRegistry> observationRegistry = new ObjectProvider<>() {
+
+            @Override
+            public ObservationRegistry getObject() throws BeansException {
+                return SpringUtil.getBean(ObservationRegistry.class);
+            }
+
+        };
+        ObjectProvider <VectorStoreObservationConvention> customObservationConvention = new ObjectProvider<>() {
+
+            @Override
+            public VectorStoreObservationConvention getObject() throws BeansException {
+                return new DefaultVectorStoreObservationConvention();
+            }
+
+        };
+        return configuration.vectorStore(embeddingModel, vectorStoreProperties, qdrantClient,
+                observationRegistry, customObservationConvention, batchingStrategy);
+    }
+
 }
--- a/yudao-server/src/main/resources/application-dev.yaml
+++ b/yudao-server/src/main/resources/application-dev.yaml
@ -4,6 +4,11 @@ server:
 --- #################### 数据库相关配置 ####################

 spring:
+  spring:
+    autoconfigure:
+      exclude:
+        - org.springframework.ai.autoconfigure.vectorstore.qdrant.QdrantVectorStoreAutoConfiguration # 禁用 AI 模块的 Qdrant，手动创建
+        - org.springframework.ai.autoconfigure.vectorstore.milvus.MilvusVectorStoreAutoConfiguration # 禁用 AI 模块的 Milvus，手动创建
  # 数据源配置项
  datasource:
    druid: # Druid 【监控】相关的全局配置
--- a/yudao-server/src/main/resources/application-local.yaml
+++ b/yudao-server/src/main/resources/application-local.yaml
@ -3,13 +3,15 @@ server:

 --- #################### 数据库相关配置 ####################
 spring:
-  # 数据源配置项
  autoconfigure:
    exclude:
      - org.springframework.boot.autoconfigure.quartz.QuartzAutoConfiguration # 默认 local 环境，不开启 Quartz 的自动配置
      - de.codecentric.boot.admin.server.config.AdminServerAutoConfiguration # 禁用 Spring Boot Admin 的 Server 的自动配置
      - de.codecentric.boot.admin.server.ui.config.AdminServerUiAutoConfiguration # 禁用 Spring Boot Admin 的 Server UI 的自动配置
      - de.codecentric.boot.admin.client.config.SpringBootAdminClientAutoConfiguration # 禁用 Spring Boot Admin 的 Client 的自动配置
+      - org.springframework.ai.autoconfigure.vectorstore.qdrant.QdrantVectorStoreAutoConfiguration # 禁用 AI 模块的 Qdrant，手动创建
+      - org.springframework.ai.autoconfigure.vectorstore.milvus.MilvusVectorStoreAutoConfiguration # 禁用 AI 模块的 Milvus，手动创建
+  # 数据源配置项
  datasource:
    druid: # Druid 【监控】相关的全局配置
      web-stat-filter:
--- a/yudao-server/src/main/resources/application.yaml
+++ b/yudao-server/src/main/resources/application.yaml
@ -151,6 +151,12 @@ spring:
      redis:
        index: default-index
        prefix: "default:"
+      qdrant:
+        collection-name: knowledge_segment
+        host: 127.0.0.1
+        port: 6334
+        use-tls: false
+        api-key:
    qianfan: # 文心一言
      api-key: x0cuLZ7XsaTCU08vuJWO87Lg
      secret-key: R9mYF9dl9KASgi5RUq0FQt3wRisSnOcK