解决AI中大文件上传的问题

electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/config/ElePropertyConfig.java

@@ -62,6 +62,10 @@ public class ElePropertyConfig {
     @Value("${backup.mysql.path}")
     private String sqlDirs;
 
+    @Getter
+    @Value("${ai.store.max.docs:200}")
+    private Integer aiMaxDocs;
+
     public String getEleTmpPath() {
         if (EleCommonUtil.isWinOs()) {
             return winPrefix + eleTmpPath;

electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/service/serviceimpl/ChatService.java

@@ -32,6 +32,7 @@ import org.springframework.ai.document.DocumentReader;
 import org.springframework.ai.ollama.OllamaChatModel;
 import org.springframework.ai.reader.TextReader;
 import org.springframework.ai.reader.pdf.PagePdfDocumentReader;
+import org.springframework.ai.transformer.splitter.TokenTextSplitter;
 import org.springframework.ai.vectorstore.VectorStore;
 import org.springframework.ai.chat.model.ChatResponse;
 import org.springframework.stereotype.Service;
@@ -115,7 +116,7 @@ public class ChatService {
             case "txt", "csv", "text" -> {
                 Path tempFile = saveUploadedFileToTemp(file);
                 DocumentReader documentReader = new TextReader(String.valueOf(tempFile.toUri().toURL()));
-                documents = documentReader.get();
+                documents = new TokenTextSplitter().apply(documentReader.get());
                 Files.deleteIfExists(tempFile);
             }
             case "xls" -> {
@@ -212,6 +213,7 @@ public class ChatService {
                 return ElectromagneticResultUtil.fail("-1", StrFormatter.format("当前格式 {} 不支持", fileType));
             }
         }
+        documents = documents.subList(0, Math.min(elePropertyConfig.getAiMaxDocs(), documents.size()));
         vectorStore.write(documents);
         for (Document document : documents) {
             aiFileUploadRecordMapper.insert(new AiFileUploadRecord().setId(IdWorker.getSnowFlakeIdString())

electrmangnetic/src/main/resources/application.properties

@@ -34,6 +34,7 @@ pagehelper.reasonable=false
 server.port=12396
 winPrefix:D:/tmp
 data.ele.tmp.path=/szsd/data/ele/prj/dev/tmp
+ai.store.max.docs=200
 # sys path
 data.sys.prj.path=/szsd/data/ele/prj/dev/sys_project/
 data.sys.upload.path=/szsd/data/ele/prj/dev/sys_upload/

electrmangnetic/src/test/java/Test1.java

@@ -2,10 +2,15 @@
 //import jakarta.annotation.Resource;
 //import org.junit.jupiter.api.Test;
 //import org.springframework.ai.document.Document;
-//import org.springframework.ai.reader.pdf.PagePdfDocumentReader;
+//import org.springframework.ai.document.DocumentReader;
+//import org.springframework.ai.reader.TextReader;
+//import org.springframework.ai.transformer.splitter.TokenTextSplitter;
 //import org.springframework.ai.vectorstore.VectorStore;
 //import org.springframework.boot.test.context.SpringBootTest;
 //
+//import java.io.File;
+//import java.net.MalformedURLException;
+//import java.nio.file.Path;
 //import java.util.List;
 //
 //@SpringBootTest(classes = Application.class)
@@ -15,11 +20,19 @@
 //    private VectorStore vectorStore;
 //
 //    @Test
-//    public void testTree() {
+//    public void testTree() throws MalformedURLException {
-//        String path = "D:/wjj.pdf";
+//        String path = "D:/明朝那些事儿.txt";
-//        PagePdfDocumentReader pagePdfDocumentReader = new PagePdfDocumentReader(path);
+//        File file = new File(path);
-//        List<Document> read = pagePdfDocumentReader.read();
+//        Path tempFile = file.toPath();
-//        System.out.println("read = " + read);
+//        DocumentReader documentReader = new TextReader(String.valueOf(tempFile.toUri().toURL()));
+//        List<Document> documents = new TokenTextSplitter().apply(documentReader.get());
+//        documents = documents.subList(0, 200);
+//        vectorStore.write(documents);
+//
+////        Path tempFile = Paths.get(file.getAbsolutePath());
+////        DocumentReader documentReader = new TextReader(String.valueOf(tempFile.toUri().toURL()));
+////        List<Document> documents = documentReader.get();
+////        vectorStore.write(documents);
 //    }
 //
 //}