diff --git a/electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/config/ElePropertyConfig.java b/electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/config/ElePropertyConfig.java index 8fc7b42..6709f98 100644 --- a/electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/config/ElePropertyConfig.java +++ b/electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/config/ElePropertyConfig.java @@ -62,6 +62,10 @@ public class ElePropertyConfig { @Value("${backup.mysql.path}") private String sqlDirs; + @Getter + @Value("${ai.store.max.docs:200}") + private Integer aiMaxDocs; + public String getEleTmpPath() { if (EleCommonUtil.isWinOs()) { return winPrefix + eleTmpPath; diff --git a/electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/service/serviceimpl/ChatService.java b/electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/service/serviceimpl/ChatService.java index 10eaa4e..a1d17e2 100644 --- a/electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/service/serviceimpl/ChatService.java +++ b/electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/service/serviceimpl/ChatService.java @@ -32,6 +32,7 @@ import org.springframework.ai.document.DocumentReader; import org.springframework.ai.ollama.OllamaChatModel; import org.springframework.ai.reader.TextReader; import org.springframework.ai.reader.pdf.PagePdfDocumentReader; +import org.springframework.ai.transformer.splitter.TokenTextSplitter; import org.springframework.ai.vectorstore.VectorStore; import org.springframework.ai.chat.model.ChatResponse; import org.springframework.stereotype.Service; @@ -115,7 +116,7 @@ public class ChatService { case "txt", "csv", "text" -> { Path tempFile = saveUploadedFileToTemp(file); DocumentReader documentReader = new TextReader(String.valueOf(tempFile.toUri().toURL())); - documents = documentReader.get(); + documents = new TokenTextSplitter().apply(documentReader.get()); Files.deleteIfExists(tempFile); } case "xls" -> { @@ -212,6 +213,7 @@ public class ChatService { return ElectromagneticResultUtil.fail("-1", StrFormatter.format("当前格式 {} 不支持", fileType)); } } + documents = documents.subList(0, Math.min(elePropertyConfig.getAiMaxDocs(), documents.size())); vectorStore.write(documents); for (Document document : documents) { aiFileUploadRecordMapper.insert(new AiFileUploadRecord().setId(IdWorker.getSnowFlakeIdString()) diff --git a/electrmangnetic/src/main/resources/application.properties b/electrmangnetic/src/main/resources/application.properties index 3be9c04..21cfb86 100644 --- a/electrmangnetic/src/main/resources/application.properties +++ b/electrmangnetic/src/main/resources/application.properties @@ -34,6 +34,7 @@ pagehelper.reasonable=false server.port=12396 winPrefix:D:/tmp data.ele.tmp.path=/szsd/data/ele/prj/dev/tmp +ai.store.max.docs=200 # sys path data.sys.prj.path=/szsd/data/ele/prj/dev/sys_project/ data.sys.upload.path=/szsd/data/ele/prj/dev/sys_upload/ diff --git a/electrmangnetic/src/test/java/Test1.java b/electrmangnetic/src/test/java/Test1.java index 28b9bf2..af8bc76 100644 --- a/electrmangnetic/src/test/java/Test1.java +++ b/electrmangnetic/src/test/java/Test1.java @@ -2,10 +2,15 @@ //import jakarta.annotation.Resource; //import org.junit.jupiter.api.Test; //import org.springframework.ai.document.Document; -//import org.springframework.ai.reader.pdf.PagePdfDocumentReader; +//import org.springframework.ai.document.DocumentReader; +//import org.springframework.ai.reader.TextReader; +//import org.springframework.ai.transformer.splitter.TokenTextSplitter; //import org.springframework.ai.vectorstore.VectorStore; //import org.springframework.boot.test.context.SpringBootTest; // +//import java.io.File; +//import java.net.MalformedURLException; +//import java.nio.file.Path; //import java.util.List; // //@SpringBootTest(classes = Application.class) @@ -15,11 +20,19 @@ // private VectorStore vectorStore; // // @Test -// public void testTree() { -// String path = "D:/wjj.pdf"; -// PagePdfDocumentReader pagePdfDocumentReader = new PagePdfDocumentReader(path); -// List read = pagePdfDocumentReader.read(); -// System.out.println("read = " + read); +// public void testTree() throws MalformedURLException { +// String path = "D:/明朝那些事儿.txt"; +// File file = new File(path); +// Path tempFile = file.toPath(); +// DocumentReader documentReader = new TextReader(String.valueOf(tempFile.toUri().toURL())); +// List documents = new TokenTextSplitter().apply(documentReader.get()); +// documents = documents.subList(0, 200); +// vectorStore.write(documents); +// +//// Path tempFile = Paths.get(file.getAbsolutePath()); +//// DocumentReader documentReader = new TextReader(String.valueOf(tempFile.toUri().toURL())); +//// List documents = documentReader.get(); +//// vectorStore.write(documents); // } // //}