chenxudong
/
electromagnetic-data-new
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

解决AI中大文件上传的问题

This commit is contained in:
chenxudong 2025-04-08 09:14:23 +08:00
parent 081bbbd629
commit 98ed6eafdb
4 changed files with 27 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/config/ElePropertyConfig.java
View File

@ -62,6 +62,10 @@ public class ElePropertyConfig {
@Value("${backup.mysql.path}")
private String sqlDirs;
@Getter
@Value("${ai.store.max.docs:200}")
private Integer aiMaxDocs;
public String getEleTmpPath() {
if (EleCommonUtil.isWinOs()) {
return winPrefix + eleTmpPath;

4
electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/service/serviceimpl/ChatService.java
View File

@ -32,6 +32,7 @@ import org.springframework.ai.document.DocumentReader;
import org.springframework.ai.ollama.OllamaChatModel;
import org.springframework.ai.reader.TextReader;
import org.springframework.ai.reader.pdf.PagePdfDocumentReader;
import org.springframework.ai.transformer.splitter.TokenTextSplitter;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.stereotype.Service;
@ -115,7 +116,7 @@ public class ChatService {
case "txt", "csv", "text" -> {
Path tempFile = saveUploadedFileToTemp(file);
DocumentReader documentReader = new TextReader(String.valueOf(tempFile.toUri().toURL()));
documents = documentReader.get();
documents = new TokenTextSplitter().apply(documentReader.get());
Files.deleteIfExists(tempFile);
}
case "xls" -> {
@ -212,6 +213,7 @@ public class ChatService {
return ElectromagneticResultUtil.fail("-1", StrFormatter.format("当前格式 {} 不支持", fileType));
}
}
documents = documents.subList(0, Math.min(elePropertyConfig.getAiMaxDocs(), documents.size()));
vectorStore.write(documents);
for (Document document : documents) {
aiFileUploadRecordMapper.insert(new AiFileUploadRecord().setId(IdWorker.getSnowFlakeIdString())

1
electrmangnetic/src/main/resources/application.properties
View File

@ -34,6 +34,7 @@ pagehelper.reasonable=false
server.port=12396
winPrefix:D:/tmp
data.ele.tmp.path=/szsd/data/ele/prj/dev/tmp
ai.store.max.docs=200
# sys path
data.sys.prj.path=/szsd/data/ele/prj/dev/sys_project/
data.sys.upload.path=/szsd/data/ele/prj/dev/sys_upload/

25
electrmangnetic/src/test/java/Test1.java
View File

@ -2,10 +2,15 @@
//import jakarta.annotation.Resource;
//import org.junit.jupiter.api.Test;
//import org.springframework.ai.document.Document;
//import org.springframework.ai.reader.pdf.PagePdfDocumentReader;
//import org.springframework.ai.document.DocumentReader;
//import org.springframework.ai.reader.TextReader;
//import org.springframework.ai.transformer.splitter.TokenTextSplitter;
//import org.springframework.ai.vectorstore.VectorStore;
//import org.springframework.boot.test.context.SpringBootTest;
//
//import java.io.File;
//import java.net.MalformedURLException;
//import java.nio.file.Path;
//import java.util.List;
//
//@SpringBootTest(classes = Application.class)
@ -15,11 +20,19 @@
// private VectorStore vectorStore;
//
// @Test
// public void testTree() {
// String path = "D:/wjj.pdf";
// PagePdfDocumentReader pagePdfDocumentReader = new PagePdfDocumentReader(path);
// List<Document> read = pagePdfDocumentReader.read();
// System.out.println("read = " + read);
// public void testTree() throws MalformedURLException {
// String path = "D:/明朝那些事儿.txt";
// File file = new File(path);
// Path tempFile = file.toPath();
// DocumentReader documentReader = new TextReader(String.valueOf(tempFile.toUri().toURL()));
// List<Document> documents = new TokenTextSplitter().apply(documentReader.get());
// documents = documents.subList(0, 200);
// vectorStore.write(documents);
//
//// Path tempFile = Paths.get(file.getAbsolutePath());
//// DocumentReader documentReader = new TextReader(String.valueOf(tempFile.toUri().toURL()));
//// List<Document> documents = documentReader.get();
//// vectorStore.write(documents);
// }
//
//}