chenxudong
/
electromagnetic-data-new
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

测试通了AI文件上传。

This commit is contained in:
chenxudong 2025-04-03 12:02:04 +08:00
parent b8476c2398
commit 87cfbb4d0e
7 changed files with 35 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
electrmangnetic/pom.xml
View File

@ -128,10 +128,6 @@
<artifactId>elasticsearch-java</artifactId>
<version>8.13.4</version>
</dependency>
<dependency>
<groupId>org.springframework.ai</groupId>
<artifactId>spring-ai-pdf-document-reader</artifactId>
</dependency>
</dependencies>
<build>

11
electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/config/LoginInterceptor.java
View File

@ -3,6 +3,7 @@ package com.electromagnetic.industry.software.manage.config;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.date.SystemClock;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.json.JSONUtil;
import com.electromagnetic.industry.software.common.annotations.UserOperation;
import com.electromagnetic.industry.software.common.cons.UserConstants;
@ -131,14 +132,14 @@ public class LoginInterceptor implements HandlerInterceptor {
@Override
public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) throws Exception {
if (request.getRequestURI().endsWith("index") || request.getRequestURI().contains("/data/ed/ai/")) {
return;
}
long accessStartTime = (long) request.getSession().getAttribute("accessStartTime");
long accessEndTime = System.currentTimeMillis();
UserOperation userOperation = ((HandlerMethod) handler).getMethod().getAnnotation(UserOperation.class);
if (ObjectUtil.isEmpty(userOperation)) {
return;
}
String reqArgs = UserThreadLocal.getReqArgs();
UserLoginInfo user = Optional.of(UserThreadLocal.getUser()).orElse(new UserLoginInfo());

7
electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/controller/AiController.java
View File

@ -1,5 +1,6 @@
package com.electromagnetic.industry.software.manage.controller;
import cn.hutool.core.util.StrUtil;
import com.electromagnetic.industry.software.common.resp.ElectromagneticResult;
import com.electromagnetic.industry.software.common.util.ElectromagneticResultUtil;
import com.electromagnetic.industry.software.manage.ai.ChatTaskThread1;
@ -44,6 +45,9 @@ public class AiController {
@PostMapping(path = "/chatStreamStr", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
public Flux<String> chatStreamStr(@RequestBody QueryDTO queryDTO) throws ExecutionException, InterruptedException {
if (StrUtil.isEmpty(queryDTO.getMsg())) {
return Flux.empty();
}
ChatTaskThread2 chatTaskThread = new ChatTaskThread2(chatService, queryDTO);
Future<Flux<String>> future = ThreadUtil.getThreadPool().submit(chatTaskThread);
return future.get();
@ -51,6 +55,9 @@ public class AiController {
@PostMapping(path = "/chatStreamResp", produces = MediaType.TEXT_EVENT_STREAM_VALUE)
public Flux<ChatResponse> chatStreamResp(@RequestBody QueryDTO queryDTO) throws ExecutionException, InterruptedException {
if (StrUtil.isEmpty(queryDTO.getMsg())) {
return Flux.empty();
}
ChatTaskThread3 chatTaskThread = new ChatTaskThread3(chatService, queryDTO);
Future<Flux<ChatResponse>> future = ThreadUtil.getThreadPool().submit(chatTaskThread);
return future.get();

7
electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/controller/FileFormatController.java
View File

@ -35,10 +35,5 @@ public class FileFormatController {
public ElectromagneticResult<?> list() {
return ElectromagneticResultUtil.success(fileFormatService.getList());
}
//
// @PostMapping("/edit")
// @UserOperation(value = "修改文件格式", modelName = UserOperationModuleEnum.DATABASE)
// public ElectromagneticResult<?> editFileFormat(@RequestBody FileFormat fileFormat) {
// return ElectromagneticResultUtil.success(fileFormatService.updateFileFormat(fileFormat));
// }
}

28
electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/controller/RepoEdFileInfoController.java
View File

@ -14,7 +14,6 @@ import com.electromagnetic.industry.software.manage.pojo.req.FileChunkDTO;
import com.electromagnetic.industry.software.manage.pojo.req.FileInfoQueryDTO;
import com.electromagnetic.industry.software.manage.pojo.req.UpdateFileInfoDTO;
import com.electromagnetic.industry.software.manage.service.EdFileInfoService;
import com.electromagnetic.industry.software.manage.service.serviceimpl.EdFileRelationServiceImpl;
import jakarta.annotation.Resource;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.core.io.InputStreamResource;
@ -30,8 +29,6 @@ public class RepoEdFileInfoController {
@Resource
private EdFileInfoService edFileInfoService;
@Resource
private EdFileRelationServiceImpl edFileRelationService;
@UserOperation(value = "查看工程树", modelName = UserOperationModuleEnum.REPO_PRJ_DATABASE)
@RequestMapping("tree")
@ -179,29 +176,4 @@ public class RepoEdFileInfoController {
String userId = UserThreadLocal.getUserId();
return ElectromagneticResultUtil.success(edFileInfoService.addFavorite(userId, id));
}
// /**
// * 从收藏夹移除
// *
// * @param id 文件id
// * @return
// */
// @GetMapping("/removeCollection")
// @UserOperation(value = "从收藏夹移除了文件", modelName = UserOperationModuleEnum.REPO_PRJ_DATABASE)
// public ElectromagneticResult<?> removeFavorite(@RequestParam String id) {
// String userId = UserThreadLocal.getUserId();
// return ElectromagneticResultUtil.success(edFileInfoService.removeFavorite(userId, id));
// }
//
// /**
// * 展示当前用户收藏夹文件
// *
// * @return
// */
// @PostMapping("/listCollection")
// @UserOperation(value = "查看了收藏夹", modelName = UserOperationModuleEnum.REPO_PRJ_DATABASE)
// public ElectromagneticResult<?> listFavorite(@RequestBody FileInfoQueryDTO fileInfoQueryDTO) {
// String userId = UserThreadLocal.getUserId();
// return ElectromagneticResultUtil.success(edFileInfoService.findFavorite(userId, fileInfoQueryDTO));
// }
}

9
electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/pojo/models/AiFileUploadRecord.java
View File

@ -1,5 +1,6 @@
package com.electromagnetic.industry.software.manage.pojo.models;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableName;
import lombok.Data;
import lombok.experimental.Accessors;
@ -9,9 +10,15 @@ import lombok.experimental.Accessors;
@Accessors(chain = true)
public class AiFileUploadRecord extends BaseModel {
@TableField(value = "id")
private String id;
@TableField(value = "file_name")
private String fileName;
private String md5;
@TableField(value = "file_md5")
private String fileMd5;
@TableField(value = "vector_id")
private String vectorId;
}

37
electrmangnetic/src/main/java/com/electromagnetic/industry/software/manage/service/serviceimpl/ChatService.java
View File

@ -1,11 +1,12 @@
package com.electromagnetic.industry.software.manage.service.serviceimpl;
import cn.hutool.core.collection.ListUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.crypto.digest.DigestUtil;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.electromagnetic.industry.software.common.resp.ElectromagneticResult;
import com.electromagnetic.industry.software.common.util.EleCommonUtil;
import com.electromagnetic.industry.software.common.util.ElectromagneticResultUtil;
import com.electromagnetic.industry.software.common.util.IdWorker;
import com.electromagnetic.industry.software.manage.mapper.AiFileUploadRecordMapper;
@ -22,16 +23,12 @@ import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.document.Document;
import org.springframework.ai.ollama.OllamaChatModel;
import org.springframework.ai.reader.pdf.PagePdfDocumentReader;
import org.springframework.ai.vectorstore.VectorStore;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.multipart.MultipartFile;
import reactor.core.publisher.Flux;
import java.io.File;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -52,19 +49,9 @@ public class ChatService {
@Resource
private QuestionAnswerAdvisor questionAnswerAdvisor;
@Value("file.md5RecordPath")
private String uploadFileMd5RecordPath;
@Resource
private AiFileUploadRecordMapper aiFileUploadRecordMapper;
@PostMapping
public void init() {
if (!FileUtil.exist(uploadFileMd5RecordPath)) {
FileUtil.touch(uploadFileMd5RecordPath);
}
}
public void add(String content) {
List<Document> documents = Stream.of(content).map(Document::new).collect(Collectors.toList());
vectorStore.write(documents);
@ -72,6 +59,11 @@ public class ChatService {
@Transactional(rollbackFor = Exception.class)
public ElectromagneticResult<?> addFromUpload(MultipartFile file) throws Exception {
if (file.isEmpty()) {
return ElectromagneticResultUtil.fail("-1", "文件为空");
}
String fileType = FileUtil.extName(file.getOriginalFilename());
if (!StrUtil.equals(fileType, "pdf")) {
return ElectromagneticResultUtil.fail("-1", "当前仅支持pdf格式文件");
@ -79,22 +71,19 @@ public class ChatService {
String fileMd5 = DigestUtil.md5Hex(file.getInputStream());
Long count = aiFileUploadRecordMapper.selectCount(Wrappers.lambdaQuery(AiFileUploadRecord.class)
.eq(AiFileUploadRecord::getMd5, fileMd5));
.eq(AiFileUploadRecord::getFileMd5, fileMd5));
if (count > 0) {
return ElectromagneticResultUtil.success(fileMd5);
}
String pdfTmpPath = FileUtil.getParent(uploadFileMd5RecordPath, 1) + File.separator + IdUtil.fastSimpleUUID() + "." + fileType;
FileUtil.writeFromStream(file.getInputStream(), pdfTmpPath);
List<Document> documents = new PagePdfDocumentReader(pdfTmpPath).read();
vectorStore.write(documents);
String content = EleCommonUtil.parse(file.getInputStream(), "pdf");
Document document = new Document(content);
vectorStore.write(ListUtil.of(document));
aiFileUploadRecordMapper.insert(new AiFileUploadRecord().setId(IdWorker.getSnowFlakeIdString())
.setMd5(fileMd5)
.setVectorId(document.getId())
.setFileMd5(fileMd5)
.setFileName(file.getOriginalFilename()));
FileUtil.del(pdfTmpPath);
return ElectromagneticResultUtil.success(fileMd5);
}