From b568affb14092f0e3450c644480ca2b662e9b6a4 Mon Sep 17 00:00:00 2001 From: Joshi <3040996759@qq.com> Date: Mon, 15 Dec 2025 17:26:21 +0800 Subject: [PATCH] =?UTF-8?q?feat(system):=20=E6=96=B0=E5=A2=9E=E5=A4=A7?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=88=86=E7=89=87=E4=B8=8A=E4=BC=A0=E5=8A=9F?= =?UTF-8?q?=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 在ISysOssService接口中增加文件流式上传方法，支持避免OOM - 实现SysOssServiceImpl中的文件流式上传逻辑 - 新增BigUploadController控制器处理分片上传、合并、恢复与清理 - 添加ChunkedUploadService服务类管理分片上传业务逻辑 - 创建BigUploadCleanupScheduler定时任务清理过期分片 - 提供FileMultipartFile适配器将File对象转换为MultipartFile --- .../common/BigUploadController.java | 111 ++++++++ .../schedule/BigUploadCleanupScheduler.java | 30 ++ .../web/service/ChunkedUploadService.java | 258 ++++++++++++++++++ .../service/support/FileMultipartFile.java | 73 +++++ .../ruoyi/system/service/ISysOssService.java | 10 + .../service/impl/SysOssServiceImpl.java | 29 ++ 6 files changed, 511 insertions(+) create mode 100644 ruoyi-admin/src/main/java/com/ruoyi/web/controller/common/BigUploadController.java create mode 100644 ruoyi-admin/src/main/java/com/ruoyi/web/schedule/BigUploadCleanupScheduler.java create mode 100644 ruoyi-admin/src/main/java/com/ruoyi/web/service/ChunkedUploadService.java create mode 100644 ruoyi-admin/src/main/java/com/ruoyi/web/service/support/FileMultipartFile.java diff --git a/ruoyi-admin/src/main/java/com/ruoyi/web/controller/common/BigUploadController.java b/ruoyi-admin/src/main/java/com/ruoyi/web/controller/common/BigUploadController.java new file mode 100644 index 0000000..9652205 --- /dev/null +++ b/ruoyi-admin/src/main/java/com/ruoyi/web/controller/common/BigUploadController.java @@ -0,0 +1,111 @@ +package com.ruoyi.web.controller.common; + +import com.ruoyi.common.core.AjaxResult; +import com.ruoyi.system.domain.vo.SysOssVo; +import com.ruoyi.web.service.ChunkedUploadService; +import lombok.RequiredArgsConstructor; +import org.springframework.http.MediaType; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.util.List; + +@Validated +@RestController +@RequiredArgsConstructor +@RequestMapping("/common/bigUpload") +public class BigUploadController { + + private final ChunkedUploadService chunkedUploadService; + + + /** + * 上传文件分片 + * + * @param chunk 文件分片数据 + * @param chunkIndex 分片索引 + * @param totalChunks 总分片数 + * @param fileMd5 文件MD5值 + * @param fileName 文件名 + * @return 上传结果 + */ + @PostMapping(value = "/chunk", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public AjaxResult uploadChunk(@RequestPart("chunk") MultipartFile chunk, + @RequestParam("chunkIndex") int chunkIndex, + @RequestParam("totalChunks") int totalChunks, + @RequestParam("fileMd5") String fileMd5, + @RequestParam("fileName") String fileName) { + try { + chunkedUploadService.saveChunk(fileMd5, fileName, chunkIndex, totalChunks, chunk); + return AjaxResult.success(); + } catch (Exception e) { + return AjaxResult.error(e.getMessage()); + } + } + + /** + * 合并文件分片 + * + * @param fileMd5 文件MD5值 + * @param fileName 文件名 + * @param totalChunks 总分片数 + * @return 合并结果及文件信息 + */ + @PostMapping("/merge") + public AjaxResult merge(@RequestParam("fileMd5") String fileMd5, + @RequestParam("fileName") String fileName, + @RequestParam("totalChunks") int totalChunks) { + try { + SysOssVo oss = chunkedUploadService.mergeChunks(fileMd5, fileName, totalChunks); + return AjaxResult.success() + .put("url", oss.getUrl()) + .put("fileName", oss.getOriginalName()) + .put("ossId", oss.getOssId()); + } catch (Exception e) { + return AjaxResult.error(e.getMessage()); + } + } + /** + * 恢复上传（删除最后一个分片） + * + * @param fileMd5 文件MD5值 + * @return 操作结果 + */ + @PostMapping("/resume") + public AjaxResult resume(@RequestParam("fileMd5") String fileMd5) { + try { + boolean deleted = chunkedUploadService.deleteLastChunk(fileMd5); + return deleted ? AjaxResult.success() : AjaxResult.error("没有可删除的分片"); + } catch (Exception e) { + return AjaxResult.error(e.getMessage()); + } + } + /** + * 获取已上传的分片列表 + * + * @param fileMd5 文件MD5值 + * @return 已上传分片索引列表 + */ + @GetMapping("/list") + public AjaxResult listUploaded(@RequestParam("fileMd5") String fileMd5) { + try { + List indices = chunkedUploadService.listUploadedChunks(fileMd5); + return AjaxResult.success().put("uploaded", indices); + } catch (Exception e) { + return AjaxResult.error(e.getMessage()); + } + } + + /** + * 清理过期的临时分片目录 + * + * @param olderThanHours 多少小时未更新则视为过期（默认24小时） + */ + @PostMapping("/cleanup") + public AjaxResult cleanup(@RequestParam(value = "olderThanHours", required = false, defaultValue = "24") int olderThanHours) { + long millis = olderThanHours * 60L * 60L * 1000L; + int removed = chunkedUploadService.cleanupStaleChunks(millis); + return AjaxResult.success().put("removed", removed).put("olderThanHours", olderThanHours); + } +} diff --git a/ruoyi-admin/src/main/java/com/ruoyi/web/schedule/BigUploadCleanupScheduler.java b/ruoyi-admin/src/main/java/com/ruoyi/web/schedule/BigUploadCleanupScheduler.java new file mode 100644 index 0000000..e74e51c --- /dev/null +++ b/ruoyi-admin/src/main/java/com/ruoyi/web/schedule/BigUploadCleanupScheduler.java @@ -0,0 +1,30 @@ +package com.ruoyi.web.schedule; + +import com.ruoyi.web.service.ChunkedUploadService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Slf4j +@Component +@EnableScheduling +@RequiredArgsConstructor +public class BigUploadCleanupScheduler { + + private final ChunkedUploadService chunkedUploadService; + + // 每天凌晨2点执行 + @Scheduled(cron = "0 0 2 * * ?") + public void cleanup() { + int hours = 24; // 阈值: 24小时未更新 + long millis = hours * 60L * 60L * 1000L; + try { + int removed = chunkedUploadService.cleanupStaleChunks(millis); + log.info("BigUploadCleanupScheduler removed {} stale chunk directories older than {} hours", removed, hours); + } catch (Exception e) { + log.error("BigUploadCleanupScheduler cleanup failed", e); + } + } +} diff --git a/ruoyi-admin/src/main/java/com/ruoyi/web/service/ChunkedUploadService.java b/ruoyi-admin/src/main/java/com/ruoyi/web/service/ChunkedUploadService.java new file mode 100644 index 0000000..348be22 --- /dev/null +++ b/ruoyi-admin/src/main/java/com/ruoyi/web/service/ChunkedUploadService.java @@ -0,0 +1,258 @@ +package com.ruoyi.web.service; + +import com.ruoyi.common.config.RuoYiConfig; +import com.ruoyi.system.domain.vo.SysOssVo; +import com.ruoyi.system.service.ISysOssService; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantLock; + +@Service +public class ChunkedUploadService { + + private final ISysOssService ossService; + private final ConcurrentMap mergeLocks = new ConcurrentHashMap<>(); + + public ChunkedUploadService(ISysOssService ossService) { + this.ossService = ossService; + } + + private File getTempDir(String fileMd5) { + String base = RuoYiConfig.getUploadPath(); + File dir = new File(base, "chunks" + File.separator + fileMd5); + if (!dir.exists()) { + //无检查结果方法调用忽略 + dir.mkdirs(); + } + return dir; + } + + private File getFinalDir() { + String base = RuoYiConfig.getUploadPath(); + File dir = new File(base, "big"); + if (!dir.exists()) { + //无检查结果方法调用忽略 + dir.mkdirs(); + } + return dir; + } + + public void saveChunk(String fileMd5, String fileName, int chunkIndex, int totalChunks, MultipartFile chunk) throws IOException { + File tempDir = getTempDir(fileMd5); + // 为了安全，记录总区块的元数据 + File meta = new File(tempDir, ".total"); + if (!meta.exists()) { + try (RandomAccessFile raf = new RandomAccessFile(meta, "rw")) { + raf.setLength(0); + raf.write(String.valueOf(totalChunks).getBytes()); + } + } + File partFile = new File(tempDir, chunkIndex + ".part"); + chunk.transferTo(partFile); + long now = System.currentTimeMillis(); + // 更新目录与元数据的修改时间 + // 无检查结果方法调用忽略 + partFile.setLastModified(now); + meta.setLastModified(now); + tempDir.setLastModified(now); + } + + public SysOssVo mergeChunks(String fileMd5, String fileName, int totalChunks) throws Exception { + ReentrantLock lock = mergeLocks.computeIfAbsent(fileMd5, k -> new ReentrantLock()); + lock.lock(); + try { + File tempDir = getTempDir(fileMd5); + if (!tempDir.exists() || !tempDir.isDirectory()) { + throw new IOException("临时分片目录不存在"); + } + // 验证 + List parts = new ArrayList<>(); + for (int i = 0; i < totalChunks; i++) { + File f = new File(tempDir, i + ".part"); + if (!f.exists()) { + throw new IOException("缺少分片: " + i); + } + parts.add(f); + } + // 计算大小和偏移量 + long[] sizes = new long[totalChunks]; + long[] offsets = new long[totalChunks]; + long totalSize = 0L; + for (int i = 0; i < totalChunks; i++) { + sizes[i] = parts.get(i).length(); + offsets[i] = totalSize; + totalSize += sizes[i]; + } + File finalDir = getFinalDir(); + String safeName = fileMd5 + "_" + (fileName != null ? fileName : "file"); + File target = new File(finalDir, safeName); + if (target.exists()) { + // 覆盖现有 + if (!target.delete()) { + throw new IOException("无法覆盖已存在的目标文件"); + } + } + // 预分配 + try (RandomAccessFile raf = new RandomAccessFile(target, "rw")) { + raf.setLength(totalSize); + } + int threads = Math.min(8, totalChunks); + ExecutorService pool = Executors.newFixedThreadPool(threads); + List> futures = new ArrayList<>(); + for (int i = 0; i < totalChunks; i++) { + final int idx = i; + futures.add(pool.submit(() -> { + try (RandomAccessFile writeRaf = new RandomAccessFile(target, "rw"); + RandomAccessFile readRaf = new RandomAccessFile(parts.get(idx), "r"); + FileChannel in = readRaf.getChannel(); + FileChannel out = writeRaf.getChannel()) { + out.position(offsets[idx]); + long remaining = sizes[idx]; + long pos = 0; + while (remaining > 0) { + long transferred = in.transferTo(pos, remaining, out); + if (transferred <= 0) break; + pos += transferred; + remaining -= transferred; + } + } catch (IOException e) { + throw new CompletionException(e); + } + })); + } + pool.shutdown(); + for (Future f : futures) { + f.get(); + } + // 合并成功后的清理部件 + for (File f : parts) { + // 无检查结果方法调用 + f.delete(); + } + // 无检查结果方法调用 + new File(tempDir, ".total").delete(); + // 移除空文件 + File[] remain = tempDir.listFiles(); + if (remain == null || remain.length == 0) { + // 无检查结果方法调用 + tempDir.delete(); + } + // 上传合并文件到 OSS 服务和记录（流式，避免OOM） + SysOssVo ossVo = ossService.upload(target, fileName, 0L); + // 上传完成后删除本地合并的临时文件（可选，节省磁盘） + if (target.exists()) { + // 无检查结果方法调用 + target.delete(); + } + return ossVo; + } finally { + try { + lock.unlock(); + } finally { + mergeLocks.remove(fileMd5, lock); + } + } + } + + public boolean deleteLastChunk(String fileMd5) { + File tempDir = getTempDir(fileMd5); + if (!tempDir.exists()) return false; + File[] files = tempDir.listFiles((dir, name) -> name.endsWith(".part")); + if (files == null || files.length == 0) return false; + Arrays.sort(files, Comparator.comparingInt(f -> Integer.parseInt(f.getName().replace(".part", "")))); + File last = files[files.length - 1]; + return last.delete(); + } + + public List listUploadedChunks(String fileMd5) throws IOException { + File tempDir = getTempDir(fileMd5); + if (!tempDir.exists()) { + return new ArrayList<>(); + } + File[] parts = tempDir.listFiles(new FilenameFilter() { + @Override + public boolean accept(File dir, String name) { + return name.endsWith(".part"); + } + }); + List result = new ArrayList<>(); + if (parts != null) { + for (File f : parts) { + String n = f.getName().replace(".part", ""); + try { + result.add(Integer.parseInt(n)); + } catch (NumberFormatException ignored) {} + } + } + result.sort(Integer::compareTo); + return result; + } + + public int cleanupStaleChunks(long olderThanMillis) { + int deleted = 0; + File base = new File(RuoYiConfig.getUploadPath(), "chunks"); + if (!base.exists() || !base.isDirectory()) { + return 0; + } + File[] dirs = base.listFiles(File::isDirectory); + if (dirs == null) return 0; + long now = System.currentTimeMillis(); + for (File dir : dirs) { + String key = dir.getName(); + ReentrantLock lock = mergeLocks.computeIfAbsent(key, k -> new ReentrantLock()); + if (!lock.tryLock()) { + continue; + } + try { + long lastMod = dir.lastModified(); + File[] files = dir.listFiles(); + if (files != null) { + for (File f : files) { + lastMod = Math.max(lastMod, f.lastModified()); + } + } + if (now - lastMod >= olderThanMillis) { + deleteDirectory(dir); + deleted++; + } + } finally { + try { + lock.unlock(); + } finally { + mergeLocks.remove(key, lock); + } + } + } + return deleted; + } + + private void deleteDirectory(File dir) { + File[] files = dir.listFiles(); + if (files != null) { + for (File f : files) { + if (f.isDirectory()) { + deleteDirectory(f); + } else { + // 无检查结果方法调用忽略 + f.delete(); + } + } + } + // 无检查结果方法调用忽略 + dir.delete(); + } +} diff --git a/ruoyi-admin/src/main/java/com/ruoyi/web/service/support/FileMultipartFile.java b/ruoyi-admin/src/main/java/com/ruoyi/web/service/support/FileMultipartFile.java new file mode 100644 index 0000000..5985e68 --- /dev/null +++ b/ruoyi-admin/src/main/java/com/ruoyi/web/service/support/FileMultipartFile.java @@ -0,0 +1,73 @@ +package com.ruoyi.web.service.support; + +import org.springframework.web.multipart.MultipartFile; + +import java.io.*; +import java.io.ByteArrayOutputStream; + +public class FileMultipartFile implements MultipartFile { + private final File file; + private final String originalFilename; + private final String contentType; + + public FileMultipartFile(File file, String originalFilename, String contentType) { + this.file = file; + this.originalFilename = originalFilename; + this.contentType = contentType; + } + + @Override + public String getName() { + return originalFilename; + } + + @Override + public String getOriginalFilename() { + return originalFilename; + } + + @Override + public String getContentType() { + return contentType; + } + + @Override + public boolean isEmpty() { + return file.length() == 0; + } + + @Override + public long getSize() { + return file.length(); + } + + @Override + public byte[] getBytes() throws IOException { + try (InputStream is = new FileInputStream(file)) { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + byte[] data = new byte[8192]; + int nRead; + while ((nRead = is.read(data, 0, data.length)) != -1) { + buffer.write(data, 0, nRead); + } + return buffer.toByteArray(); + } + } + + @Override + public InputStream getInputStream() throws IOException { + return new FileInputStream(file); + } + + @Override + public void transferTo(File dest) throws IOException { + try (InputStream in = new FileInputStream(file); + OutputStream out = new FileOutputStream(dest)) { + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + } + } +} diff --git a/ruoyi-system/src/main/java/com/ruoyi/system/service/ISysOssService.java b/ruoyi-system/src/main/java/com/ruoyi/system/service/ISysOssService.java index a123500..49d1b61 100644 --- a/ruoyi-system/src/main/java/com/ruoyi/system/service/ISysOssService.java +++ b/ruoyi-system/src/main/java/com/ruoyi/system/service/ISysOssService.java @@ -8,6 +8,7 @@ import org.springframework.web.multipart.MultipartFile; import javax.servlet.http.HttpServletResponse; import java.io.IOException; +import java.io.File; import java.util.Collection; import java.util.List; @@ -26,6 +27,15 @@ public interface ISysOssService { SysOssVo upload(MultipartFile file, Long isPublic); + /** + * 文件流式上传（避免OOM） + * + * @param file 本地文件 + * @param originalFileName 原始文件名（用于记录与后缀） + * @param isPublic 是否公开 + */ + SysOssVo upload(File file, String originalFileName, Long isPublic); + void download(Long ossId, HttpServletResponse response) throws IOException; Boolean deleteWithValidByIds(Collection ids, Boolean isValid); diff --git a/ruoyi-system/src/main/java/com/ruoyi/system/service/impl/SysOssServiceImpl.java b/ruoyi-system/src/main/java/com/ruoyi/system/service/impl/SysOssServiceImpl.java index 8ecd608..0e1ef55 100644 --- a/ruoyi-system/src/main/java/com/ruoyi/system/service/impl/SysOssServiceImpl.java +++ b/ruoyi-system/src/main/java/com/ruoyi/system/service/impl/SysOssServiceImpl.java @@ -34,6 +34,7 @@ import org.springframework.web.multipart.MultipartFile; import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.InputStream; +import java.io.File; import java.util.*; import java.util.stream.Collectors; @@ -147,6 +148,34 @@ public class SysOssServiceImpl implements ISysOssService, OssService { return this.matchingUrl(sysOssVo); } + @Override + public SysOssVo upload(File file, String originalFileName, Long isPublic) { + // 1. 解析文件后缀 + String suffix = StringUtils.substring(originalFileName, originalFileName.lastIndexOf("."), originalFileName.length()); + // 2. 获取OSS客户端 + OssClient storage = OssFactory.instance(); + UploadResult uploadResult; + try { + // 关键：用InputStream流式上传，而非加载整个文件到内存 + uploadResult = storage.uploadSuffix(file, suffix); + } catch (Exception e) { + throw new ServiceException(e.getMessage()); + } + SysOss oss = new SysOss(); + oss.setUrl(uploadResult.getUrl()); + oss.setFileSuffix(suffix); + oss.setFileName(uploadResult.getFilename()); + oss.setOriginalName(originalFileName); + oss.setService(storage.getConfigKey()); + oss.setCreateBy(LoginHelper.getNickName()); + oss.setOwnerId(LoginHelper.getUserId()); + oss.setIsPublic(isPublic == null ? 0 : isPublic); + baseMapper.insert(oss); + SysOssVo sysOssVo = new SysOssVo(); + BeanCopyUtils.copy(oss, sysOssVo); + return this.matchingUrl(sysOssVo); + } + @Override public Boolean deleteWithValidByIds(Collection ids, Boolean isValid) { if (isValid) {