DeXun/fad_oa
9
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(system): 新增大文件分片上传功能

Browse Source 
- 在ISysOssService接口中增加文件流式上传方法，支持避免OOM
- 实现SysOssServiceImpl中的文件流式上传逻辑
- 新增BigUploadController控制器处理分片上传、合并、恢复与清理
- 添加ChunkedUploadService服务类管理分片上传业务逻辑
- 创建BigUploadCleanupScheduler定时任务清理过期分片
- 提供FileMultipartFile适配器将File对象转换为MultipartFile
This commit is contained in:
Joshi
2025-12-15 17:26:21 +08:00
parent 7f5d6c1143
commit b568affb14
6 changed files with 511 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
ruoyi-admin/src/main/java/com/ruoyi/web/controller/common/BigUploadController.java Normal file
View File

@@ -0,0 +1,111 @@
package com.ruoyi.web.controller.common;
import com.ruoyi.common.core.AjaxResult;
import com.ruoyi.system.domain.vo.SysOssVo;
import com.ruoyi.web.service.ChunkedUploadService;
import lombok.RequiredArgsConstructor;
import org.springframework.http.MediaType;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import java.util.List;
@Validated
@RestController
@RequiredArgsConstructor
@RequestMapping("/common/bigUpload")
public class BigUploadController {
private final ChunkedUploadService chunkedUploadService;
/**
* 上传文件分片
*
* @param chunk 文件分片数据
* @param chunkIndex 分片索引
* @param totalChunks 总分片数
* @param fileMd5 文件MD5值
* @param fileName 文件名
* @return 上传结果
*/
@PostMapping(value = "/chunk", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public AjaxResult uploadChunk(@RequestPart("chunk") MultipartFile chunk,
@RequestParam("chunkIndex") int chunkIndex,
@RequestParam("totalChunks") int totalChunks,
@RequestParam("fileMd5") String fileMd5,
@RequestParam("fileName") String fileName) {
try {
chunkedUploadService.saveChunk(fileMd5, fileName, chunkIndex, totalChunks, chunk);
return AjaxResult.success();
} catch (Exception e) {
return AjaxResult.error(e.getMessage());
}
}
/**
* 合并文件分片
*
* @param fileMd5 文件MD5值
* @param fileName 文件名
* @param totalChunks 总分片数
* @return 合并结果及文件信息
*/
@PostMapping("/merge")
public AjaxResult merge(@RequestParam("fileMd5") String fileMd5,
@RequestParam("fileName") String fileName,
@RequestParam("totalChunks") int totalChunks) {
try {
SysOssVo oss = chunkedUploadService.mergeChunks(fileMd5, fileName, totalChunks);
return AjaxResult.success()
.put("url", oss.getUrl())
.put("fileName", oss.getOriginalName())
.put("ossId", oss.getOssId());
} catch (Exception e) {
return AjaxResult.error(e.getMessage());
}
}
/**
* 恢复上传（删除最后一个分片）
*
* @param fileMd5 文件MD5值
* @return 操作结果
*/
@PostMapping("/resume")
public AjaxResult resume(@RequestParam("fileMd5") String fileMd5) {
try {
boolean deleted = chunkedUploadService.deleteLastChunk(fileMd5);
return deleted ? AjaxResult.success() : AjaxResult.error("没有可删除的分片");
} catch (Exception e) {
return AjaxResult.error(e.getMessage());
}
}
/**
* 获取已上传的分片列表
*
* @param fileMd5 文件MD5值
* @return 已上传分片索引列表
*/
@GetMapping("/list")
public AjaxResult listUploaded(@RequestParam("fileMd5") String fileMd5) {
try {
List<Integer> indices = chunkedUploadService.listUploadedChunks(fileMd5);
return AjaxResult.success().put("uploaded", indices);
} catch (Exception e) {
return AjaxResult.error(e.getMessage());
}
}
/**
* 清理过期的临时分片目录
*
* @param olderThanHours 多少小时未更新则视为过期（默认24小时）
*/
@PostMapping("/cleanup")
public AjaxResult cleanup(@RequestParam(value = "olderThanHours", required = false, defaultValue = "24") int olderThanHours) {
long millis = olderThanHours * 60L * 60L * 1000L;
int removed = chunkedUploadService.cleanupStaleChunks(millis);
return AjaxResult.success().put("removed", removed).put("olderThanHours", olderThanHours);
}
}

30
ruoyi-admin/src/main/java/com/ruoyi/web/schedule/BigUploadCleanupScheduler.java Normal file
View File

@@ -0,0 +1,30 @@
package com.ruoyi.web.schedule;
import com.ruoyi.web.service.ChunkedUploadService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
@Slf4j
@Component
@EnableScheduling
@RequiredArgsConstructor
public class BigUploadCleanupScheduler {
private final ChunkedUploadService chunkedUploadService;
// 每天凌晨2点执行
@Scheduled(cron = "0 0 2 * * ?")
public void cleanup() {
int hours = 24; // 阈值: 24小时未更新
long millis = hours * 60L * 60L * 1000L;
try {
int removed = chunkedUploadService.cleanupStaleChunks(millis);
log.info("BigUploadCleanupScheduler removed {} stale chunk directories older than {} hours", removed, hours);
} catch (Exception e) {
log.error("BigUploadCleanupScheduler cleanup failed", e);
}
}
}

258
ruoyi-admin/src/main/java/com/ruoyi/web/service/ChunkedUploadService.java Normal file
View File

@@ -0,0 +1,258 @@
package com.ruoyi.web.service;
import com.ruoyi.common.config.RuoYiConfig;
import com.ruoyi.system.domain.vo.SysOssVo;
import com.ruoyi.system.service.ISysOssService;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantLock;
@Service
public class ChunkedUploadService {
private final ISysOssService ossService;
private final ConcurrentMap<String, ReentrantLock> mergeLocks = new ConcurrentHashMap<>();
public ChunkedUploadService(ISysOssService ossService) {
this.ossService = ossService;
}
private File getTempDir(String fileMd5) {
String base = RuoYiConfig.getUploadPath();
File dir = new File(base, "chunks" + File.separator + fileMd5);
if (!dir.exists()) {
//无检查结果方法调用忽略
dir.mkdirs();
}
return dir;
}
private File getFinalDir() {
String base = RuoYiConfig.getUploadPath();
File dir = new File(base, "big");
if (!dir.exists()) {
//无检查结果方法调用忽略
dir.mkdirs();
}
return dir;
}
public void saveChunk(String fileMd5, String fileName, int chunkIndex, int totalChunks, MultipartFile chunk) throws IOException {
File tempDir = getTempDir(fileMd5);
// 为了安全，记录总区块的元数据
File meta = new File(tempDir, ".total");
if (!meta.exists()) {
try (RandomAccessFile raf = new RandomAccessFile(meta, "rw")) {
raf.setLength(0);
raf.write(String.valueOf(totalChunks).getBytes());
}
}
File partFile = new File(tempDir, chunkIndex + ".part");
chunk.transferTo(partFile);
long now = System.currentTimeMillis();
// 更新目录与元数据的修改时间
// 无检查结果方法调用忽略
partFile.setLastModified(now);
meta.setLastModified(now);
tempDir.setLastModified(now);
}
public SysOssVo mergeChunks(String fileMd5, String fileName, int totalChunks) throws Exception {
ReentrantLock lock = mergeLocks.computeIfAbsent(fileMd5, k -> new ReentrantLock());
lock.lock();
try {
File tempDir = getTempDir(fileMd5);
if (!tempDir.exists() || !tempDir.isDirectory()) {
throw new IOException("临时分片目录不存在");
}
// 验证
List<File> parts = new ArrayList<>();
for (int i = 0; i < totalChunks; i++) {
File f = new File(tempDir, i + ".part");
if (!f.exists()) {
throw new IOException("缺少分片: " + i);
}
parts.add(f);
}
// 计算大小和偏移量
long[] sizes = new long[totalChunks];
long[] offsets = new long[totalChunks];
long totalSize = 0L;
for (int i = 0; i < totalChunks; i++) {
sizes[i] = parts.get(i).length();
offsets[i] = totalSize;
totalSize += sizes[i];
}
File finalDir = getFinalDir();
String safeName = fileMd5 + "_" + (fileName != null ? fileName : "file");
File target = new File(finalDir, safeName);
if (target.exists()) {
// 覆盖现有
if (!target.delete()) {
throw new IOException("无法覆盖已存在的目标文件");
}
}
// 预分配
try (RandomAccessFile raf = new RandomAccessFile(target, "rw")) {
raf.setLength(totalSize);
}
int threads = Math.min(8, totalChunks);
ExecutorService pool = Executors.newFixedThreadPool(threads);
List<Future<?>> futures = new ArrayList<>();
for (int i = 0; i < totalChunks; i++) {
final int idx = i;
futures.add(pool.submit(() -> {
try (RandomAccessFile writeRaf = new RandomAccessFile(target, "rw");
RandomAccessFile readRaf = new RandomAccessFile(parts.get(idx), "r");
FileChannel in = readRaf.getChannel();
FileChannel out = writeRaf.getChannel()) {
out.position(offsets[idx]);
long remaining = sizes[idx];
long pos = 0;
while (remaining > 0) {
long transferred = in.transferTo(pos, remaining, out);
if (transferred <= 0) break;
pos += transferred;
remaining -= transferred;
}
} catch (IOException e) {
throw new CompletionException(e);
}
}));
}
pool.shutdown();
for (Future<?> f : futures) {
f.get();
}
// 合并成功后的清理部件
for (File f : parts) {
// 无检查结果方法调用
f.delete();
}
// 无检查结果方法调用
new File(tempDir, ".total").delete();
// 移除空文件
File[] remain = tempDir.listFiles();
if (remain == null || remain.length == 0) {
// 无检查结果方法调用
tempDir.delete();
}
// 上传合并文件到 OSS 服务和记录（流式，避免OOM）
SysOssVo ossVo = ossService.upload(target, fileName, 0L);
// 上传完成后删除本地合并的临时文件（可选，节省磁盘）
if (target.exists()) {
// 无检查结果方法调用
target.delete();
}
return ossVo;
} finally {
try {
lock.unlock();
} finally {
mergeLocks.remove(fileMd5, lock);
}
}
}
public boolean deleteLastChunk(String fileMd5) {
File tempDir = getTempDir(fileMd5);
if (!tempDir.exists()) return false;
File[] files = tempDir.listFiles((dir, name) -> name.endsWith(".part"));
if (files == null || files.length == 0) return false;
Arrays.sort(files, Comparator.comparingInt(f -> Integer.parseInt(f.getName().replace(".part", ""))));
File last = files[files.length - 1];
return last.delete();
}
public List<Integer> listUploadedChunks(String fileMd5) throws IOException {
File tempDir = getTempDir(fileMd5);
if (!tempDir.exists()) {
return new ArrayList<>();
}
File[] parts = tempDir.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
return name.endsWith(".part");
}
});
List<Integer> result = new ArrayList<>();
if (parts != null) {
for (File f : parts) {
String n = f.getName().replace(".part", "");
try {
result.add(Integer.parseInt(n));
} catch (NumberFormatException ignored) {}
}
}
result.sort(Integer::compareTo);
return result;
}
public int cleanupStaleChunks(long olderThanMillis) {
int deleted = 0;
File base = new File(RuoYiConfig.getUploadPath(), "chunks");
if (!base.exists() || !base.isDirectory()) {
return 0;
}
File[] dirs = base.listFiles(File::isDirectory);
if (dirs == null) return 0;
long now = System.currentTimeMillis();
for (File dir : dirs) {
String key = dir.getName();
ReentrantLock lock = mergeLocks.computeIfAbsent(key, k -> new ReentrantLock());
if (!lock.tryLock()) {
continue;
}
try {
long lastMod = dir.lastModified();
File[] files = dir.listFiles();
if (files != null) {
for (File f : files) {
lastMod = Math.max(lastMod, f.lastModified());
}
}
if (now - lastMod >= olderThanMillis) {
deleteDirectory(dir);
deleted++;
}
} finally {
try {
lock.unlock();
} finally {
mergeLocks.remove(key, lock);
}
}
}
return deleted;
}
private void deleteDirectory(File dir) {
File[] files = dir.listFiles();
if (files != null) {
for (File f : files) {
if (f.isDirectory()) {
deleteDirectory(f);
} else {
// 无检查结果方法调用忽略
f.delete();
}
}
}
// 无检查结果方法调用忽略
dir.delete();
}
}

73
ruoyi-admin/src/main/java/com/ruoyi/web/service/support/FileMultipartFile.java Normal file
View File

@@ -0,0 +1,73 @@
package com.ruoyi.web.service.support;
import org.springframework.web.multipart.MultipartFile;
import java.io.*;
import java.io.ByteArrayOutputStream;
public class FileMultipartFile implements MultipartFile {
private final File file;
private final String originalFilename;
private final String contentType;
public FileMultipartFile(File file, String originalFilename, String contentType) {
this.file = file;
this.originalFilename = originalFilename;
this.contentType = contentType;
}
@Override
public String getName() {
return originalFilename;
}
@Override
public String getOriginalFilename() {
return originalFilename;
}
@Override
public String getContentType() {
return contentType;
}
@Override
public boolean isEmpty() {
return file.length() == 0;
}
@Override
public long getSize() {
return file.length();
}
@Override
public byte[] getBytes() throws IOException {
try (InputStream is = new FileInputStream(file)) {
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
byte[] data = new byte[8192];
int nRead;
while ((nRead = is.read(data, 0, data.length)) != -1) {
buffer.write(data, 0, nRead);
}
return buffer.toByteArray();
}
}
@Override
public InputStream getInputStream() throws IOException {
return new FileInputStream(file);
}
@Override
public void transferTo(File dest) throws IOException {
try (InputStream in = new FileInputStream(file);
OutputStream out = new FileOutputStream(dest)) {
byte[] buffer = new byte[8192];
int bytesRead;
while ((bytesRead = in.read(buffer)) != -1) {
out.write(buffer, 0, bytesRead);
}
}
}
}

10
ruoyi-system/src/main/java/com/ruoyi/system/service/ISysOssService.java
View File

@@ -8,6 +8,7 @@ import org.springframework.web.multipart.MultipartFile;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.File;
import java.util.Collection;
import java.util.List;
@@ -26,6 +27,15 @@ public interface ISysOssService {
SysOssVo upload(MultipartFile file, Long isPublic);
/**
* 文件流式上传（避免OOM）
*
* @param file 本地文件
* @param originalFileName 原始文件名（用于记录与后缀）
* @param isPublic 是否公开
*/
SysOssVo upload(File file, String originalFileName, Long isPublic);
void download(Long ossId, HttpServletResponse response) throws IOException;
Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid);

29
ruoyi-system/src/main/java/com/ruoyi/system/service/impl/SysOssServiceImpl.java
View File

@@ -34,6 +34,7 @@ import org.springframework.web.multipart.MultipartFile;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.InputStream;
import java.io.File;
import java.util.*;
import java.util.stream.Collectors;
@@ -147,6 +148,34 @@ public class SysOssServiceImpl implements ISysOssService, OssService {
return this.matchingUrl(sysOssVo);
}
@Override
public SysOssVo upload(File file, String originalFileName, Long isPublic) {
// 1. 解析文件后缀
String suffix = StringUtils.substring(originalFileName, originalFileName.lastIndexOf("."), originalFileName.length());
// 2. 获取OSS客户端
OssClient storage = OssFactory.instance();
UploadResult uploadResult;
try {
// 关键：用InputStream流式上传，而非加载整个文件到内存
uploadResult = storage.uploadSuffix(file, suffix);
} catch (Exception e) {
throw new ServiceException(e.getMessage());
}
SysOss oss = new SysOss();
oss.setUrl(uploadResult.getUrl());
oss.setFileSuffix(suffix);
oss.setFileName(uploadResult.getFilename());
oss.setOriginalName(originalFileName);
oss.setService(storage.getConfigKey());
oss.setCreateBy(LoginHelper.getNickName());
oss.setOwnerId(LoginHelper.getUserId());
oss.setIsPublic(isPublic == null ? 0 : isPublic);
baseMapper.insert(oss);
SysOssVo sysOssVo = new SysOssVo();
BeanCopyUtils.copy(oss, sysOssVo);
return this.matchingUrl(sysOssVo);
}
@Override
public Boolean deleteWithValidByIds(Collection<Long> ids, Boolean isValid) {
if (isValid) {