Joshi
99a8a943bc
- 新增 alarmBatchBo 类用于批量处理报警记录 - 移除报警记录控制器中的权限注解 - 批量处理接口改为接收 alarmBatchBo 对象 - 引入 ruoyi-quartz 依赖用于定时任务处理- 恢复并优化 InspectionTaskServiceImpl 中设备信息设置 - 更新任务执行时更新最后执行时间与下次执行时间- 视频分析服务中增加报警记录时更新任务报警次数
462 lines
20 KiB
Java
462 lines
20 KiB
Java
package com.ruoyi.video.service;
|
||
|
||
import com.ruoyi.common.utils.spring.SpringUtils;
|
||
import com.ruoyi.framework.service.MinioService;
|
||
import com.ruoyi.video.domain.AlarmRecord;
|
||
import com.ruoyi.video.domain.Detection;
|
||
import com.ruoyi.video.domain.InspectionTask;
|
||
import com.ruoyi.video.domain.VMinioObject;
|
||
import com.ruoyi.video.mapper.AlarmRecordMapper;
|
||
import com.ruoyi.video.mapper.InspectionTaskMapper;
|
||
import com.ruoyi.video.thread.detector.HttpYoloDetector;
|
||
import com.ruoyi.video.utils.Overlay;
|
||
import com.ruoyi.video.utils.CustomMultipartFile;
|
||
|
||
import lombok.extern.slf4j.Slf4j;
|
||
|
||
import org.bytedeco.ffmpeg.avcodec.AVPacket;
|
||
import org.bytedeco.ffmpeg.global.avcodec;
|
||
import org.bytedeco.ffmpeg.global.avutil;
|
||
import org.bytedeco.javacv.FFmpegFrameGrabber;
|
||
import org.bytedeco.javacv.FFmpegFrameRecorder;
|
||
import org.bytedeco.javacv.Frame;
|
||
import org.bytedeco.javacv.OpenCVFrameConverter;
|
||
import org.bytedeco.opencv.opencv_core.Mat;
|
||
import org.bytedeco.opencv.opencv_core.Point;
|
||
import org.bytedeco.opencv.opencv_core.Rect;
|
||
import org.bytedeco.opencv.opencv_core.Scalar;
|
||
import org.springframework.beans.factory.annotation.Autowired;
|
||
import org.springframework.scheduling.annotation.Async;
|
||
import org.springframework.stereotype.Service;
|
||
import org.springframework.util.CollectionUtils;
|
||
|
||
import java.io.File;
|
||
import java.io.FileOutputStream;
|
||
import java.io.InputStream;
|
||
import java.nio.IntBuffer;
|
||
import java.util.ArrayList;
|
||
import java.util.Date;
|
||
import java.util.HashMap;
|
||
import java.util.List;
|
||
import java.util.Map;
|
||
import java.util.UUID;
|
||
|
||
import static org.bytedeco.opencv.global.opencv_imgproc.*;
|
||
|
||
/**
|
||
* 视频分析服务 - 离线分析视频并处理结果
|
||
*/
|
||
@Slf4j
|
||
@Service
|
||
public class VideoAnalysisService {
|
||
|
||
@Autowired
|
||
private MinioService minioService;
|
||
|
||
@Autowired
|
||
private IVMinioObjectService vMinioObjectService;
|
||
|
||
@Autowired
|
||
private InspectionTaskMapper taskMapper;
|
||
|
||
@Autowired
|
||
private AlarmRecordMapper alarmRecordMapper;
|
||
|
||
@Autowired
|
||
private com.ruoyi.video.mapper.InspectionTaskRecordMapper inspectionTaskRecordMapper;
|
||
@Autowired
|
||
private InspectionTaskMapper inspectionTaskMapper;
|
||
|
||
// 检测器配置 - 支持环境变量配置
|
||
private static final String PYTHON_API_URL = System.getenv().getOrDefault("PYTHON_API_URL", "http://localhost:8000") + "/api/detect/file";
|
||
private static final String MODEL_NAME = "smoke"; // 默认使用吸烟检测模型
|
||
|
||
/**
|
||
* 分析视频并更新记录（同步调用）
|
||
* @param task 巡检任务
|
||
* @param videoFile 视频文件
|
||
*/
|
||
public void analyzeVideoWithRecord(InspectionTask task, com.ruoyi.video.domain.InspectionTaskRecord record, File videoFile) {
|
||
log.info("开始分析视频并更新记录: 任务ID={}, 记录ID={}", task.getTaskId(), record.getRecordId());
|
||
|
||
try {
|
||
// 创建输出视频文件
|
||
File outputVideoFile = File.createTempFile("analysis_output_", ".mp4");
|
||
|
||
// 创建检测器
|
||
String chosenModel = (task.getModelName() != null && !task.getModelName().trim().isEmpty())
|
||
? task.getModelName().trim()
|
||
: MODEL_NAME;
|
||
log.info("使用模型进行分析: {} (taskId={})", chosenModel, task.getTaskId());
|
||
HttpYoloDetector detector = new HttpYoloDetector("yolov8", PYTHON_API_URL, chosenModel, 0x00FF00);
|
||
|
||
// 处理视频并记录检测结果
|
||
String detectionResult = processVideoWithRecord(videoFile, outputVideoFile, detector, task, record);
|
||
|
||
// 更新记录的识别结果
|
||
record.setResult(detectionResult);
|
||
inspectionTaskRecordMapper.updateInspectionTaskRecord(record);
|
||
|
||
// 清理临时输出文件
|
||
if (outputVideoFile.exists()) {
|
||
outputVideoFile.delete();
|
||
}
|
||
|
||
log.info("视频分析完成: 任务ID={}, 记录ID={}, 检测结果={}", task.getTaskId(), record.getRecordId(), detectionResult);
|
||
} catch (Exception e) {
|
||
log.error("视频分析失败: 任务ID={}, 记录ID={}, 错误={}", task.getTaskId(), record.getRecordId(), e.getMessage(), e);
|
||
// 更新记录为部分成功
|
||
record.setStatus(2);
|
||
record.setResult("分析失败: " + e.getMessage());
|
||
inspectionTaskRecordMapper.updateInspectionTaskRecord(record);
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 处理视频并记录检测结果
|
||
* @return 检测结果摘要
|
||
*/
|
||
private String processVideoWithRecord(File inputFile, File outputFile, HttpYoloDetector detector,
|
||
InspectionTask task, com.ruoyi.video.domain.InspectionTaskRecord record) throws Exception {
|
||
FFmpegFrameGrabber grabber = new FFmpegFrameGrabber(inputFile);
|
||
grabber.start();
|
||
|
||
FFmpegFrameRecorder recorder = new FFmpegFrameRecorder(outputFile,
|
||
grabber.getImageWidth(), grabber.getImageHeight(), grabber.getAudioChannels());
|
||
|
||
recorder.setFormat("mp4");
|
||
recorder.setVideoCodec(avcodec.AV_CODEC_ID_H264);
|
||
recorder.setFrameRate(grabber.getFrameRate());
|
||
recorder.setVideoBitrate(grabber.getVideoBitrate());
|
||
if (grabber.getAudioChannels() > 0) {
|
||
recorder.setAudioCodec(avcodec.AV_CODEC_ID_AAC);
|
||
recorder.setAudioBitrate(grabber.getAudioBitrate());
|
||
recorder.setAudioChannels(grabber.getAudioChannels());
|
||
recorder.setSampleRate(grabber.getSampleRate());
|
||
}
|
||
recorder.start();
|
||
|
||
OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat();
|
||
|
||
// 用于去重的垃圾检测结果缓存
|
||
Map<String, Long> detectedGarbageCache = new HashMap<>();
|
||
Map<String, Integer> detectionCounts = new HashMap<>(); // 统计每种类别的数量
|
||
|
||
// 跟踪检测到的垃圾ID
|
||
final Long[] detectionId = {1L};
|
||
// 帧计数
|
||
long frameCount = 0;
|
||
int totalDetections = 0;
|
||
|
||
Frame frame;
|
||
while ((frame = grabber.grab()) != null) {
|
||
frameCount++;
|
||
|
||
if (frame.image != null) {
|
||
// 处理视频帧
|
||
Mat mat = converter.convert(frame);
|
||
|
||
if (mat != null && !mat.isNull()) {
|
||
// 每10帧执行一次检测，减少API调用频率
|
||
if (frameCount % 10 == 0) {
|
||
List<Detection> detections = detector.detect(mat);
|
||
|
||
if (!CollectionUtils.isEmpty(detections)) {
|
||
for (Detection detection : detections) {
|
||
// 检查是否为新的垃圾检测结果
|
||
String detectionKey = generateDetectionKey(detection);
|
||
if (!detectedGarbageCache.containsKey(detectionKey)) {
|
||
// 这是新检测到的垃圾
|
||
detectedGarbageCache.put(detectionKey, detectionId[0]++);
|
||
totalDetections++;
|
||
|
||
// 统计类别数量
|
||
String label = detection.getLabel();
|
||
detectionCounts.put(label, detectionCounts.getOrDefault(label, 0) + 1);
|
||
|
||
// 创建告警记录（不重复）
|
||
createAlarmRecordForRecord(task, record, detection, mat, frameCount);
|
||
} else {
|
||
// 更新上次检测时间
|
||
detectedGarbageCache.put(detectionKey, detectionId[0]++);
|
||
}
|
||
}
|
||
|
||
// 清理超过60秒未检测到的垃圾（假设30fps）
|
||
Long currentId = detectionId[0];
|
||
detectedGarbageCache.entrySet().removeIf(entry ->
|
||
(currentId - entry.getValue()) > grabber.getFrameRate() * 60);
|
||
}
|
||
}
|
||
|
||
// 转回Frame并写入录像
|
||
Frame processedFrame = converter.convert(mat);
|
||
recorder.record(processedFrame);
|
||
} else {
|
||
// 原样写入
|
||
recorder.record(frame);
|
||
}
|
||
} else if (frame.samples != null) {
|
||
// 音频帧原样写入
|
||
recorder.record(frame);
|
||
}
|
||
}
|
||
|
||
recorder.stop();
|
||
recorder.close();
|
||
grabber.stop();
|
||
grabber.close();
|
||
|
||
// 上传处理后的视频到MinIO
|
||
uploadProcessedVideoForRecord(outputFile, task, record);
|
||
|
||
// 生成检测结果摘要
|
||
StringBuilder resultSummary = new StringBuilder();
|
||
resultSummary.append("共检测到 ").append(totalDetections).append(" 个问题");
|
||
if (!detectionCounts.isEmpty()) {
|
||
resultSummary.append("，详情：");
|
||
detectionCounts.forEach((label, count) ->
|
||
resultSummary.append(label).append("(").append(count).append(") "));
|
||
}
|
||
|
||
return resultSummary.toString();
|
||
}
|
||
|
||
/**
|
||
* 为记录创建告警
|
||
*/
|
||
private void createAlarmRecordForRecord(InspectionTask task, com.ruoyi.video.domain.InspectionTaskRecord record,
|
||
Detection detection, Mat frame, long frameCount) throws Exception {
|
||
//创建记录之前应该给task加上报警次数
|
||
task.setAlarmCount(task.getAlarmCount() + 1);
|
||
inspectionTaskMapper.updateInspectionTask(task);
|
||
// 创建告警图像临时文件
|
||
File alarmImageFile = File.createTempFile("alarm_", ".jpg");
|
||
|
||
// 裁剪检测区域，略微扩大区域
|
||
Rect rect = detection.getRect();
|
||
int x = Math.max(0, rect.x() - 10);
|
||
int y = Math.max(0, rect.y() - 10);
|
||
int w = Math.min(frame.cols() - x, rect.width() + 20);
|
||
int h = Math.min(frame.rows() - y, rect.height() + 20);
|
||
|
||
// 使用OpenCV保存告警图片
|
||
Mat roi = new Mat(frame, new Rect(x, y, w, h));
|
||
org.bytedeco.opencv.global.opencv_imgcodecs.imwrite(alarmImageFile.getAbsolutePath(), roi);
|
||
|
||
// 上传告警图片到MinIO
|
||
String fileName = "alarm_" + System.currentTimeMillis() + ".jpg";
|
||
String bucketName = "alarm-images";
|
||
|
||
CustomMultipartFile multipartFile = new CustomMultipartFile(alarmImageFile, fileName, "image/jpeg");
|
||
String imagePath = minioService.putObject(bucketName, fileName, multipartFile.getInputStream());
|
||
|
||
// 创建告警记录（严格按照DDL）
|
||
AlarmRecord alarmRecord = new AlarmRecord();
|
||
alarmRecord.setTaskId(task.getTaskId());
|
||
alarmRecord.setTaskName(task.getTaskName());
|
||
alarmRecord.setDeviceId(task.getDeviceId());
|
||
alarmRecord.setDeviceName(task.getDeviceName());
|
||
alarmRecord.setAlarmType("detection");
|
||
alarmRecord.setAlarmLevel("2"); // 2=中级
|
||
alarmRecord.setAlarmDesc(detection.getLabel() + " - 置信度: " + String.format("%.2f", detection.getConfidence()));
|
||
alarmRecord.setConfidence(java.math.BigDecimal.valueOf(detection.getConfidence()));
|
||
alarmRecord.setImagePath(imagePath);
|
||
alarmRecord.setAlarmTime(new Date());
|
||
alarmRecord.setHandleStatus("0"); // 0=未处理
|
||
alarmRecord.setCreateBy("system");
|
||
alarmRecord.setCreateTime(new Date());
|
||
|
||
alarmRecordMapper.insertAlarmRecord(alarmRecord);
|
||
|
||
log.info("创建告警记录: 类型={}, 任务ID={}, 记录ID={}, 告警ID={}",
|
||
detection.getLabel(), task.getTaskId(), record.getRecordId(), alarmRecord.getAlarmId());
|
||
|
||
// 删除临时文件
|
||
if (alarmImageFile.exists()) {
|
||
alarmImageFile.delete();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 上传处理后的视频（针对记录）
|
||
*/
|
||
private void uploadProcessedVideoForRecord(File videoFile, InspectionTask task,
|
||
com.ruoyi.video.domain.InspectionTaskRecord record) throws Exception {
|
||
String fileName = "processed_" + System.currentTimeMillis() + ".mp4";
|
||
String bucketName = "inspection-videos";
|
||
|
||
CustomMultipartFile multipartFile = new CustomMultipartFile(videoFile, fileName, "video/mp4");
|
||
String objectUrl = minioService.putObject(bucketName, fileName, multipartFile.getInputStream());
|
||
|
||
VMinioObject minioObject = new VMinioObject();
|
||
minioObject.setBucketName(bucketName);
|
||
minioObject.setObjectName(fileName);
|
||
minioObject.setUrl(objectUrl);
|
||
minioObject.setCreateBy("system");
|
||
minioObject.setCreateTime(new Date());
|
||
Long objectId = vMinioObjectService.insertVMinioObject(minioObject);
|
||
|
||
// 更新记录的附件信息，添加处理后的视频URL
|
||
String currentAccessory = record.getAccessory();
|
||
if (currentAccessory == null || currentAccessory.isEmpty()) {
|
||
record.setAccessory(objectUrl);
|
||
} else {
|
||
record.setAccessory(currentAccessory + "," + objectUrl);
|
||
}
|
||
inspectionTaskRecordMapper.updateInspectionTaskRecord(record);
|
||
|
||
log.info("处理后视频已上传: 任务ID={}, 记录ID={}, MinIO对象ID={}, URL={}",
|
||
task.getTaskId(), record.getRecordId(), objectId, objectUrl);
|
||
}
|
||
|
||
|
||
|
||
/**
|
||
* 处理视频记录
|
||
* @param inputFile 输入视频文件
|
||
* @param outputFile 输出视频文件
|
||
* @param detector 检测器
|
||
* @param task 巡检任务
|
||
* @param record 巡检记录
|
||
*/
|
||
private void processVideoForRecord(File inputFile, File outputFile, HttpYoloDetector detector,
|
||
InspectionTask task, com.ruoyi.video.domain.InspectionTaskRecord record) throws Exception {
|
||
FFmpegFrameGrabber grabber = new FFmpegFrameGrabber(inputFile);
|
||
grabber.start();
|
||
|
||
FFmpegFrameRecorder recorder = new FFmpegFrameRecorder(outputFile,
|
||
grabber.getImageWidth(), grabber.getImageHeight(), grabber.getAudioChannels());
|
||
|
||
recorder.setFormat("mp4");
|
||
recorder.setVideoCodec(avcodec.AV_CODEC_ID_H264);
|
||
recorder.setFrameRate(grabber.getFrameRate());
|
||
recorder.setVideoBitrate(grabber.getVideoBitrate());
|
||
if (grabber.getAudioChannels() > 0) {
|
||
recorder.setAudioCodec(avcodec.AV_CODEC_ID_AAC);
|
||
recorder.setAudioBitrate(grabber.getAudioBitrate());
|
||
recorder.setAudioChannels(grabber.getAudioChannels());
|
||
recorder.setSampleRate(grabber.getSampleRate());
|
||
}
|
||
recorder.start();
|
||
|
||
OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat();
|
||
|
||
// 用于去重的垃圾检测结果缓存
|
||
Map<String, Long> detectedGarbageCache = new HashMap<>();
|
||
// 跟踪检测到的垃圾ID
|
||
final Long[] detectionId = {1L};
|
||
// 帧计数
|
||
long frameCount = 0;
|
||
|
||
Frame frame;
|
||
while ((frame = grabber.grab()) != null) {
|
||
frameCount++;
|
||
|
||
if (frame.image != null) {
|
||
// 处理视频帧
|
||
Mat mat = converter.convert(frame);
|
||
|
||
if (mat != null && !mat.isNull()) {
|
||
// 每10帧执行一次检测，减少API调用频率
|
||
if (frameCount % 10 == 0) {
|
||
List<Detection> detections = detector.detect(mat);
|
||
|
||
if (!CollectionUtils.isEmpty(detections)) {
|
||
// 绘制检测框
|
||
for (Detection detection : detections) {
|
||
drawDetection(mat, detection);
|
||
|
||
// 检查是否为新的垃圾检测结果
|
||
String detectionKey = generateDetectionKey(detection);
|
||
if (!detectedGarbageCache.containsKey(detectionKey)) {
|
||
// 这是新检测到的垃圾
|
||
detectedGarbageCache.put(detectionKey, detectionId[0]++);
|
||
|
||
// 创建告警记录（关联到记录）
|
||
createAlarmRecordForRecord(task, record, detection, mat, frameCount);
|
||
} else {
|
||
// 更新上次检测时间
|
||
detectedGarbageCache.put(detectionKey, detectionId[0]++);
|
||
}
|
||
}
|
||
|
||
// 清理超过60秒未检测到的垃圾（假设30fps）
|
||
Long currentId = detectionId[0];
|
||
detectedGarbageCache.entrySet().removeIf(entry ->
|
||
(currentId - entry.getValue()) > grabber.getFrameRate() * 60);
|
||
}
|
||
}
|
||
|
||
// 转回Frame并写入录像
|
||
Frame processedFrame = converter.convert(mat);
|
||
recorder.record(processedFrame);
|
||
} else {
|
||
// 原样写入
|
||
recorder.record(frame);
|
||
}
|
||
} else if (frame.samples != null) {
|
||
// 音频帧原样写入
|
||
recorder.record(frame);
|
||
}
|
||
}
|
||
|
||
recorder.stop();
|
||
recorder.close();
|
||
grabber.stop();
|
||
grabber.close();
|
||
|
||
// 上传处理后的视频（更新记录）
|
||
uploadProcessedVideoForRecord(outputFile, task, record);
|
||
}
|
||
|
||
|
||
|
||
/**
|
||
* 生成检测结果的唯一键，用于检测结果去重
|
||
*/
|
||
private String generateDetectionKey(Detection detection) {
|
||
// 使用检测框的位置和大小来生成键
|
||
// 允许小范围波动，认为是同一个物体
|
||
Rect rect = detection.getRect();
|
||
int x = rect.x() / 10 * 10; // 取10的倍数，允许小波动
|
||
int y = rect.y() / 10 * 10;
|
||
int w = rect.width() / 10 * 10;
|
||
int h = rect.height() / 10 * 10;
|
||
return String.format("%s_%d_%d_%d_%d", detection.getLabel(), x, y, w, h);
|
||
}
|
||
|
||
|
||
|
||
/**
|
||
* 在图像上绘制检测框
|
||
*/
|
||
private void drawDetection(Mat frame, Detection detection) {
|
||
// 绘制边界框
|
||
Rect rect = detection.getRect();
|
||
Scalar color = new Scalar(detection.getColorBGR());
|
||
int thickness = 2;
|
||
|
||
rectangle(frame, rect, color, thickness, LINE_8, 0);
|
||
|
||
// 绘制标签背景
|
||
String label = String.format("%s: %.2f", detection.getLabel(), detection.getConfidence());
|
||
Point textPosition = new Point(rect.x(), rect.y() - 10);
|
||
|
||
// 获取文本大小 - 添加缺失的IntBuffer参数
|
||
IntBuffer baseline = IntBuffer.allocate(1);
|
||
org.bytedeco.opencv.opencv_core.Size textSize = getTextSize(
|
||
label, FONT_HERSHEY_SIMPLEX, 0.5, thickness, baseline);
|
||
|
||
// 绘制标签背景矩形
|
||
rectangle(frame,
|
||
new Rect(rect.x(), rect.y() - textSize.height() - 10,
|
||
textSize.width(), textSize.height() + 10),
|
||
color, FILLED, LINE_8, 0);
|
||
|
||
// 绘制文本
|
||
putText(frame, label, textPosition,
|
||
FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 0),
|
||
1, LINE_AA, false);
|
||
}
|
||
} |