package com.ruoyi.video.service; import com.ruoyi.common.utils.spring.SpringUtils; import com.ruoyi.framework.service.MinioService; import com.ruoyi.video.domain.AlarmRecord; import com.ruoyi.video.domain.Detection; import com.ruoyi.video.domain.InspectionTask; import com.ruoyi.video.domain.VMinioObject; import com.ruoyi.video.mapper.AlarmRecordMapper; import com.ruoyi.video.mapper.InspectionTaskMapper; import com.ruoyi.video.thread.detector.HttpYoloDetector; import com.ruoyi.video.utils.Overlay; import com.ruoyi.video.utils.CustomMultipartFile; import lombok.extern.slf4j.Slf4j; import org.bytedeco.ffmpeg.avcodec.AVPacket; import org.bytedeco.ffmpeg.global.avcodec; import org.bytedeco.ffmpeg.global.avutil; import org.bytedeco.javacv.FFmpegFrameGrabber; import org.bytedeco.javacv.FFmpegFrameRecorder; import org.bytedeco.javacv.Frame; import org.bytedeco.javacv.OpenCVFrameConverter; import org.bytedeco.opencv.opencv_core.Mat; import org.bytedeco.opencv.opencv_core.Point; import org.bytedeco.opencv.opencv_core.Rect; import org.bytedeco.opencv.opencv_core.Scalar; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; import java.io.File; import java.io.FileOutputStream; import java.io.InputStream; import java.nio.IntBuffer; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import static org.bytedeco.opencv.global.opencv_imgproc.*; /** * 视频分析服务 - 离线分析视频并处理结果 */ @Slf4j @Service public class VideoAnalysisService { @Autowired private MinioService minioService; @Autowired private IVMinioObjectService vMinioObjectService; @Autowired private InspectionTaskMapper taskMapper; @Autowired private AlarmRecordMapper alarmRecordMapper; @Autowired private com.ruoyi.video.mapper.InspectionTaskRecordMapper inspectionTaskRecordMapper; @Autowired private InspectionTaskMapper inspectionTaskMapper; // 检测器配置 - 支持环境变量配置 private static final String PYTHON_API_URL = System.getenv().getOrDefault("PYTHON_API_URL", "http://localhost:8000") + "/api/detect/file"; private static final String MODEL_NAME = "smoke"; // 默认使用吸烟检测模型 /** * 分析视频并更新记录（同步调用） * @param task 巡检任务 * @param videoFile 视频文件 */ public void analyzeVideoWithRecord(InspectionTask task, com.ruoyi.video.domain.InspectionTaskRecord record, File videoFile) { log.info("开始分析视频并更新记录: 任务ID={}, 记录ID={}", task.getTaskId(), record.getRecordId()); try { // 创建输出视频文件 File outputVideoFile = File.createTempFile("analysis_output_", ".mp4"); // 创建检测器 String chosenModel = (task.getModelName() != null && !task.getModelName().trim().isEmpty()) ? task.getModelName().trim() : MODEL_NAME; log.info("使用模型进行分析: {} (taskId={})", chosenModel, task.getTaskId()); HttpYoloDetector detector = new HttpYoloDetector("yolov8", PYTHON_API_URL, chosenModel, 0x00FF00); // 处理视频并记录检测结果 String detectionResult = processVideoWithRecord(videoFile, outputVideoFile, detector, task, record); // 更新记录的识别结果 record.setResult(detectionResult); inspectionTaskRecordMapper.updateInspectionTaskRecord(record); // 清理临时输出文件 if (outputVideoFile.exists()) { outputVideoFile.delete(); } log.info("视频分析完成: 任务ID={}, 记录ID={}, 检测结果={}", task.getTaskId(), record.getRecordId(), detectionResult); } catch (Exception e) { log.error("视频分析失败: 任务ID={}, 记录ID={}, 错误={}", task.getTaskId(), record.getRecordId(), e.getMessage(), e); // 更新记录为部分成功 record.setStatus(2); record.setResult("分析失败: " + e.getMessage()); inspectionTaskRecordMapper.updateInspectionTaskRecord(record); } } /** * 处理视频并记录检测结果 * @return 检测结果摘要 */ private String processVideoWithRecord(File inputFile, File outputFile, HttpYoloDetector detector, InspectionTask task, com.ruoyi.video.domain.InspectionTaskRecord record) throws Exception { FFmpegFrameGrabber grabber = new FFmpegFrameGrabber(inputFile); grabber.start(); FFmpegFrameRecorder recorder = new FFmpegFrameRecorder(outputFile, grabber.getImageWidth(), grabber.getImageHeight(), grabber.getAudioChannels()); recorder.setFormat("mp4"); recorder.setVideoCodec(avcodec.AV_CODEC_ID_H264); recorder.setFrameRate(grabber.getFrameRate()); recorder.setVideoBitrate(grabber.getVideoBitrate()); if (grabber.getAudioChannels() > 0) { recorder.setAudioCodec(avcodec.AV_CODEC_ID_AAC); recorder.setAudioBitrate(grabber.getAudioBitrate()); recorder.setAudioChannels(grabber.getAudioChannels()); recorder.setSampleRate(grabber.getSampleRate()); } recorder.start(); OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat(); // 用于去重的垃圾检测结果缓存 Map detectedGarbageCache = new HashMap<>(); Map detectionCounts = new HashMap<>(); // 统计每种类别的数量 // 跟踪检测到的垃圾ID final Long[] detectionId = {1L}; // 帧计数 long frameCount = 0; int totalDetections = 0; Frame frame; while ((frame = grabber.grab()) != null) { frameCount++; if (frame.image != null) { // 处理视频帧 Mat mat = converter.convert(frame); if (mat != null && !mat.isNull()) { // 每10帧执行一次检测，减少API调用频率 if (frameCount % 10 == 0) { List detections = detector.detect(mat); if (!CollectionUtils.isEmpty(detections)) { for (Detection detection : detections) { // 检查是否为新的垃圾检测结果 String detectionKey = generateDetectionKey(detection); if (!detectedGarbageCache.containsKey(detectionKey)) { // 这是新检测到的垃圾 detectedGarbageCache.put(detectionKey, detectionId[0]++); totalDetections++; // 统计类别数量 String label = detection.getLabel(); detectionCounts.put(label, detectionCounts.getOrDefault(label, 0) + 1); // 创建告警记录（不重复） createAlarmRecordForRecord(task, record, detection, mat, frameCount); } else { // 更新上次检测时间 detectedGarbageCache.put(detectionKey, detectionId[0]++); } } // 清理超过60秒未检测到的垃圾（假设30fps） Long currentId = detectionId[0]; detectedGarbageCache.entrySet().removeIf(entry -> (currentId - entry.getValue()) > grabber.getFrameRate() * 60); } } // 转回Frame并写入录像 Frame processedFrame = converter.convert(mat); recorder.record(processedFrame); } else { // 原样写入 recorder.record(frame); } } else if (frame.samples != null) { // 音频帧原样写入 recorder.record(frame); } } recorder.stop(); recorder.close(); grabber.stop(); grabber.close(); // 上传处理后的视频到MinIO uploadProcessedVideoForRecord(outputFile, task, record); // 生成检测结果摘要 StringBuilder resultSummary = new StringBuilder(); resultSummary.append("共检测到 ").append(totalDetections).append(" 个问题"); if (!detectionCounts.isEmpty()) { resultSummary.append("，详情："); detectionCounts.forEach((label, count) -> resultSummary.append(label).append("(").append(count).append(") ")); } return resultSummary.toString(); } /** * 为记录创建告警 */ private void createAlarmRecordForRecord(InspectionTask task, com.ruoyi.video.domain.InspectionTaskRecord record, Detection detection, Mat frame, long frameCount) throws Exception { //创建记录之前应该给task加上报警次数 task.setAlarmCount(task.getAlarmCount() + 1); inspectionTaskMapper.updateInspectionTask(task); // 创建告警图像临时文件 File alarmImageFile = File.createTempFile("alarm_", ".jpg"); // 裁剪检测区域，略微扩大区域 Rect rect = detection.getRect(); int x = Math.max(0, rect.x() - 10); int y = Math.max(0, rect.y() - 10); int w = Math.min(frame.cols() - x, rect.width() + 20); int h = Math.min(frame.rows() - y, rect.height() + 20); // 使用OpenCV保存告警图片 Mat roi = new Mat(frame, new Rect(x, y, w, h)); org.bytedeco.opencv.global.opencv_imgcodecs.imwrite(alarmImageFile.getAbsolutePath(), roi); // 上传告警图片到MinIO String fileName = "alarm_" + System.currentTimeMillis() + ".jpg"; String bucketName = "alarm-images"; CustomMultipartFile multipartFile = new CustomMultipartFile(alarmImageFile, fileName, "image/jpeg"); String imagePath = minioService.putObject(bucketName, fileName, multipartFile.getInputStream()); // 创建告警记录（严格按照DDL） AlarmRecord alarmRecord = new AlarmRecord(); alarmRecord.setTaskId(task.getTaskId()); alarmRecord.setTaskName(task.getTaskName()); alarmRecord.setDeviceId(task.getDeviceId()); alarmRecord.setDeviceName(task.getDeviceName()); alarmRecord.setAlarmType("detection"); alarmRecord.setAlarmLevel("2"); // 2=中级 alarmRecord.setAlarmDesc(detection.getLabel() + " - 置信度: " + String.format("%.2f", detection.getConfidence())); alarmRecord.setConfidence(java.math.BigDecimal.valueOf(detection.getConfidence())); alarmRecord.setImagePath(imagePath); alarmRecord.setAlarmTime(new Date()); alarmRecord.setHandleStatus("0"); // 0=未处理 alarmRecord.setCreateBy("system"); alarmRecord.setCreateTime(new Date()); alarmRecordMapper.insertAlarmRecord(alarmRecord); log.info("创建告警记录: 类型={}, 任务ID={}, 记录ID={}, 告警ID={}", detection.getLabel(), task.getTaskId(), record.getRecordId(), alarmRecord.getAlarmId()); // 删除临时文件 if (alarmImageFile.exists()) { alarmImageFile.delete(); } } /** * 上传处理后的视频（针对记录） */ private void uploadProcessedVideoForRecord(File videoFile, InspectionTask task, com.ruoyi.video.domain.InspectionTaskRecord record) throws Exception { String fileName = "processed_" + System.currentTimeMillis() + ".mp4"; String bucketName = "inspection-videos"; CustomMultipartFile multipartFile = new CustomMultipartFile(videoFile, fileName, "video/mp4"); String objectUrl = minioService.putObject(bucketName, fileName, multipartFile.getInputStream()); VMinioObject minioObject = new VMinioObject(); minioObject.setBucketName(bucketName); minioObject.setObjectName(fileName); minioObject.setUrl(objectUrl); minioObject.setCreateBy("system"); minioObject.setCreateTime(new Date()); Long objectId = vMinioObjectService.insertVMinioObject(minioObject); // 更新记录的附件信息，添加处理后的视频URL String currentAccessory = record.getAccessory(); if (currentAccessory == null || currentAccessory.isEmpty()) { record.setAccessory(objectUrl); } else { record.setAccessory(currentAccessory + "," + objectUrl); } inspectionTaskRecordMapper.updateInspectionTaskRecord(record); log.info("处理后视频已上传: 任务ID={}, 记录ID={}, MinIO对象ID={}, URL={}", task.getTaskId(), record.getRecordId(), objectId, objectUrl); } /** * 处理视频记录 * @param inputFile 输入视频文件 * @param outputFile 输出视频文件 * @param detector 检测器 * @param task 巡检任务 * @param record 巡检记录 */ private void processVideoForRecord(File inputFile, File outputFile, HttpYoloDetector detector, InspectionTask task, com.ruoyi.video.domain.InspectionTaskRecord record) throws Exception { FFmpegFrameGrabber grabber = new FFmpegFrameGrabber(inputFile); grabber.start(); FFmpegFrameRecorder recorder = new FFmpegFrameRecorder(outputFile, grabber.getImageWidth(), grabber.getImageHeight(), grabber.getAudioChannels()); recorder.setFormat("mp4"); recorder.setVideoCodec(avcodec.AV_CODEC_ID_H264); recorder.setFrameRate(grabber.getFrameRate()); recorder.setVideoBitrate(grabber.getVideoBitrate()); if (grabber.getAudioChannels() > 0) { recorder.setAudioCodec(avcodec.AV_CODEC_ID_AAC); recorder.setAudioBitrate(grabber.getAudioBitrate()); recorder.setAudioChannels(grabber.getAudioChannels()); recorder.setSampleRate(grabber.getSampleRate()); } recorder.start(); OpenCVFrameConverter.ToMat converter = new OpenCVFrameConverter.ToMat(); // 用于去重的垃圾检测结果缓存 Map detectedGarbageCache = new HashMap<>(); // 跟踪检测到的垃圾ID final Long[] detectionId = {1L}; // 帧计数 long frameCount = 0; Frame frame; while ((frame = grabber.grab()) != null) { frameCount++; if (frame.image != null) { // 处理视频帧 Mat mat = converter.convert(frame); if (mat != null && !mat.isNull()) { // 每10帧执行一次检测，减少API调用频率 if (frameCount % 10 == 0) { List detections = detector.detect(mat); if (!CollectionUtils.isEmpty(detections)) { // 绘制检测框 for (Detection detection : detections) { drawDetection(mat, detection); // 检查是否为新的垃圾检测结果 String detectionKey = generateDetectionKey(detection); if (!detectedGarbageCache.containsKey(detectionKey)) { // 这是新检测到的垃圾 detectedGarbageCache.put(detectionKey, detectionId[0]++); // 创建告警记录（关联到记录） createAlarmRecordForRecord(task, record, detection, mat, frameCount); } else { // 更新上次检测时间 detectedGarbageCache.put(detectionKey, detectionId[0]++); } } // 清理超过60秒未检测到的垃圾（假设30fps） Long currentId = detectionId[0]; detectedGarbageCache.entrySet().removeIf(entry -> (currentId - entry.getValue()) > grabber.getFrameRate() * 60); } } // 转回Frame并写入录像 Frame processedFrame = converter.convert(mat); recorder.record(processedFrame); } else { // 原样写入 recorder.record(frame); } } else if (frame.samples != null) { // 音频帧原样写入 recorder.record(frame); } } recorder.stop(); recorder.close(); grabber.stop(); grabber.close(); // 上传处理后的视频（更新记录） uploadProcessedVideoForRecord(outputFile, task, record); } /** * 生成检测结果的唯一键，用于检测结果去重 */ private String generateDetectionKey(Detection detection) { // 使用检测框的位置和大小来生成键 // 允许小范围波动，认为是同一个物体 Rect rect = detection.getRect(); int x = rect.x() / 10 * 10; // 取10的倍数，允许小波动 int y = rect.y() / 10 * 10; int w = rect.width() / 10 * 10; int h = rect.height() / 10 * 10; return String.format("%s_%d_%d_%d_%d", detection.getLabel(), x, y, w, h); } /** * 在图像上绘制检测框 */ private void drawDetection(Mat frame, Detection detection) { // 绘制边界框 Rect rect = detection.getRect(); Scalar color = new Scalar(detection.getColorBGR()); int thickness = 2; rectangle(frame, rect, color, thickness, LINE_8, 0); // 绘制标签背景 String label = String.format("%s: %.2f", detection.getLabel(), detection.getConfidence()); Point textPosition = new Point(rect.x(), rect.y() - 10); // 获取文本大小 - 添加缺失的IntBuffer参数 IntBuffer baseline = IntBuffer.allocate(1); org.bytedeco.opencv.opencv_core.Size textSize = getTextSize( label, FONT_HERSHEY_SIMPLEX, 0.5, thickness, baseline); // 绘制标签背景矩形 rectangle(frame, new Rect(rect.x(), rect.y() - textSize.height() - 10, textSize.width(), textSize.height() + 10), color, FILLED, LINE_8, 0); // 绘制文本 putText(frame, label, textPosition, FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 0), 1, LINE_AA, false); } }