diff --git a/fad-hrm/src/main/java/com/ruoyi/hrm/service/impl/HrmInvoiceOcrServiceImpl.java b/fad-hrm/src/main/java/com/ruoyi/hrm/service/impl/HrmInvoiceOcrServiceImpl.java index b5f1af7..b1c2b81 100644 --- a/fad-hrm/src/main/java/com/ruoyi/hrm/service/impl/HrmInvoiceOcrServiceImpl.java +++ b/fad-hrm/src/main/java/com/ruoyi/hrm/service/impl/HrmInvoiceOcrServiceImpl.java @@ -1,6 +1,10 @@ package com.ruoyi.hrm.service.impl; import cn.hutool.core.io.IoUtil; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.zxing.BinaryBitmap; import com.google.zxing.DecodeHintType; import com.google.zxing.MultiFormatReader; @@ -26,22 +30,34 @@ import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; import java.awt.image.BufferedImage; +import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.File; import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; import java.math.BigDecimal; +import java.net.HttpURLConnection; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.EnumMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; /** - * 发票识别服务实现：本地三段式管线，无任何外部 API 调用。 + * 单据识别服务实现：按附件类型分流。 * + *

图片单据（jpg/png 等，可不是规范发票）：直接调用小米 MiMo 多模态大模型识别， + * 返回标题 + 金额（复用 application.yml 中 mimo.* 配置）。 + * + *

PDF 电子发票：本地三段式管线，无外部 API： *

    *
  1. PDFBox 文本层抽取：原生电子发票直接搞定（毫秒级，几乎零算力）
    2. *
  2. ZXing 二维码识别：拍照/扫描发票 PDF，从二维码读结构化字段
    3. @@ -57,11 +73,28 @@ import java.util.regex.Pattern; public class HrmInvoiceOcrServiceImpl implements IHrmInvoiceOcrService { private final SysOssMapper sysOssMapper; + private final ObjectMapper jsonMapper = new ObjectMapper(); /** 可通过 application.yml 覆盖；默认 jar 同级目录下的 tessdata */ @Value("${fad.ocr.tessdata-path:}") private String tessdataPathConfig; + // === 小米 MiMo 多模态大模型（识别图片单据，复用 application.yml 里的 mimo.* 配置）=== + @Value("${mimo.base-url:https://api.xiaomimimo.com/v1}") + private String mimoBaseUrl; + @Value("${mimo.api-key:}") + private String mimoApiKey; + @Value("${mimo.model:mimo-v2.5}") + private String mimoModel; + @Value("${mimo.max-tokens:8192}") + private Integer mimoMaxTokens; + @Value("${mimo.timeout:180}") + private Integer mimoTimeout; + + /** 走多模态识别的图片后缀 */ + private static final Set IMAGE_SUFFIXES = + new HashSet<>(Arrays.asList("jpg", "jpeg", "png", "webp", "bmp", "gif")); + private String tessdataPath; @PostConstruct @@ -108,9 +141,6 @@ public class HrmInvoiceOcrServiceImpl implements IHrmInvoiceOcrService { throw new ServiceException("附件不存在: " + ossId); } String suffix = StringUtils.defaultIfBlank(oss.getFileSuffix(), "").toLowerCase().replace(".", ""); - if (!"pdf".equals(suffix)) { - throw new ServiceException("仅支持 PDF 电子发票，当前文件类型: " + suffix); - } byte[] fileBytes; try (InputStream in = OssFactory.instance().getObjectContent(oss.getUrl())) { @@ -119,6 +149,15 @@ public class HrmInvoiceOcrServiceImpl implements IHrmInvoiceOcrService { throw new ServiceException("读取附件失败: " + e.getMessage()); } + // 图片单据：直接走小米多模态大模型识别（可识别非发票单据） + if (IMAGE_SUFFIXES.contains(suffix)) { + log.info("[Invoice] 图片单据走多模态识别, suffix={}", suffix); + return recognizeImageByMiMo(fileBytes, suffix); + } + if (!"pdf".equals(suffix)) { + throw new ServiceException("仅支持 PDF 电子发票或图片单据(jpg/png)，当前文件类型: " + suffix); + } + try (PDDocument doc = PDDocument.load(new ByteArrayInputStream(fileBytes))) { // 第一步：文本层 String text = extractText(doc); @@ -342,4 +381,149 @@ public class HrmInvoiceOcrServiceImpl implements IHrmInvoiceOcrService { return null; } } + + // ==================== 图片单据：小米多模态识别 ==================== + + private static final String VISION_SYSTEM_PROMPT = + "你是财务单据识别助手。用户上传的是报销/拨款单据图片，可能是增值税发票、收据、出租车/火车票、" + + "餐饮小票、合同、对账单等，不一定是规范发票。请仔细识别图片中的费用条目与金额。"; + + private static final String VISION_USER_PROMPT = + "请识别这张单据，只输出严格的 JSON（不要任何解释文字、不要 markdown 代码块），格式：" + + "{\"items\":[{\"title\":\"简洁的费用标题，如 滴滴出行-市内交通 / 餐饮-海底捞 / XX酒店住宿\",\"amount\":数字}]," + + "\"totalAmount\":数字}。amount 与 totalAmount 均为纯数字（不带货币符号/逗号）；" + + "识别不到金额则填 0；一般一张单据对应一条 items，若单据内含多笔明细可拆成多条。"; + + /** 图片 → 多模态识别 → 结构化结果 */ + private HrmInvoiceOcrResultVo recognizeImageByMiMo(byte[] bytes, String suffix) { + if (StringUtils.isBlank(mimoApiKey)) { + throw new ServiceException("未配置 AI 识别服务（mimo.api-key），无法识别图片单据"); + } + String mime = "jpg".equals(suffix) ? "jpeg" : suffix; + String dataUri = "data:image/" + mime + ";base64," + + java.util.Base64.getEncoder().encodeToString(bytes); + String content = callMiMoVision(dataUri); + return parseVisionJson(content); + } + + /** 调用小米 MiMo /chat/completions（OpenAI 兼容，多模态、非流式），返回 message.content */ + private String callMiMoVision(String imageDataUri) { + ObjectNode body = jsonMapper.createObjectNode(); + body.put("model", mimoModel); + body.put("max_completion_tokens", mimoMaxTokens); + body.put("max_tokens", mimoMaxTokens); + body.put("temperature", 0.2); + body.put("stream", false); + ArrayNode messages = body.putArray("messages"); + messages.addObject().put("role", "system").put("content", VISION_SYSTEM_PROMPT); + ObjectNode userMsg = messages.addObject(); + userMsg.put("role", "user"); + ArrayNode contentArr = userMsg.putArray("content"); + contentArr.addObject().put("type", "text").put("text", VISION_USER_PROMPT); + ObjectNode img = contentArr.addObject(); + img.put("type", "image_url"); + img.putObject("image_url").put("url", imageDataUri); + + HttpURLConnection conn = null; + try { + URL url = new URL(mimoBaseUrl + "/chat/completions"); + conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setConnectTimeout(10_000); + conn.setReadTimeout((mimoTimeout == null ? 180 : mimoTimeout) * 1000); + conn.setRequestProperty("Content-Type", "application/json"); + conn.setRequestProperty("api-key", mimoApiKey); + conn.setRequestProperty("Authorization", "Bearer " + mimoApiKey); + try (OutputStream os = conn.getOutputStream()) { + os.write(jsonMapper.writeValueAsString(body).getBytes(StandardCharsets.UTF_8)); + } + int code = conn.getResponseCode(); + if (code >= 400) { + String err = readStream(conn.getErrorStream()); + throw new ServiceException("AI 识别服务返回 " + code + "：" + StringUtils.substring(err, 0, 300)); + } + JsonNode root = jsonMapper.readTree(readStream(conn.getInputStream())); + JsonNode message = root.path("choices").path(0).path("message"); + String content = message.path("content").asText(""); + if (StringUtils.isBlank(content)) { + String finish = root.path("choices").path(0).path("finish_reason").asText(""); + if ("length".equals(finish)) { + throw new ServiceException("AI 识别输出被截断，请重试或提高 mimo.max-tokens"); + } + throw new ServiceException("AI 未返回有效识别内容"); + } + return content; + } catch (ServiceException e) { + throw e; + } catch (Exception e) { + log.error("[Invoice] 调用 MiMo 多模态识别失败", e); + throw new ServiceException("AI 识别服务调用失败：" + e.getMessage()); + } finally { + if (conn != null) conn.disconnect(); + } + } + + /** 解析模型返回的 JSON（容错：去掉 markdown 代码块，截取首尾花括号） */ + private HrmInvoiceOcrResultVo parseVisionJson(String content) { + HrmInvoiceOcrResultVo vo = new HrmInvoiceOcrResultVo(); + vo.setInvoiceType("AI图片识别"); + List items = new ArrayList<>(); + try { + String jsonStr = extractJson(content); + JsonNode root = jsonMapper.readTree(jsonStr); + JsonNode arr = root.path("items"); + if (arr.isArray()) { + for (JsonNode n : arr) { + HrmInvoiceOcrResultVo.Item it = new HrmInvoiceOcrResultVo.Item(); + it.setItemName(StringUtils.trimToEmpty(n.path("title").asText(""))); + it.setAmount(parseBigDecimal(n.path("amount").asText(null))); + items.add(it); + } + } + JsonNode total = root.get("totalAmount"); + if (total != null && !total.isNull()) { + vo.setTotalAmount(parseBigDecimal(total.asText(null))); + } + } catch (Exception e) { + log.warn("[Invoice] 解析 AI 识别 JSON 失败，原始内容: {}", StringUtils.substring(content, 0, 300)); + } + if (items.isEmpty()) { + HrmInvoiceOcrResultVo.Item it = new HrmInvoiceOcrResultVo.Item(); + it.setItemName(""); + it.setAmount(vo.getTotalAmount()); + items.add(it); + } + // 无总额时用明细汇总 + if (vo.getTotalAmount() == null) { + BigDecimal sum = BigDecimal.ZERO; + for (HrmInvoiceOcrResultVo.Item it : items) { + if (it.getAmount() != null) sum = sum.add(it.getAmount()); + } + vo.setTotalAmount(sum); + } + vo.setItems(items); + return vo; + } + + /** 从可能带 ```json 包裹或前后缀文字的字符串里截取 JSON 主体 */ + private static String extractJson(String raw) { + if (raw == null) return "{}"; + String s = raw.trim(); + int begin = s.indexOf('{'); + int end = s.lastIndexOf('}'); + return (begin >= 0 && end > begin) ? s.substring(begin, end + 1) : s; + } + + private static String readStream(InputStream in) { + if (in == null) return ""; + try (BufferedReader r = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) { + StringBuilder sb = new StringBuilder(); + String l; + while ((l = r.readLine()) != null) sb.append(l).append('\n'); + return sb.toString(); + } catch (Exception e) { + return ""; + } + } } diff --git a/ruoyi-ui/src/views/hrm/requests/appropriation.vue b/ruoyi-ui/src/views/hrm/requests/appropriation.vue index 802fdaa..9b746f1 100644 --- a/ruoyi-ui/src/views/hrm/requests/appropriation.vue +++ b/ruoyi-ui/src/views/hrm/requests/appropriation.vue @@ -53,40 +53,47 @@ - +
    - 仅支持 PDF 电子发票（数电票 / 电子普通发票 / 电子专用发票），上传后自动解析金额与明细。
    - 扫描件 / 图片票 / 纸质票请先在开票平台下载 PDF 原件再上传。 + 可上传多份单据：PDF 电子发票（数电票/电子普通/专用发票）走文字提取；
    + 单据图片（jpg/png，可不是发票样式，如收据/车票/小票）将自动调用 AI 识别金额与标题。
    - +
    - 正在解析发票 PDF… + 正在识别单据…
    - +
    - 拨款明细 - （上传 PDF 后自动解析；解析失败或无发票时可手动添加） + 单据明细 + （上传后自动识别标题与金额；事由请手动填写，可手动添加条目） 手动添加条目
    - 事由说明 + 标题 + 事由 金额（元） 附件
    + @@ -104,7 +111,7 @@ + accept=".pdf,.jpg,.jpeg,.png" class="row-upload">
    @@ -375,26 +382,27 @@ export default { const res = await ocrAppropriationInvoice(ossId) if (res.code === 200 && res.data) { const { items, totalAmount, sellerName, invoiceDate, invoiceType } = res.data - // 拼接发票头部信息作为事由前缀：发票类型 · 销售方 · 开票日期 + // 发票头部信息（类型 · 销售方 · 开票日期）作为标题前缀，PDF 票据才有 const prefix = [invoiceType, sellerName, invoiceDate].filter(Boolean).join(' · ') if (items && items.length) { const startIdx = this.invoiceItems.length items.forEach((item, i) => { - const reason = [prefix, item.itemName].filter(Boolean).join(' / ') + // 标题 = 识别结果（图片单据直接为 AI 标题；PDF 票据拼接发票头部） + const title = [prefix, item.itemName].filter(Boolean).join(' / ') this.invoiceItems.push({ ossId: ossId, - itemName: item.itemName || '', - reason, + itemName: title, + reason: '', // 事由由用户手动填写 amount: item.amount || 0, sortNo: startIdx + i }) }) } else if (totalAmount) { - // 没有明细时用总金额创建一条，事由取发票头部信息 + // 没有明细时用总金额创建一条，标题取发票头部信息 this.invoiceItems.push({ ossId: ossId, - itemName: sellerName || '', - reason: prefix || '', + itemName: prefix || sellerName || '', + reason: '', amount: totalAmount, sortNo: this.invoiceItems.length }) @@ -408,10 +416,10 @@ export default { this.invoiceItems.push({ ossId: ossId, itemName: '', reason: '', amount: 0, sortNo: this.invoiceItems.length }) } } catch (e) { - console.error('[Invoice] 解析失败', e) + console.error('[Invoice] 识别失败', e) const msg = (e && e.msg) || (e && e.message) || '' this.$message.warning( - '发票解析失败，请确认上传的是开票平台下载的正规 PDF 电子发票（数电票 / 电子普通发票 / 电子专用发票）。' + + '单据识别失败，已为你添加空白条目，可手动填写标题、事由与金额。' + (msg ? ' 错误信息：' + msg : '') ) this.invoiceItems.push({ ossId: ossId, itemName: '', reason: '', amount: 0, sortNo: this.invoiceItems.length }) @@ -551,8 +559,8 @@ export default { invoiceItems: this.invoiceItems.map((item, i) => ({ ossId: item.ossId || null, sortNo: i, - itemName: item.itemName || '', - reason: item.reason || item.itemName || '', + itemName: item.itemName || '', // 标题 + reason: item.reason || '', // 事由（手写） amount: item.amount || 0 })) } @@ -688,11 +696,13 @@ export default { &:last-child { border-bottom: none; } } +.col-title { flex: 1; } .col-reason { flex: 1; } .col-amount { width: 140px; flex-shrink: 0; } .col-file { width: 64px; flex-shrink: 0; display: flex; align-items: center; justify-content: center; } .col-action { width: 32px; flex-shrink: 0; display: flex; justify-content: center; } +.invoice-table-header .col-title { flex: 1; } .invoice-table-header .col-reason { flex: 1; } .invoice-table-header .col-amount { width: 140px; } .invoice-table-header .col-file { width: 64px; } diff --git a/ruoyi-ui/src/views/hrm/requests/appropriationDetail.vue b/ruoyi-ui/src/views/hrm/requests/appropriationDetail.vue index 607a500..f50e8a1 100644 --- a/ruoyi-ui/src/views/hrm/requests/appropriationDetail.vue +++ b/ruoyi-ui/src/views/hrm/requests/appropriationDetail.vue @@ -26,11 +26,11 @@