1
  def compute_iou(box_a, box_b):
2
      """计算两个 bbox 的 IoU"""
3
      x0 = max(box_a["x0"], box_b["x0"])
4
      y0 = max(box_a["top"], box_b["top"])
5
      x1 = min(box_a["x1"], box_b["x1"])
6
      y1 = min(box_a["bottom"], box_b["bottom"])
7

8
      inter = max(0, x1 - x0) * max(0, y1 - y0)
9
      area_a = (box_a["x1"] - box_a["x0"]) * (box_a["bottom"] - box_a["top"])
10
      area_b = (box_b["x1"] - box_b["x0"]) * (box_b["bottom"] - box_b["top"])
11
      return inter / (area_a + area_b - inter + 1e-6)
12

13
  def remove_seal(image, seal_bbox, hsv_lower=(0, 80, 80), hsv_upper=(15, 255, 255)):
14
      """
15
      基于颜色直方图 + 连通域分析分离印章层，用修复算法填充印章区域。
16
      默认 HSV 范围针对红色印章，可根据实际印章颜色调整。
17
      """
18
      x0, y0 = int(seal_bbox["x0"]), int(seal_bbox["top"])
19
      x1, y1 = int(seal_bbox["x1"]), int(seal_bbox["bottom"])
20
      roi = image[y0:y1, x0:x1]
21

22
      hsv = cv2.cvtColor(roi, cv2.COLOR_BGR2HSV)
23

24
      # 红色在 HSV 空间中跨越 0 度，需要两个范围
25
      mask1 = cv2.inRange(hsv, np.array(hsv_lower), np.array(hsv_upper))
26
      mask2 = cv2.inRange(hsv, np.array([160, 80, 80]), np.array([180, 255, 255]))
27
      seal_mask = cv2.bitwise_or(mask1, mask2)
28

29
      # 连通域分析，过滤小噪点
30
      num_labels, labels, stats, _ = cv2.connectedComponentsWithStats(seal_mask)
31
      for i in range(1, num_labels):
32
          area = stats[i, cv2.CC_STAT_AREA]
33
          if area < 50:  # 面积太小的不是印章，清除
34
              seal_mask[labels == i] = 0
35

36
      # 膨胀 mask，确保覆盖印章边缘
37
      kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))
38
      seal_mask = cv2.dilate(seal_mask, kernel, iterations=2)
39

40
      # 用 inpaint 修复印章区域，恢复被遮挡的文本
41
      roi_clean = cv2.inpaint(roi, seal_mask, inpaintRadius=5, flags=cv2.INPAINT_TELEA)
42

43
      result = image.copy()
44
      result[y0:y1, x0:x1] = roi_clean
45
      return result
46

47
  def ocr_with_seal_removal(image, ocr_results, page_layout, ocr_engine,
48
                            conf_thr=0.6, iou_thr=0.3):
49
      """
50
      遮挡检查模块：当 text 块 OCR 置信度低且与印章区域重叠时，
51
      先去除印章再重新 OCR。
52

53
      参数:
54
          image:        原始页面图像 (numpy array)
55
          ocr_results:  OCR 结果列表, 每项包含 bbox 和 (text, score)
56
          page_layout:  版面分析结果, 每项包含 type / x0 / x1 / top / bottom
57
          ocr_engine:   OCR 引擎实例 (项目中的 OCR 类)
58
          conf_thr:     置信度阈值，低于此值视为可疑
59
          iou_thr:      IoU 阈值，高于此值视为被印章遮挡
60
      """
61
      seal_regions = [lt for lt in page_layout if lt["type"] in ("seal", "figure")]
62

63
      if not seal_regions:
64
          return ocr_results
65

66
      cleaned_image = None
67

68
      for i, (box, (text, score)) in enumerate(ocr_results):
69
          if score >= conf_thr:
70
              continue
71

72
          text_bbox = {
73
              "x0": min(p[0] for p in box), "x1": max(p[0] for p in box),
74
              "top": min(p[1] for p in box), "bottom": max(p[1] for p in box),
75
          }
76

77
          for seal in seal_regions:
78
              if compute_iou(text_bbox, seal) < iou_thr:
79
                  continue
80

81
              # 命中：低置信度 + 高重叠 → 去印章后重新识别
82
              if cleaned_image is None:
83
                  cleaned_image = image.copy()
84
                  for s in seal_regions:
85
                      cleaned_image = remove_seal(cleaned_image, s)
86

87
              new_text = ocr_engine.recognize(cleaned_image, np.array(box, dtype=np.float32))
88
              if new_text:
89
                  ocr_results[i] = (box, (new_text, score))
90
              break
91

92
      return ocr_results

1
def preprocess_scanned_image(img):
2
    # 模糊检测（拉普拉斯方差）
3

4
    blur_score = cv2.Laplacian(img, cv2.CV_64F).var()
5

6
    if blur_score > 100:  # 清晰图片
7
        return img
8

9
    elif 50 < blur_score <= 100:  # 中度模糊
10
        # 去噪 → 锐化 → 超分辨率 → 对比度增强
11
        img = cv2.fastNlMeansDenoising(img)
12
        img = cv2.filter2D(img, -1, sharpen_kernel)
13
        img = sr_model.predict(img)  # Real-ESRGAN
14
        img = cv2.convertScaleAbs(img, alpha=1.5, beta=10)
15

16
    else:  # 严重模糊（blur_score <= 50）
17
        # 直接用MinerU 2.5（它内置了更强的预处理）
18
        return minerU_process(img)
19

20
    return img

1
def extract_hierarchy(parsed_doc):
2
    """
3
    提取文档层级结构
4

5
    优先级：
6
    1. 法律编号（第X条） > 数字编号（1.1） > 字母编号（a）
7
    2. 字体大小：H1 > H2 > H3
8
    3. 缩进层级
9
    """
10
    # 策略1: 正则匹配常见编号模式
11
    patterns = [
12
        r'^第[一二三四五六七八九十百]+条',   # 第三条
13
        r'^\d+\.\d+\.\d+',                  # 1.1.1
14
        r'^（([一二三四五]+)）',             # （一）
15
        r'^\d+\.',                          # 1.
16
    ]
17

18
    # 策略2: 利用解析模块输出的样式信息
19
    # parsed_doc包含: font_size, bold, indent_level
20

21
    # 策略3: 训练一个层级分类器（XGBoost）
22
    # 特征: 编号类型、字体大小、是否加粗、缩进、位置
23
    hierarchy_level = hierarchy_classifier.predict(features)
24

25
    return build_tree(hierarchy_level)

1
def split_large_section(section, max_size=1024, min_size=256):
2
    """
3
    超长章节的切分策略
4
    原则：尽量保持语义完整性
5
    """
6
    chunks = []
7

8
    # 策略1: 先尝试按"小节"切
9
    subsections = section.get_subsections()
10
    if subsections:
11
        for sub in subsections:
12
            if count_tokens(sub) <= max_size:
13
                chunks.append(sub)
14
            else:
15
                # 递归切分
16
                chunks.extend(split_large_section(sub))
17
        return chunks
18

19
    # 策略2: 没有小节，按"段落"切
20
    paragraphs = section.get_paragraphs()
21
    current_chunk = []
22
    current_tokens = 0
23

24
    for para in paragraphs:
25
        para_tokens = count_tokens(para)
26

27
        # 关键判断：是否应该合并到当前chunk
28
        if current_tokens + para_tokens <= max_size:
29
            current_chunk.append(para)
30
            current_tokens += para_tokens
31
        else:
32
            # 保存当前chunk
33
            if current_tokens >= min_size:  # 避免太小的chunk
34
                chunks.append(merge(current_chunk))
35
                current_chunk = [para]
36
                current_tokens = para_tokens
37

38
    # 最后一个chunk
39
    if current_chunk:
40
        chunks.append(merge(current_chunk))
41

42
    # 策略3: 单个段落仍超长，按句子切（最后手段）
43
    chunks = [split_by_sentence(c) if count_tokens(c) > max_size else c
44
              for c in chunks]
45

46
    return chunks

1
def add_smart_overlap(chunks, overlap_tokens=100):
2
    """
3
    智能overlap：确保overlap边界是完整句子
4
    """
5
    result = []
6

7
    for i, chunk in enumerate(chunks):
8
        if i == 0:
9
            result.append(chunk)
10
            continue
11

12
        # 获取前一个chunk的最后N个tokens
13
        prev_chunk = chunks[i-1]
14
        overlap_text = get_last_n_tokens(prev_chunk.content, overlap_tokens)
15

16
        # 关键：找到最近的句子边界
17
        overlap_text = truncate_to_sentence_boundary(overlap_text)
18

19
        # 合并
20
        new_content = overlap_text + chunk.content
21
        result.append(create_chunk(new_content, chunk.metadata))
22

23
    return result
24

25
 def truncate_to_sentence_boundary(text):
26
    """
27
    截断到最近的句子边界
28
    """
29
    # 找到最后一个句号/问号/叹号的位置
30
    sentence_ends = ['.', '。', '?', '？', '!', '！']
31
    last_end = -1
32
    for end in sentence_ends:
33
        pos = text.rfind(end)
34
        if pos > last_end:
35
            last_end = pos
36

37
    if last_end > 0:
38
        return text[last_end+1:]  # 返回最后一个完整句子之后的部分
39
    else:
40
        return text  # 找不到句子边界，返回原文

1
def handle_table(table, max_size=1024):
2
    """
3
    表格的智能切分
4
    """
5
    table_tokens = count_tokens(table)
6

7
    if table_tokens <= max_size:
8
        # 情况1: 表格不大，整体作为chunk
9
        return [create_table_chunk(table)]
10

11
    else:
12
        # 情况2: 大表格，按"语义单元"切分
13

14
        # 策略A: 如果表格有分组
15
        if has_row_groups(table):
16
            return split_by_row_groups(table)
17

18
        # 策略B: 按固定行数切分，但保留表头
19
        else:
20
            chunks = []
21
            header = table.header
22
            rows_per_chunk = estimate_rows_per_chunk(table, max_size)
23

24
            for i in range(0, len(table.rows), rows_per_chunk):
25
                chunk_rows = table.rows[i:i+rows_per_chunk]
26
                # 关键：每个chunk都包含表头
27
                chunk = Table(header=header, rows=chunk_rows)
28
                chunks.append(create_table_chunk(chunk))
29

30
            return chunks

1
def handle_image(image):
2
    """
3
    图片的处理策略
4
    """
5
    # 策略1: 用多模态模型生成描述
6
    if is_chart_or_diagram(image):
7
        # 对于流程图、示意图
8
        description = gpt5.generate_description(image)
9
        return create_chunk(
10
            content=description,
11
            metadata={'type': 'image', 'image_path': image.path}
12
        )
13

14
    # 策略2: 对于数据图表，提取结构化数据
15
    elif is_data_chart(image):
16
        # 用Deplot模型提取数据
17
        data = deplot.extract(image)
18
        return create_chunk(
19
            content=f"图表数据: {data}",
20
            metadata={'type': 'chart', 'image_path': image.path}
21
        )
22

23
    # 策略3: OCR提取文字
24
    else:
25
        text = ocr.extract(image)
26
        return create_chunk(content=text, metadata={'type': 'image'})