From 3ba8ba80b302b0db143fa5d144259e84cc7e5596 Mon Sep 17 00:00:00 2001 From: DingVero Date: Thu, 14 May 2026 22:12:40 +0800 Subject: [PATCH] update --- client/gui/editor/EditorCanvas.cpp | 180 +++++++++++++++--------- client/gui/editor/EditorCanvas.h | 8 +- client/gui/editor/EntityCutoutUtils.cpp | 25 ++++ client/gui/editor/EntityCutoutUtils.h | 3 + client/gui/main_window/MainWindow.cpp | 22 +++ python_server/model/Seg/seg_loader.py | 83 ++++++++++- 6 files changed, 252 insertions(+), 69 deletions(-) diff --git a/client/gui/editor/EditorCanvas.cpp b/client/gui/editor/EditorCanvas.cpp index 2f55fa5..2fdacb2 100644 --- a/client/gui/editor/EditorCanvas.cpp +++ b/client/gui/editor/EditorCanvas.cpp @@ -1493,6 +1493,117 @@ void EditorCanvas::invalidateViewportLod() { ++m_vpToken; } +void EditorCanvas::syncFullBackgroundImageFromDiskIfDirty() { + if (m_bgAbsPath.isEmpty() || m_useViewportLod) { + return; + } + if (!m_bgImageDirty) { + return; + } + m_bgImageDirty = false; + m_bgImage = readImageTolerant(m_bgAbsPath); + if (!m_bgImage.isNull() && m_bgImage.format() != QImage::Format_ARGB32_Premultiplied) { + m_bgImage = m_bgImage.convertToFormat(QImage::Format_ARGB32_Premultiplied); + } + m_bgCutoutDirty = true; +} + +void EditorCanvas::rebuildFullBackgroundCutoutIfDirty() { + if (m_useViewportLod || m_bgAbsPath.isEmpty()) { + return; + } + if (!m_bgCutoutDirty) { + return; + } + if (m_bgImage.isNull()) { + syncFullBackgroundImageFromDiskIfDirty(); + } + if (m_bgImage.isNull()) { + return; + } + m_bgCutoutDirty = false; + m_bgImageCutout = m_bgImage; + for (const auto& ent : m_entities) { + if (ent.blackholeVisible && !ent.cutoutPolygonWorld.isEmpty()) { + entity_cutout::applyBlackFillToBackground(m_bgImageCutout, ent.cutoutPolygonWorld); + } + } +} + +QVector> EditorCanvas::visibleBlackholeCutoutPolysWorld() const { + QVector> out; + for (const auto& ent : m_entities) { + if (ent.blackholeVisible && ent.cutoutPolygonWorld.size() >= 3) { + out.push_back(ent.cutoutPolygonWorld); + } + } + return out; +} + +void EditorCanvas::applyBlackHolesWorldToCrop(QImage& cropPremul, const QRect& cropWorld) const { + entity_cutout::applyBlackHolesToImageRegion(cropPremul, cropWorld, visibleBlackholeCutoutPolysWorld()); +} + +bool EditorCanvas::loadRgbCropForEntitySegmentStroke(const QRectF& polyBrWorld, QRect& outCropWorld, + QImage& outCropRgb) { + outCropRgb = QImage(); + if (m_bgAbsPath.isEmpty()) { + return false; + } + ensurePixmapLoaded(); + QRect cropWorld = entity_cutout::clampRectToImage( + polyBrWorld.adjusted(-kSamCropMargin, -kSamCropMargin, kSamCropMargin, kSamCropMargin).toAlignedRect(), + QSize(backgroundLogicalWidth(), backgroundLogicalHeight())); + if (cropWorld.isEmpty()) { + return false; + } + outCropWorld = cropWorld; + + if (!m_useViewportLod) { + syncFullBackgroundImageFromDiskIfDirty(); + rebuildFullBackgroundCutoutIfDirty(); + if (!m_bgImageCutout.isNull()) { + outCropRgb = m_bgImageCutout.copy(cropWorld); + } + } + if (outCropRgb.isNull()) { + (void)core::image_file::loadRegionToQImage(m_bgAbsPath, cropWorld, 4096, 4096, &outCropRgb); + if (!outCropRgb.isNull() && outCropRgb.format() != QImage::Format_ARGB32_Premultiplied) { + outCropRgb = outCropRgb.convertToFormat(QImage::Format_ARGB32_Premultiplied); + } + } + if (outCropRgb.isNull() && !m_bgImage.isNull()) { + const QRect cw = entity_cutout::clampRectToImage( + polyBrWorld.adjusted(-kSamCropMargin, -kSamCropMargin, kSamCropMargin, kSamCropMargin).toAlignedRect(), + m_bgImage.size()); + if (!cw.isEmpty()) { + outCropRgb = m_bgImage.copy(cw); + outCropWorld = cw; + } + } + if (outCropRgb.isNull()) { + syncFullBackgroundImageFromDiskIfDirty(); + rebuildFullBackgroundCutoutIfDirty(); + if (!m_bgImage.isNull()) { + const QRect cw2 = entity_cutout::clampRectToImage( + polyBrWorld.adjusted(-kSamCropMargin, -kSamCropMargin, kSamCropMargin, kSamCropMargin).toAlignedRect(), + m_bgImage.size()); + if (!cw2.isEmpty()) { + outCropRgb = m_bgImage.copy(cw2); + outCropWorld = cw2; + } + } + } + if (outCropRgb.isNull()) { + return false; + } + if (outCropRgb.format() != QImage::Format_ARGB32_Premultiplied) { + outCropRgb = outCropRgb.convertToFormat(QImage::Format_ARGB32_Premultiplied); + } + applyBlackHolesWorldToCrop(outCropRgb, outCropWorld); + return true; +} + void EditorCanvas::ensurePixmapLoaded() const { if (!m_pixmapDirty) { return; @@ -1827,23 +1938,8 @@ void EditorCanvas::paintEvent(QPaintEvent* e) { const bool showBg = m_presentationPreviewMode || m_backgroundVisible; if (showBg && !m_useViewportLod) { - if (m_bgImageDirty) { - m_bgImageDirty = false; - m_bgImage = readImageTolerant(m_bgAbsPath); - if (m_bgImage.format() != QImage::Format_ARGB32_Premultiplied && !m_bgImage.isNull()) { - m_bgImage = m_bgImage.convertToFormat(QImage::Format_ARGB32_Premultiplied); - } - m_bgCutoutDirty = true; - } - if (m_bgCutoutDirty) { - m_bgCutoutDirty = false; - m_bgImageCutout = m_bgImage; - for (const auto& ent : m_entities) { - if (ent.blackholeVisible && !ent.cutoutPolygonWorld.isEmpty() && !m_bgImageCutout.isNull()) { - entity_cutout::applyBlackFillToBackground(m_bgImageCutout, ent.cutoutPolygonWorld); - } - } - } + syncFullBackgroundImageFromDiskIfDirty(); + rebuildFullBackgroundCutoutIfDirty(); } // 以“世界坐标”绘制：支持缩放/平移 @@ -3136,69 +3232,27 @@ void EditorCanvas::mouseReleaseEvent(QMouseEvent* e) { } } else if (m_entityCreateSegmentMode == EntityCreateSegmentMode::Snap) { if (m_strokeWorld.size() >= kMinStrokePointsManual) { - ensurePixmapLoaded(); const QRectF polyBr = QPolygonF(m_strokeWorld).boundingRect(); - const QRect cropWorld = entity_cutout::clampRectToImage( - polyBr.adjusted(-kSamCropMargin, -kSamCropMargin, kSamCropMargin, kSamCropMargin) - .toAlignedRect(), - QSize(backgroundLogicalWidth(), backgroundLogicalHeight())); + QRect cropWorld; QImage crop; - if (!cropWorld.isEmpty() && - core::image_file::loadRegionToQImage(m_bgAbsPath, cropWorld, 4096, 4096, &crop) && - !crop.isNull()) { + if (loadRgbCropForEntitySegmentStroke(polyBr, cropWorld, crop)) { const QImage gray = crop.convertToFormat(QImage::Format_Grayscale8); const QVector snapped = snapStrokeToEdgesWorld(m_strokeWorld, gray, cropWorld.topLeft(), cropWorld.size(), 6.0); setPendingEntityPolygonWorld(snapped); - } else if (!m_bgImage.isNull()) { - const QRect cw = entity_cutout::clampRectToImage( - polyBr.adjusted(-kSamCropMargin, -kSamCropMargin, kSamCropMargin, kSamCropMargin) - .toAlignedRect(), - m_bgImage.size()); - const QImage gray = m_bgImage.copy(cw).convertToFormat(QImage::Format_Grayscale8); - const QVector snapped = - snapStrokeToEdgesWorld(m_strokeWorld, gray, cw.topLeft(), cw.size(), 6.0); - setPendingEntityPolygonWorld(snapped); } } } else if (m_strokeWorld.size() >= kMinStrokePointsSam) { - ensurePixmapLoaded(); const QRectF polyBr = QPolygonF(m_strokeWorld).boundingRect(); - QRect cropWorld = entity_cutout::clampRectToImage( - polyBr.adjusted(-kSamCropMargin, -kSamCropMargin, kSamCropMargin, kSamCropMargin) - .toAlignedRect(), - QSize(backgroundLogicalWidth(), backgroundLogicalHeight())); + QRect cropWorld; QImage crop; - if (!cropWorld.isEmpty()) { - (void)core::image_file::loadRegionToQImage(m_bgAbsPath, cropWorld, 4096, 4096, &crop); - } - if (crop.isNull() && !m_bgImage.isNull()) { - cropWorld = entity_cutout::clampRectToImage( - polyBr.adjusted(-kSamCropMargin, -kSamCropMargin, kSamCropMargin, kSamCropMargin) - .toAlignedRect(), - m_bgImage.size()); - crop = m_bgImage.copy(cropWorld); - } - if (crop.isNull()) { - m_bgImageDirty = false; - m_bgImage = readImageTolerant(m_bgAbsPath); - if (!m_bgImage.isNull() && - m_bgImage.format() != QImage::Format_ARGB32_Premultiplied) { - m_bgImage = m_bgImage.convertToFormat(QImage::Format_ARGB32_Premultiplied); - } - cropWorld = entity_cutout::clampRectToImage( - polyBr.adjusted(-kSamCropMargin, -kSamCropMargin, kSamCropMargin, kSamCropMargin) - .toAlignedRect(), - m_bgImage.size()); - crop = m_bgImage.copy(cropWorld); - } QByteArray cropPng; QByteArray ovPng; QPointF cropOrigin; QJsonArray pts; QJsonArray labs; QJsonArray box; - if (!crop.isNull() && + if (loadRgbCropForEntitySegmentStroke(polyBr, cropWorld, crop) && buildSamSegmentPayloadFromStrokeMapped(m_strokeWorld, crop, cropWorld, cropPng, ovPng, cropOrigin, pts, labs, box)) { emit requestSamSegment(cropPng, ovPng, cropOrigin, pts, labs, box); diff --git a/client/gui/editor/EditorCanvas.h b/client/gui/editor/EditorCanvas.h index f30a1df..3e568f2 100644 --- a/client/gui/editor/EditorCanvas.h +++ b/client/gui/editor/EditorCanvas.h @@ -209,7 +209,13 @@ private: void beginPresentationZoomRestore(); void presentationComputeZoomTarget(int entityIndex, QPointF* outPan, qreal* outScale) const; -private: + void syncFullBackgroundImageFromDiskIfDirty(); + void rebuildFullBackgroundCutoutIfDirty(); + QVector> visibleBlackholeCutoutPolysWorld() const; + void applyBlackHolesWorldToCrop(QImage& cropPremul, const QRect& cropWorld) const; + /// 与画布黑洞一致：优先整图 cutout 缓存，否则区域解码后再填黑。 + bool loadRgbCropForEntitySegmentStroke(const QRectF& polyBrWorld, QRect& outCropWorld, QImage& outCropRgb); + struct Entity { QString id; QRectF rect; // world 坐标（用于拖拽与约束） diff --git a/client/gui/editor/EntityCutoutUtils.cpp b/client/gui/editor/EntityCutoutUtils.cpp index c1af50d..7a5ac42 100644 --- a/client/gui/editor/EntityCutoutUtils.cpp +++ b/client/gui/editor/EntityCutoutUtils.cpp @@ -123,4 +123,29 @@ void applyBlackFillToBackground(QImage& bgCutout, const QVector& polyWo p.end(); } +void applyBlackHolesToImageRegion(QImage& regionPremul, const QRect& cropWorldRect, + const QVector>& holePolysWorld) { + if (regionPremul.isNull() || !cropWorldRect.isValid()) { + return; + } + QPainter painter(®ionPremul); + painter.setRenderHint(QPainter::Antialiasing, true); + painter.setPen(Qt::NoPen); + painter.setBrush(QColor(0, 0, 0, 255)); + const double ox = static_cast(cropWorldRect.left()); + const double oy = static_cast(cropWorldRect.top()); + for (const QVector& polyWorld : holePolysWorld) { + if (polyWorld.size() < 3) { + continue; + } + QPolygonF local; + local.reserve(polyWorld.size()); + for (const QPointF& w : polyWorld) { + local.append(QPointF(w.x() - ox, w.y() - oy)); + } + painter.drawPolygon(local); + } + painter.end(); +} + } // namespace entity_cutout diff --git a/client/gui/editor/EntityCutoutUtils.h b/client/gui/editor/EntityCutoutUtils.h index 866c90c..01104bc 100644 --- a/client/gui/editor/EntityCutoutUtils.h +++ b/client/gui/editor/EntityCutoutUtils.h @@ -14,5 +14,8 @@ QPointF polygonCentroid(const QVector& poly); QRect clampRectToImage(const QRect& r, const QSize& size); QImage extractEntityImage(const QImage& bg, const QVector& polyWorld, QPointF& outTopLeftWorld); void applyBlackFillToBackground(QImage& bgCutout, const QVector& polyWorld); +/// 子图左上角与 cropWorldRect.topLeft() 对齐（世界坐标）；将 holePolysWorld 内像素置黑，供分割/吸附与画布一致。 +void applyBlackHolesToImageRegion(QImage& regionPremul, const QRect& cropWorldRect, + const QVector>& holePolysWorld); } // namespace entity_cutout diff --git a/client/gui/main_window/MainWindow.cpp b/client/gui/main_window/MainWindow.cpp index e6b5864..4893469 100644 --- a/client/gui/main_window/MainWindow.cpp +++ b/client/gui/main_window/MainWindow.cpp @@ -114,6 +114,25 @@ constexpr int kProjectTreeDockStartupHeight = 170; constexpr int kPropertiesDockStartupHeight = 380; constexpr char kMimeHotspotAnimationJson[] = "application/x-hfut-hotspot-animation+json"; +void applyVisibleBlackholesToBackground(QImage& bgPremul, const QVector& entities) { + for (const auto& ex : entities) { + if (ex.blackholeVisible && ex.cutoutPolygonWorld.size() >= 3) { + entity_cutout::applyBlackFillToBackground(bgPremul, ex.cutoutPolygonWorld); + } + } +} + +[[nodiscard]] QVector> visibleBlackholePolygonsFromEntities( + const QVector& entities) { + QVector> holes; + for (const auto& ex : entities) { + if (ex.blackholeVisible && ex.cutoutPolygonWorld.size() >= 3) { + holes.push_back(ex.cutoutPolygonWorld); + } + } + return holes; +} + [[nodiscard]] QString firstAnimationSchemeIdNonNoneWs(const core::Project& proj) { for (const auto& a : proj.animations()) { if (a.id != QStringLiteral("none")) @@ -3901,6 +3920,7 @@ void MainWindow::rebuildCentralPages() { if (!bg.isNull() && bg.format() != QImage::Format_ARGB32_Premultiplied) { bg = bg.convertToFormat(QImage::Format_ARGB32_Premultiplied); } + applyVisibleBlackholesToBackground(bg, m_workspace.entities()); QImage cutout; if (!bg.isNull()) { QPointF topLeft; @@ -4028,6 +4048,8 @@ void MainWindow::rebuildCentralPages() { if (bgRegion.format() != QImage::Format_ARGB32_Premultiplied) { bgRegion = bgRegion.convertToFormat(QImage::Format_ARGB32_Premultiplied); } + entity_cutout::applyBlackHolesToImageRegion( + bgRegion, region, visibleBlackholePolygonsFromEntities(m_workspace.entities())); QVector localPoly; localPoly.reserve(ent.cutoutPolygonWorld.size()); for (const auto& p : ent.cutoutPolygonWorld) { diff --git a/python_server/model/Seg/seg_loader.py b/python_server/model/Seg/seg_loader.py index 048f57d..bcbafd8 100644 --- a/python_server/model/Seg/seg_loader.py +++ b/python_server/model/Seg/seg_loader.py @@ -87,14 +87,87 @@ def run_sam_prompt( box=box_arg, multimask_output=True, ) - # masks: C x H x W - best = int(np.argmax(scores)) - m = masks[best] - if m.dtype != np.bool_: - m = m > 0.5 + m = _pick_best_sam_mask(masks, scores, pc, pl) + m = _refine_sam_instance_mask(m) return m +def _mask_contains_foreground_points( + mask_bool: np.ndarray, + point_coords: np.ndarray, + point_labels: np.ndarray, +) -> bool: + for (x, y), lab in zip(point_coords, point_labels): + if int(lab) != 1: + continue + xi = int(round(float(x))) + yi = int(round(float(y))) + if yi < 0 or xi < 0 or yi >= mask_bool.shape[0] or xi >= mask_bool.shape[1]: + return False + if not bool(mask_bool[yi, xi]): + return False + return True + + +def _to_bool_mask(m: np.ndarray) -> np.ndarray: + if m.dtype == np.bool_: + return m + return m > 0.5 + + +def _pick_best_sam_mask( + masks: np.ndarray, + scores: np.ndarray, + point_coords: np.ndarray, + point_labels: np.ndarray, +) -> np.ndarray: + """ + 多掩膜时：优先覆盖全部前景点；在高分候选中取面积最大，减少“只切到局部/缺边”的概率。 + """ + n = int(masks.shape[0]) + valid_idx: List[int] = [] + for i in range(n): + mb = _to_bool_mask(masks[i]) + if _mask_contains_foreground_points(mb, point_coords, point_labels): + valid_idx.append(i) + if not valid_idx: + best = int(np.argmax(scores)) + return _to_bool_mask(masks[best]) + smax = max(float(scores[j]) for j in valid_idx) + near = [j for j in valid_idx if float(scores[j]) >= smax - 0.12] + best_area = -1 + best_j = near[0] + for j in near: + mb = _to_bool_mask(masks[j]) + area = int(np.count_nonzero(mb)) + if area > best_area: + best_area = area + best_j = j + return _to_bool_mask(masks[best_j]) + + +def _refine_sam_instance_mask(mask_bool: np.ndarray) -> np.ndarray: + """填内部孔洞 + 轻微闭运算，使复杂物体轮廓更完整。""" + m = np.asarray(mask_bool, dtype=bool) + try: + from scipy import ndimage as ndi # type: ignore[import] + + m = ndi.binary_fill_holes(m) + m = ndi.binary_closing(m, structure=np.ones((3, 3), dtype=bool)) + return m + except Exception: + pass + try: + import cv2 # type: ignore[import] + + u8 = (m.astype(np.uint8) * 255) + k = np.ones((3, 3), np.uint8) + u8 = cv2.morphologyEx(u8, cv2.MORPH_CLOSE, k) + return u8 > 127 + except Exception: + return m + + def expand_mask(mask_bool: np.ndarray, expand_px: int = 0) -> np.ndarray: """ 轻微扩大二值 mask（用于消除边界过软/过细带来的“漏边”观感）。