From 7a2ffc91a278f6e6a4a9be724b790dcfb85d1fc6 Mon Sep 17 00:00:00 2001 From: DingVero Date: Thu, 14 May 2026 18:37:18 +0800 Subject: [PATCH] update --- client/gui/dialogs/BlackholeResolveDialog.cpp | 11 +- client/gui/dialogs/BlackholeResolveDialog.h | 3 - client/gui/main_window/MainWindow.cpp | 16 +-- python_server/.env | 1 + python_server/app/routes/inpaint.py | 10 +- python_server/app/routes/meta.py | 22 ++-- python_server/app/schemas.py | 2 + python_server/app/services/predictors.py | 5 + python_server/config.py | 23 ++-- python_server/config_loader.py | 6 + python_server/model/Inpaint/__init__.py | 2 + .../model/Inpaint/flux_fill/__init__.py | 3 + .../model/Inpaint/flux_fill/loader.py | 109 ++++++++++++++++++ python_server/model/Inpaint/inpaint_loader.py | 34 +++++- python_server/server.py | 9 +- 15 files changed, 200 insertions(+), 56 deletions(-) create mode 100644 python_server/.env create mode 100644 python_server/model/Inpaint/flux_fill/__init__.py create mode 100644 python_server/model/Inpaint/flux_fill/loader.py diff --git a/client/gui/dialogs/BlackholeResolveDialog.cpp b/client/gui/dialogs/BlackholeResolveDialog.cpp index 1b44249..73e0dd4 100644 --- a/client/gui/dialogs/BlackholeResolveDialog.cpp +++ b/client/gui/dialogs/BlackholeResolveDialog.cpp @@ -25,14 +25,6 @@ QPushButton* makeAlgoButton(const QString& title, QWidget* parent) { } // namespace -QString BlackholeResolveDialog::defaultModelInpaintPrompt() { - return QStringLiteral( - "中国画局部，需去除人物并只补背景：请严格依据掩膜外已见的笔墨皴擦、设色层次、留白与纸绢肌理，" - "在掩膜内自然延展，补全为山石、坡岸、云水、苔点或空灵留白等，与外缘笔法、墨色与气韵连贯衔接；" - "禁止出现人物轮廓、肢体、衣褶、五官、手足、发丝、肤色、投影及去人后的模糊晕带、鬼影边缘与色块台阶，" - "整体须像原画从未画入人物一样自然，不留修补感。"); -} - BlackholeResolveDialog::BlackholeResolveDialog(const QString& blackholeName, QWidget* parent) : QDialog(parent), m_blackholeName(blackholeName) { @@ -142,8 +134,7 @@ void BlackholeResolveDialog::buildDetailPage() { pLay->setSpacing(8); m_promptEdit = new QPlainTextEdit(panel); - m_promptEdit->setPlainText(BlackholeResolveDialog::defaultModelInpaintPrompt()); - m_promptEdit->setPlaceholderText(QStringLiteral("可按画面增删；若全部删空，请求时仍使用内置默认描述")); + m_promptEdit->setPlaceholderText(QStringLiteral("可选")); m_promptEdit->setMinimumHeight(72); pLay->addWidget(m_promptEdit); diff --git a/client/gui/dialogs/BlackholeResolveDialog.h b/client/gui/dialogs/BlackholeResolveDialog.h index 5b202af..772c8d2 100644 --- a/client/gui/dialogs/BlackholeResolveDialog.h +++ b/client/gui/dialogs/BlackholeResolveDialog.h @@ -19,9 +19,6 @@ public: Algorithm selectedAlgorithm() const { return m_selectedAlgorithm; } QString promptText() const; - /// 模型补全（LaMa 等）内置正向提示：中国画去人物、补背景纹理，无人物残留。 - [[nodiscard]] static QString defaultModelInpaintPrompt(); - private: void buildSelectPage(); void buildDetailPage(); diff --git a/client/gui/main_window/MainWindow.cpp b/client/gui/main_window/MainWindow.cpp index a2c7f34..e6b5864 100644 --- a/client/gui/main_window/MainWindow.cpp +++ b/client/gui/main_window/MainWindow.cpp @@ -5050,21 +5050,13 @@ void MainWindow::showBlackholeContextMenu(const QPoint& globalPos, const QString client->setBaseUrl(QUrl(base)); QString immediateErr; - QString prompt = dlg.promptText(); - if (prompt.isEmpty()) { - prompt = BlackholeResolveDialog::defaultModelInpaintPrompt(); - } + const QString prompt = dlg.promptText(); QNetworkReply* reply = client->inpaintAsync( cropPng, maskPng, - // LaMa（物体移除 / 擦除） - QStringLiteral("lama"), + QStringLiteral("flux_fill"), prompt, - // 抑制人物与去人常见伪影（中英混合便于通用后端） - QStringLiteral( - "person, human, face, body, hands, feet, portrait, silhouette, skin, clothing folds, hair, " - "shadow, ghosting, blur halo, seam, banding, watermark, text, logo, signature, jpeg artifacts"), - // 去人物场景：strength 太低容易“保留人物残影”；适当提高更干净 + QString(), 0.72, 1024, &immediateErr); @@ -5077,7 +5069,7 @@ void MainWindow::showBlackholeContextMenu(const QPoint& globalPos, const QString } auto* task = new CancelableTaskDialog(QStringLiteral("黑洞修复"), - QStringLiteral("正在请求后端模型补全，请稍候……"), + QStringLiteral("补全中…"), this); task->setAttribute(Qt::WA_DeleteOnClose, true); connect(task, &CancelableTaskDialog::canceled, this, [reply, task]() { diff --git a/python_server/.env b/python_server/.env new file mode 100644 index 0000000..4e17793 --- /dev/null +++ b/python_server/.env @@ -0,0 +1 @@ +HF_TOKEN=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 \ No newline at end of file diff --git a/python_server/app/routes/inpaint.py b/python_server/app/routes/inpaint.py index e7e99b8..9926d0f 100644 --- a/python_server/app/routes/inpaint.py +++ b/python_server/app/routes/inpaint.py @@ -15,7 +15,7 @@ router = APIRouter(tags=["inpaint"]) @router.post("/inpaint") def inpaint(req: InpaintRequest) -> Dict[str, Any]: try: - model_name = req.model_name or "sdxl_inpaint" + model_name = req.model_name or "flux_fill" pil = b64_to_pil_image(req.image_b64).convert("RGB") if req.mask_b64: @@ -24,13 +24,17 @@ def inpaint(req: InpaintRequest) -> Dict[str, Any]: mask = default_half_mask(pil) predictor = get_inpaint_predictor(model_name) + call_kw: Dict[str, Any] = {"strength": req.strength, "max_side": req.max_side} + if req.guidance_scale is not None: + call_kw["guidance_scale"] = req.guidance_scale + if req.num_inference_steps is not None: + call_kw["num_inference_steps"] = req.num_inference_steps out = predictor( pil, mask, req.prompt or "", req.negative_prompt or "", - strength=req.strength, - max_side=req.max_side, + **call_kw, ) out_dir = OUTPUT_DIR / "inpaint" diff --git a/python_server/app/routes/meta.py b/python_server/app/routes/meta.py index c446df3..8b358cf 100644 --- a/python_server/app/routes/meta.py +++ b/python_server/app/routes/meta.py @@ -43,19 +43,19 @@ def get_models() -> Dict[str, Any]: "mask2former": {"name": "Mask2Former (not implemented)"}, }, "inpaint": { - "copy": {"name": "Copy (no-op)", "params": []}, - "sdxl_inpaint": { - "name": "SDXL Inpaint", - "params": [ - {"id": "prompt", "label": "提示词", "optional": True}, - ], + "flux_fill": { + "name": "FLUX.1 Fill (default)", + "params": [{"id": "prompt", "label": "prompt", "optional": True}], }, - "lama": {"name": "LaMa (Erase / Remove Object)", "params": []}, + "sdxl_inpaint": { + "name": "sdxl_inpaint", + "params": [{"id": "prompt", "label": "prompt", "optional": True}], + }, + "lama": {"name": "lama", "params": []}, + "copy": {"name": "copy (debug)", "params": []}, "controlnet": { - "name": "ControlNet Inpaint (canny)", - "params": [ - {"id": "prompt", "label": "提示词", "optional": True}, - ], + "name": "controlnet", + "params": [{"id": "prompt", "label": "prompt", "optional": True}], }, }, "animation": { diff --git a/python_server/app/schemas.py b/python_server/app/schemas.py index 21a0a06..e318062 100644 --- a/python_server/app/schemas.py +++ b/python_server/app/schemas.py @@ -52,6 +52,8 @@ class InpaintRequest(ImageInput): negative_prompt: Optional[str] = Field("", description="负向 prompt") mask_b64: Optional[str] = Field(None, description="mask PNG base64（可选）") max_side: int = Field(1024, ge=128, le=2048) + guidance_scale: Optional[float] = Field(None, ge=0.0, le=200.0, description="扩散 CFG；不传则各后端默认") + num_inference_steps: Optional[int] = Field(None, ge=1, le=150, description="采样步数；不传则各后端默认") class AnimateRequest(BaseModel): diff --git a/python_server/app/services/predictors.py b/python_server/app/services/predictors.py index 61e2523..89a32d6 100644 --- a/python_server/app/services/predictors.py +++ b/python_server/app/services/predictors.py @@ -76,6 +76,11 @@ def get_inpaint_predictor(model_name: str): _inpaint_cache[model_name] = _copy return _copy + if model_name == "flux_fill": + pred, _ = build_inpaint_predictor(UnifiedInpaintConfig(backend=InpaintBackend.FLUX_FILL)) + _inpaint_cache[model_name] = pred + return pred + if model_name == "sdxl_inpaint": pred, _ = build_inpaint_predictor(UnifiedInpaintConfig(backend=InpaintBackend.SDXL_INPAINT)) _inpaint_cache[model_name] = pred diff --git a/python_server/config.py b/python_server/config.py index 7e69d03..56bb8f2 100644 --- a/python_server/config.py +++ b/python_server/config.py @@ -86,10 +86,17 @@ class InpaintModelInfo(TypedDict): family: str name: str description: str - backend: str # "sdxl_inpaint" | "controlnet" + backend: str # "flux_fill" | "sdxl_inpaint" | "controlnet" INPAINT_MODELS: List[InpaintModelInfo] = [ + { + "id": "flux_fill", + "family": "FLUX", + "name": "FLUX.1 Fill", + "description": "FLUX Fill 专用补全", + "backend": "flux_fill", + }, { "id": "sdxl_inpaint", "family": "SDXL", @@ -100,8 +107,8 @@ INPAINT_MODELS: List[InpaintModelInfo] = [ { "id": "controlnet", "family": "ControlNet", - "name": "ControlNet (placeholder)", - "description": "ControlNet 补全/控制生成（当前统一封装暂未实现）。", + "name": "SD1.5 + ControlNet Inpaint", + "description": "runway inpaint + v11p inpaint ControlNet。", "backend": "controlnet", }, ] @@ -159,10 +166,12 @@ class DepthConfig: @dataclass class InpaintConfig: - # 统一补全默认后端 - backend: Literal["sdxl_inpaint", "controlnet"] = "sdxl_inpaint" - # SDXL Inpaint 的基础模型（可写 HuggingFace model id 或本地目录） - sdxl_base_model: str = "stabilityai/stable-diffusion-xl-base-1.0" + # 统一补全默认后端（与 model key 对齐；实际路由以请求的 model_name 为准） + backend: Literal["flux_fill", "sdxl_inpaint", "controlnet"] = "flux_fill" + # FLUX Fill（gated：须在 Hugging Face 同意条款，并建议设置环境变量 HF_TOKEN） + flux_fill_model: str = "black-forest-labs/FLUX.1-Fill-dev" + # SDXL：须为专用 inpaint 权重（勿用 xl-base 当补全底模） + sdxl_base_model: str = "diffusers/stable-diffusion-xl-1.0-inpainting-0.1" # ControlNet Inpaint 基础模型与 controlnet 权重 controlnet_base_model: str = "runwayml/stable-diffusion-inpainting" controlnet_model: str = "lllyasviel/control_v11p_sd15_inpaint" diff --git a/python_server/config_loader.py b/python_server/config_loader.py index 1d1aea1..ad37f31 100644 --- a/python_server/config_loader.py +++ b/python_server/config_loader.py @@ -85,6 +85,12 @@ def get_sdxl_base_model_from_app(app_cfg: AppConfig | None = None) -> str: return app_cfg.inpaint.sdxl_base_model +def get_flux_fill_model_from_app(app_cfg: AppConfig | None = None) -> str: + if app_cfg is None: + app_cfg = load_app_config() + return app_cfg.inpaint.flux_fill_model + + def get_controlnet_base_model_from_app(app_cfg: AppConfig | None = None) -> str: if app_cfg is None: app_cfg = load_app_config() diff --git a/python_server/model/Inpaint/__init__.py b/python_server/model/Inpaint/__init__.py index 8b13789..56e0a44 100644 --- a/python_server/model/Inpaint/__init__.py +++ b/python_server/model/Inpaint/__init__.py @@ -1 +1,3 @@ +from .flux_fill.loader import make_flux_fill_predictor +__all__ = ["make_flux_fill_predictor"] diff --git a/python_server/model/Inpaint/flux_fill/__init__.py b/python_server/model/Inpaint/flux_fill/__init__.py new file mode 100644 index 0000000..e486e28 --- /dev/null +++ b/python_server/model/Inpaint/flux_fill/__init__.py @@ -0,0 +1,3 @@ +from .loader import make_flux_fill_predictor + +__all__ = ["make_flux_fill_predictor"] diff --git a/python_server/model/Inpaint/flux_fill/loader.py b/python_server/model/Inpaint/flux_fill/loader.py new file mode 100644 index 0000000..1c94f7e --- /dev/null +++ b/python_server/model/Inpaint/flux_fill/loader.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import os +from typing import Any, Callable + +import numpy as np +from PIL import Image + + +def make_flux_fill_predictor( + model_id: str, + device: str | None = None, +) -> Callable[[Image.Image, Image.Image, str, str], Image.Image]: + import torch + from diffusers import FluxFillPipeline + + if device is None: + device = "cuda" if torch.cuda.is_available() else "cpu" + + if device == "cuda" and torch.cuda.is_bf16_supported(): + torch_dtype = torch.bfloat16 + elif device == "cuda": + torch_dtype = torch.float16 + else: + torch_dtype = torch.float32 + + hf_token = os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") + + pipe = FluxFillPipeline.from_pretrained( + model_id, + torch_dtype=torch_dtype, + token=hf_token, + ) + + if device == "cuda": + try: + pipe.enable_model_cpu_offload() + except Exception: + try: + pipe.enable_vae_tiling() + except Exception: + pass + pipe.to("cuda") + else: + pipe.to(device) + + def _predict( + image: Image.Image, + mask: Image.Image, + prompt: str, + negative_prompt: str = "", + strength: float = 1.0, + guidance_scale: float = 30.0, + num_inference_steps: int = 28, + max_side: int = 1024, + max_sequence_length: int = 512, + **_: Any, + ) -> Image.Image: + _ = negative_prompt + image = image.convert("RGB") + mask = mask.convert("L") + + orig_w, orig_h = image.size + run_w, run_h = orig_w, orig_h + if max(orig_w, orig_h) > max_side: + scale = max_side / float(max(orig_w, orig_h)) + run_w = int(round(orig_w * scale)) + run_h = int(round(orig_h * scale)) + run_w = max(8, run_w - (run_w % 8)) + run_h = max(8, run_h - (run_h % 8)) + + if (run_w, run_h) != (orig_w, orig_h): + image_run = image.resize((run_w, run_h), resample=Image.BICUBIC) + mask_run = mask.resize((run_w, run_h), resample=Image.NEAREST) + else: + image_run = image + mask_run = mask + + p = (prompt or "").strip() + if not p: + p = "Chinese ink painting background, natural texture, seamless fill, no figures" + + st = float(strength) + st = min(1.0, max(0.05, st)) + + if device == "cuda": + try: + torch.cuda.empty_cache() + except Exception: + pass + + out = pipe( + prompt=p, + image=image_run, + mask_image=mask_run, + height=run_h, + width=run_w, + strength=st, + guidance_scale=float(guidance_scale), + num_inference_steps=int(num_inference_steps), + max_sequence_length=int(max_sequence_length), + ).images[0] + + out = out.convert("RGB") + if out.size != (orig_w, orig_h): + out = out.resize((orig_w, orig_h), resample=Image.BICUBIC) + return out + + return _predict diff --git a/python_server/model/Inpaint/inpaint_loader.py b/python_server/model/Inpaint/inpaint_loader.py index 660ea63..f6457aa 100644 --- a/python_server/model/Inpaint/inpaint_loader.py +++ b/python_server/model/Inpaint/inpaint_loader.py @@ -4,6 +4,7 @@ from __future__ import annotations 统一的补全（Inpaint）模型加载入口。 当前支持： +- FLUX Fill（diffusers FluxFillPipeline，如 FLUX.1-Fill-dev） - SDXL Inpaint（diffusers AutoPipelineForInpainting） - ControlNet（Stable Diffusion + ControlNet Inpaint） - LaMa（torchscript big-lama，本地封装） @@ -11,7 +12,7 @@ from __future__ import annotations from dataclasses import dataclass from enum import Enum -from typing import Callable +from typing import Any, Callable import numpy as np from PIL import Image @@ -21,10 +22,12 @@ from config_loader import ( get_sdxl_base_model_from_app, get_controlnet_base_model_from_app, get_controlnet_model_from_app, + get_flux_fill_model_from_app, ) class InpaintBackend(str, Enum): + FLUX_FILL = "flux_fill" SDXL_INPAINT = "sdxl_inpaint" CONTROLNET = "controlnet" LAMA = "lama" @@ -36,6 +39,7 @@ class UnifiedInpaintConfig: device: str | None = None # SDXL base model (HF id 或本地目录)，不填则用 config.py 的默认值 sdxl_base_model: str | None = None + flux_fill_model: str | None = None @dataclass @@ -100,7 +104,7 @@ def _make_sdxl_inpaint_predictor( - 输出：PIL RGB 结果图 """ import torch - from diffusers import AutoPipelineForText2Image, AutoPipelineForInpainting + from diffusers import AutoPipelineForInpainting app_cfg = load_app_config() base_model = cfg.sdxl_base_model or get_sdxl_base_model_from_app(app_cfg) @@ -112,13 +116,12 @@ def _make_sdxl_inpaint_predictor( torch_dtype = torch.float16 if device == "cuda" else torch.float32 - pipe_t2i = AutoPipelineForText2Image.from_pretrained( + pipe = AutoPipelineForInpainting.from_pretrained( base_model, torch_dtype=torch_dtype, variant="fp16" if device == "cuda" else None, use_safetensors=True, ).to(device) - pipe = AutoPipelineForInpainting.from_pipe(pipe_t2i).to(device) # 省显存设置（尽量不改变输出语义） # 注意：CPU offload 会明显变慢，但能显著降低显存占用。 @@ -149,6 +152,7 @@ def _make_sdxl_inpaint_predictor( guidance_scale: float = 7.5, num_inference_steps: int = 30, max_side: int = 1024, + **_: Any, ) -> Image.Image: image = image.convert("RGB") # diffusers 要求 mask 为单通道，白色区域为需要重绘 @@ -257,6 +261,7 @@ def _make_controlnet_predictor(_: UnifiedInpaintConfig): num_inference_steps: int = 30, controlnet_conditioning_scale: float = 1.0, max_side: int = 768, + **_: Any, ) -> Image.Image: import cv2 import numpy as np @@ -316,6 +321,22 @@ def _make_controlnet_predictor(_: UnifiedInpaintConfig): return _predict +def _make_flux_fill_predictor(cfg: UnifiedInpaintConfig): + import torch + + from .flux_fill.loader import make_flux_fill_predictor as _flux_fill_factory + + app_cfg = load_app_config() + model_id = cfg.flux_fill_model or get_flux_fill_model_from_app(app_cfg) + + device = cfg.device + if device is None: + device = app_cfg.inpaint.device + device = "cuda" if device.startswith("cuda") and torch.cuda.is_available() else "cpu" + + return _flux_fill_factory(model_id=model_id, device=device) + + def _make_lama_predictor(cfg: UnifiedInpaintConfig): import torch @@ -340,7 +361,7 @@ def _make_lama_predictor(cfg: UnifiedInpaintConfig): max_side: int = 1024, **_kwargs, ) -> Image.Image: - # LaMa 不使用 prompt/negative_prompt，也不使用 strength/steps 等扩散参数 + # big-lama：仅 (RGB, mask)；无文本条件。 _ = (strength, guidance_scale, num_inference_steps, max_side) return lama(image, mask) @@ -355,6 +376,9 @@ def build_inpaint_predictor( """ cfg = cfg or UnifiedInpaintConfig() + if cfg.backend == InpaintBackend.FLUX_FILL: + return _make_flux_fill_predictor(cfg), InpaintBackend.FLUX_FILL + if cfg.backend == InpaintBackend.SDXL_INPAINT: return _make_sdxl_inpaint_predictor(cfg), InpaintBackend.SDXL_INPAINT diff --git a/python_server/server.py b/python_server/server.py index baace26..9d51aec 100644 --- a/python_server/server.py +++ b/python_server/server.py @@ -1,12 +1,11 @@ -""" - source .venv/bin/activate # 仓库根 hfut-bishe/.venv - cd python_server && uvicorn app.main:app --host 0.0.0.0 --port 8000 -""" - from __future__ import annotations import os +from dotenv import load_dotenv + +load_dotenv() + from app.main import app __all__ = ["app"]