christian.schwarz/Deep-Live-Cam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better mouth mask

Browse Source 
better mouth mask showing and tracking the lips part only.
This commit is contained in:
Kenneth Estanislao
2026-02-10 12:21:42 +08:00
parent 2b36300b8c
commit 9a33f5e184
2 changed files with 89 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/processors/frame/face_swapper.py
+69 -57
View File
@@ -119,6 +119,12 @@ def swap_face(source_face: Face, target_face: Face, temp_frame: Frame) -> Frame:
update_status("Face swapper model not loaded or failed to load. Skipping swap.", NAME)
return temp_frame
# Safety check for faces
if source_face is None or target_face is None:
return temp_frame
if not hasattr(source_face, 'normed_embedding') or source_face.normed_embedding is None:
return temp_frame
# Store a copy of the original frame before swapping for opacity blending
original_frame = temp_frame.copy()
@@ -194,34 +200,34 @@ def swap_face(source_face: Face, target_face: Face, temp_frame: Frame) -> Frame:
swapped_frame, target_face, mouth_mask_data
)
# --- Poisson Blending ---
if getattr(modules.globals, "poisson_blend", False):
face_mask = create_face_mask(target_face, temp_frame)
if face_mask is not None:
# Find bounding box of the mask
y_indices, x_indices = np.where(face_mask > 0)
if len(x_indices) > 0 and len(y_indices) > 0:
x_min, x_max = np.min(x_indices), np.max(x_indices)
y_min, y_max = np.min(y_indices), np.max(y_indices)
# Calculate center
center = (int((x_min + x_max) / 2), int((y_min + y_max) / 2))
# Crop src and mask
src_crop = swapped_frame[y_min : y_max + 1, x_min : x_max + 1]
mask_crop = face_mask[y_min : y_max + 1, x_min : x_max + 1]
try:
# Use original_frame as destination to blend the swapped face onto it
swapped_frame = cv2.seamlessClone(
src_crop,
original_frame,
mask_crop,
center,
cv2.NORMAL_CLONE,
)
except Exception as e:
print(f"Poisson blending failed: {e}")
# --- Poisson Blending ---
if getattr(modules.globals, "poisson_blend", False):
face_mask = create_face_mask(target_face, temp_frame)
if face_mask is not None:
# Find bounding box of the mask
y_indices, x_indices = np.where(face_mask > 0)
if len(x_indices) > 0 and len(y_indices) > 0:
x_min, x_max = np.min(x_indices), np.max(x_indices)
y_min, y_max = np.min(y_indices), np.max(y_indices)
# Calculate center
center = (int((x_min + x_max) / 2), int((y_min + y_max) / 2))
# Crop src and mask
src_crop = swapped_frame[y_min : y_max + 1, x_min : x_max + 1]
mask_crop = face_mask[y_min : y_max + 1, x_min : x_max + 1]
try:
# Use original_frame as destination to blend the swapped face onto it
swapped_frame = cv2.seamlessClone(
src_crop,
original_frame,
mask_crop,
center,
cv2.NORMAL_CLONE,
)
except Exception as e:
print(f"Poisson blending failed: {e}")
# Apply opacity blend between the original frame and the swapped frame
opacity = getattr(modules.globals, "opacity", 1.0)
@@ -746,8 +752,9 @@ def create_lower_mouth_mask(
return mask, mouth_cutout, mouth_box, lower_lip_polygon
try: # Wrap main logic in try-except
# 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
lower_lip_order = [65, 66, 62, 70, 69, 18, 19, 20, 21, 22, 23, 24, 0, 8, 7, 6, 5, 4, 3, 2, 65] # 21 points
# Use outer mouth landmarks (52-63) to capture the lips only
# This avoids including the chin/jawline, preserving the face shape from the swap
lower_lip_order = list(range(52, 64))
# Check if all indices are valid for the loaded landmarks (already partially done by < 106 check)
if max(lower_lip_order) >= landmarks.shape[0]:
@@ -771,31 +778,6 @@ def create_lower_mouth_mask(
expansion_factor = 1 + mask_down_size
expanded_landmarks = (lower_lip_landmarks - center) * expansion_factor + center
mask_size = getattr(modules.globals, "mask_size", 1.0) # Default 1.0
toplip_extension = mask_size * 0.5
# Define toplip indices relative to lower_lip_order (safer)
toplip_local_indices = [0, 1, 2, 3, 4, 5, 19] # Indices in lower_lip_order for [65, 66, 62, 70, 69, 18, 2]
for idx in toplip_local_indices:
if idx < len(expanded_landmarks): # Boundary check
direction = expanded_landmarks[idx] - center
norm = np.linalg.norm(direction)
if norm > 1e-6: # Avoid division by zero
direction_normalized = direction / norm
expanded_landmarks[idx] += direction_normalized * toplip_extension
# Define chin indices relative to lower_lip_order
chin_local_indices = [9, 10, 11, 12, 13, 14] # Indices for [22, 23, 24, 0, 8, 7]
chin_extension = 2 * 0.2
for idx in chin_local_indices:
if idx < len(expanded_landmarks): # Boundary check
# Extend vertically based on distance from center y
y_diff = expanded_landmarks[idx][1] - center[1]
expanded_landmarks[idx][1] += y_diff * chin_extension
# Ensure landmarks are finite after adjustments
if not np.all(np.isfinite(expanded_landmarks)):
# print("Warning: Non-finite values detected after expanding landmarks.")
@@ -1094,13 +1076,43 @@ def create_face_mask(face: Face, frame: Frame) -> np.ndarray:
landmarks_int = landmarks.astype(np.int32)
# Use standard face outline landmarks (0-32)
face_outline_points = landmarks_int[0:33] # Points 0 to 32 cover chin and sides
# Use standard face outline (0-32)
face_outline = landmarks_int[0:33]
# Estimate forehead points to ensure mask covers the whole face (including forehead)
# This is critical for Poisson blending to work correctly on the forehead
eyebrows = landmarks_int[33:43]
if eyebrows.shape[0] > 0:
chin = landmarks_int[16]
eyebrow_center = np.mean(eyebrows, axis=0)
# Vector from chin to eyebrows (upwards)
up_vector = eyebrow_center - chin
norm = np.linalg.norm(up_vector)
if norm > 0:
up_vector /= norm
# Extend upwards by 1.0 of the chin-to-eyebrow distance (aggressive coverage)
# This ensures the mask covers the entire forehead for proper blending
forehead_offset = up_vector * (norm * 1.0)
# Shift eyebrows up to create forehead points
forehead_points = eyebrows + forehead_offset
# Expand the top points slightly outwards to cover forehead corners
# Calculate the center of the new top points
top_center = np.mean(forehead_points, axis=0)
# Expand outwards by 20%
forehead_points = (forehead_points - top_center) * 1.2 + top_center
# Combine outline and forehead points
face_outline = np.concatenate((face_outline, forehead_points.astype(np.int32)), axis=0)
# Calculate convex hull of these points
# Use try-except as convexHull can fail on degenerate input
try:
hull = cv2.convexHull(full_face_poly.astype(np.float32)) # Use float for accuracy
hull = cv2.convexHull(face_outline.astype(np.float32)) # Use float for accuracy
if hull is None or len(hull) < 3:
# print("Warning: Convex hull calculation failed or returned too few points.")
# Fallback: use bounding box of landmarks? Or just return empty mask?