christian.schwarz/Deep-Live-Cam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better mouth mask

Browse Source 
better mouth mask showing and tracking the lips part only.
This commit is contained in:
Kenneth Estanislao
2026-02-10 12:21:42 +08:00
parent 2b36300b8c
commit 9a33f5e184
2 changed files with 89 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/processors/frame/face_masking.py
+19 -62
View File
@@ -45,6 +45,7 @@ def create_face_mask(face: Face, frame: Frame) -> np.ndarray:
) # 5% of face width ) # 5% of face width
# Create a slightly larger convex hull for padding # Create a slightly larger convex hull for padding
face_outline = landmarks[0:33]
hull = cv2.convexHull(face_outline) hull = cv2.convexHull(face_outline)
hull_padded = [] hull_padded = []
for point in hull: for point in hull:
@@ -70,77 +71,30 @@ def create_lower_mouth_mask(
) -> (np.ndarray, np.ndarray, tuple, np.ndarray): ) -> (np.ndarray, np.ndarray, tuple, np.ndarray):
mask = np.zeros(frame.shape[:2], dtype=np.uint8) mask = np.zeros(frame.shape[:2], dtype=np.uint8)
mouth_cutout = None mouth_cutout = None
lower_lip_polygon = None
mouth_box = (0,0,0,0)
landmarks = face.landmark_2d_106 landmarks = face.landmark_2d_106
if landmarks is not None: if landmarks is not None:
# 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 # Use outer mouth landmarks (52-63) to capture the lips only
lower_lip_order = [ lower_lip_order = list(range(52, 64))
65,
66, if max(lower_lip_order) >= landmarks.shape[0]:
62, return mask, mouth_cutout, mouth_box, lower_lip_polygon
70,
69, lower_lip_landmarks = landmarks[lower_lip_order].astype(np.float32)
18,
19,
20,
21,
22,
23,
24,
0,
8,
7,
6,
5,
4,
3,
2,
65,
]
lower_lip_landmarks = landmarks[lower_lip_order].astype(
np.float32
) # Use float for precise calculations
# Calculate the center of the landmarks # Calculate the center of the landmarks
center = np.mean(lower_lip_landmarks, axis=0) center = np.mean(lower_lip_landmarks, axis=0)
# Expand the landmarks outward using the mouth_mask_size # Expand the landmarks outward using the mouth_mask_size
# Use a more conservative expansion to avoid affecting face shape
expansion_factor = ( expansion_factor = (
1 + modules.globals.mask_down_size * modules.globals.mouth_mask_size 1 + modules.globals.mask_down_size * modules.globals.mouth_mask_size
) # Adjust expansion based on slider )
expanded_landmarks = (lower_lip_landmarks - center) * expansion_factor + center expanded_landmarks = (lower_lip_landmarks - center) * expansion_factor + center
# Extend the top lip part # Removed specific top/chin extensions to preserve face shape
toplip_indices = [
20,
0,
1,
2,
3,
4,
5,
] # Indices for landmarks 2, 65, 66, 62, 70, 69, 18
toplip_extension = (
modules.globals.mask_size * modules.globals.mouth_mask_size * 0.5
) # Adjust extension based on slider
for idx in toplip_indices:
direction = expanded_landmarks[idx] - center
direction = direction / np.linalg.norm(direction)
expanded_landmarks[idx] += direction * toplip_extension
# Extend the bottom part (chin area)
chin_indices = [
11,
12,
13,
14,
15,
16,
] # Indices for landmarks 21, 22, 23, 24, 0, 8
chin_extension = 2 * 0.2 # Adjust this factor to control the extension
for idx in chin_indices:
expanded_landmarks[idx][1] += (
expanded_landmarks[idx][1] - center[1]
) * chin_extension
# Convert back to integer coordinates # Convert back to integer coordinates
expanded_landmarks = expanded_landmarks.astype(np.int32) expanded_landmarks = expanded_landmarks.astype(np.int32)
@@ -165,7 +119,9 @@ def create_lower_mouth_mask(
# Create the mask # Create the mask
mask_roi = np.zeros((max_y - min_y, max_x - min_x), dtype=np.uint8) mask_roi = np.zeros((max_y - min_y, max_x - min_x), dtype=np.uint8)
cv2.fillPoly(mask_roi, [expanded_landmarks - [min_x, min_y]], 255) # Shift polygon coordinates relative to the ROI's top-left corner
polygon_relative_to_roi = expanded_landmarks - [min_x, min_y]
cv2.fillPoly(mask_roi, [polygon_relative_to_roi], 255)
# Apply Gaussian blur to soften the mask edges # Apply Gaussian blur to soften the mask edges
mask_roi = cv2.GaussianBlur(mask_roi, (15, 15), 5) mask_roi = cv2.GaussianBlur(mask_roi, (15, 15), 5)
@@ -178,8 +134,9 @@ def create_lower_mouth_mask(
# Return the expanded lower lip polygon in original frame coordinates # Return the expanded lower lip polygon in original frame coordinates
lower_lip_polygon = expanded_landmarks lower_lip_polygon = expanded_landmarks
mouth_box = (min_x, min_y, max_x, max_y)
return mask, mouth_cutout, (min_x, min_y, max_x, max_y), lower_lip_polygon return mask, mouth_cutout, mouth_box, lower_lip_polygon
def create_eyes_mask(face: Face, frame: Frame) -> (np.ndarray, np.ndarray, tuple, np.ndarray): def create_eyes_mask(face: Face, frame: Frame) -> (np.ndarray, np.ndarray, tuple, np.ndarray):
mask = np.zeros(frame.shape[:2], dtype=np.uint8) mask = np.zeros(frame.shape[:2], dtype=np.uint8)
modules/processors/frame/face_swapper.py
+66 -54
View File
@@ -119,6 +119,12 @@ def swap_face(source_face: Face, target_face: Face, temp_frame: Frame) -> Frame:
update_status("Face swapper model not loaded or failed to load. Skipping swap.", NAME) update_status("Face swapper model not loaded or failed to load. Skipping swap.", NAME)
return temp_frame return temp_frame
# Safety check for faces
if source_face is None or target_face is None:
return temp_frame
if not hasattr(source_face, 'normed_embedding') or source_face.normed_embedding is None:
return temp_frame
# Store a copy of the original frame before swapping for opacity blending # Store a copy of the original frame before swapping for opacity blending
original_frame = temp_frame.copy() original_frame = temp_frame.copy()
@@ -194,34 +200,34 @@ def swap_face(source_face: Face, target_face: Face, temp_frame: Frame) -> Frame:
swapped_frame, target_face, mouth_mask_data swapped_frame, target_face, mouth_mask_data
) )
# --- Poisson Blending --- # --- Poisson Blending ---
if getattr(modules.globals, "poisson_blend", False): if getattr(modules.globals, "poisson_blend", False):
face_mask = create_face_mask(target_face, temp_frame) face_mask = create_face_mask(target_face, temp_frame)
if face_mask is not None: if face_mask is not None:
# Find bounding box of the mask # Find bounding box of the mask
y_indices, x_indices = np.where(face_mask > 0) y_indices, x_indices = np.where(face_mask > 0)
if len(x_indices) > 0 and len(y_indices) > 0: if len(x_indices) > 0 and len(y_indices) > 0:
x_min, x_max = np.min(x_indices), np.max(x_indices) x_min, x_max = np.min(x_indices), np.max(x_indices)
y_min, y_max = np.min(y_indices), np.max(y_indices) y_min, y_max = np.min(y_indices), np.max(y_indices)
# Calculate center # Calculate center
center = (int((x_min + x_max) / 2), int((y_min + y_max) / 2)) center = (int((x_min + x_max) / 2), int((y_min + y_max) / 2))
# Crop src and mask # Crop src and mask
src_crop = swapped_frame[y_min : y_max + 1, x_min : x_max + 1] src_crop = swapped_frame[y_min : y_max + 1, x_min : x_max + 1]
mask_crop = face_mask[y_min : y_max + 1, x_min : x_max + 1] mask_crop = face_mask[y_min : y_max + 1, x_min : x_max + 1]
try: try:
# Use original_frame as destination to blend the swapped face onto it # Use original_frame as destination to blend the swapped face onto it
swapped_frame = cv2.seamlessClone( swapped_frame = cv2.seamlessClone(
src_crop, src_crop,
original_frame, original_frame,
mask_crop, mask_crop,
center, center,
cv2.NORMAL_CLONE, cv2.NORMAL_CLONE,
) )
except Exception as e: except Exception as e:
print(f"Poisson blending failed: {e}") print(f"Poisson blending failed: {e}")
# Apply opacity blend between the original frame and the swapped frame # Apply opacity blend between the original frame and the swapped frame
opacity = getattr(modules.globals, "opacity", 1.0) opacity = getattr(modules.globals, "opacity", 1.0)
@@ -746,8 +752,9 @@ def create_lower_mouth_mask(
return mask, mouth_cutout, mouth_box, lower_lip_polygon return mask, mouth_cutout, mouth_box, lower_lip_polygon
try: # Wrap main logic in try-except try: # Wrap main logic in try-except
# 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 # Use outer mouth landmarks (52-63) to capture the lips only
lower_lip_order = [65, 66, 62, 70, 69, 18, 19, 20, 21, 22, 23, 24, 0, 8, 7, 6, 5, 4, 3, 2, 65] # 21 points # This avoids including the chin/jawline, preserving the face shape from the swap
lower_lip_order = list(range(52, 64))
# Check if all indices are valid for the loaded landmarks (already partially done by < 106 check) # Check if all indices are valid for the loaded landmarks (already partially done by < 106 check)
if max(lower_lip_order) >= landmarks.shape[0]: if max(lower_lip_order) >= landmarks.shape[0]:
@@ -771,31 +778,6 @@ def create_lower_mouth_mask(
expansion_factor = 1 + mask_down_size expansion_factor = 1 + mask_down_size
expanded_landmarks = (lower_lip_landmarks - center) * expansion_factor + center expanded_landmarks = (lower_lip_landmarks - center) * expansion_factor + center
mask_size = getattr(modules.globals, "mask_size", 1.0) # Default 1.0
toplip_extension = mask_size * 0.5
# Define toplip indices relative to lower_lip_order (safer)
toplip_local_indices = [0, 1, 2, 3, 4, 5, 19] # Indices in lower_lip_order for [65, 66, 62, 70, 69, 18, 2]
for idx in toplip_local_indices:
if idx < len(expanded_landmarks): # Boundary check
direction = expanded_landmarks[idx] - center
norm = np.linalg.norm(direction)
if norm > 1e-6: # Avoid division by zero
direction_normalized = direction / norm
expanded_landmarks[idx] += direction_normalized * toplip_extension
# Define chin indices relative to lower_lip_order
chin_local_indices = [9, 10, 11, 12, 13, 14] # Indices for [22, 23, 24, 0, 8, 7]
chin_extension = 2 * 0.2
for idx in chin_local_indices:
if idx < len(expanded_landmarks): # Boundary check
# Extend vertically based on distance from center y
y_diff = expanded_landmarks[idx][1] - center[1]
expanded_landmarks[idx][1] += y_diff * chin_extension
# Ensure landmarks are finite after adjustments # Ensure landmarks are finite after adjustments
if not np.all(np.isfinite(expanded_landmarks)): if not np.all(np.isfinite(expanded_landmarks)):
# print("Warning: Non-finite values detected after expanding landmarks.") # print("Warning: Non-finite values detected after expanding landmarks.")
@@ -1094,13 +1076,43 @@ def create_face_mask(face: Face, frame: Frame) -> np.ndarray:
landmarks_int = landmarks.astype(np.int32) landmarks_int = landmarks.astype(np.int32)
# Use standard face outline landmarks (0-32) # Use standard face outline landmarks (0-32)
face_outline_points = landmarks_int[0:33] # Points 0 to 32 cover chin and sides # Use standard face outline (0-32)
face_outline = landmarks_int[0:33]
# Estimate forehead points to ensure mask covers the whole face (including forehead)
# This is critical for Poisson blending to work correctly on the forehead
eyebrows = landmarks_int[33:43]
if eyebrows.shape[0] > 0:
chin = landmarks_int[16]
eyebrow_center = np.mean(eyebrows, axis=0)
# Vector from chin to eyebrows (upwards)
up_vector = eyebrow_center - chin
norm = np.linalg.norm(up_vector)
if norm > 0:
up_vector /= norm
# Extend upwards by 1.0 of the chin-to-eyebrow distance (aggressive coverage)
# This ensures the mask covers the entire forehead for proper blending
forehead_offset = up_vector * (norm * 1.0)
# Shift eyebrows up to create forehead points
forehead_points = eyebrows + forehead_offset
# Expand the top points slightly outwards to cover forehead corners
# Calculate the center of the new top points
top_center = np.mean(forehead_points, axis=0)
# Expand outwards by 20%
forehead_points = (forehead_points - top_center) * 1.2 + top_center
# Combine outline and forehead points
face_outline = np.concatenate((face_outline, forehead_points.astype(np.int32)), axis=0)
# Calculate convex hull of these points # Calculate convex hull of these points
# Use try-except as convexHull can fail on degenerate input # Use try-except as convexHull can fail on degenerate input
try: try:
hull = cv2.convexHull(full_face_poly.astype(np.float32)) # Use float for accuracy hull = cv2.convexHull(face_outline.astype(np.float32)) # Use float for accuracy
if hull is None or len(hull) < 3: if hull is None or len(hull) < 3:
# print("Warning: Convex hull calculation failed or returned too few points.") # print("Warning: Convex hull calculation failed or returned too few points.")
# Fallback: use bounding box of landmarks? Or just return empty mask? # Fallback: use bounding box of landmarks? Or just return empty mask?