From 9a33f5e184e8f5d4de6b9cf08d175162868c3d73 Mon Sep 17 00:00:00 2001 From: Kenneth Estanislao Date: Tue, 10 Feb 2026 12:21:42 +0800 Subject: [PATCH] better mouth mask better mouth mask showing and tracking the lips part only. --- modules/processors/frame/face_masking.py | 83 ++++----------- modules/processors/frame/face_swapper.py | 126 +++++++++++++---------- 2 files changed, 89 insertions(+), 120 deletions(-) diff --git a/modules/processors/frame/face_masking.py b/modules/processors/frame/face_masking.py index 152ea6f..2566116 100644 --- a/modules/processors/frame/face_masking.py +++ b/modules/processors/frame/face_masking.py @@ -45,6 +45,7 @@ def create_face_mask(face: Face, frame: Frame) -> np.ndarray: ) # 5% of face width # Create a slightly larger convex hull for padding + face_outline = landmarks[0:33] hull = cv2.convexHull(face_outline) hull_padded = [] for point in hull: @@ -70,77 +71,30 @@ def create_lower_mouth_mask( ) -> (np.ndarray, np.ndarray, tuple, np.ndarray): mask = np.zeros(frame.shape[:2], dtype=np.uint8) mouth_cutout = None + lower_lip_polygon = None + mouth_box = (0,0,0,0) + landmarks = face.landmark_2d_106 if landmarks is not None: - # 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 - lower_lip_order = [ - 65, - 66, - 62, - 70, - 69, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 0, - 8, - 7, - 6, - 5, - 4, - 3, - 2, - 65, - ] - lower_lip_landmarks = landmarks[lower_lip_order].astype( - np.float32 - ) # Use float for precise calculations + # Use outer mouth landmarks (52-63) to capture the lips only + lower_lip_order = list(range(52, 64)) + + if max(lower_lip_order) >= landmarks.shape[0]: + return mask, mouth_cutout, mouth_box, lower_lip_polygon + + lower_lip_landmarks = landmarks[lower_lip_order].astype(np.float32) # Calculate the center of the landmarks center = np.mean(lower_lip_landmarks, axis=0) # Expand the landmarks outward using the mouth_mask_size + # Use a more conservative expansion to avoid affecting face shape expansion_factor = ( 1 + modules.globals.mask_down_size * modules.globals.mouth_mask_size - ) # Adjust expansion based on slider + ) expanded_landmarks = (lower_lip_landmarks - center) * expansion_factor + center - # Extend the top lip part - toplip_indices = [ - 20, - 0, - 1, - 2, - 3, - 4, - 5, - ] # Indices for landmarks 2, 65, 66, 62, 70, 69, 18 - toplip_extension = ( - modules.globals.mask_size * modules.globals.mouth_mask_size * 0.5 - ) # Adjust extension based on slider - for idx in toplip_indices: - direction = expanded_landmarks[idx] - center - direction = direction / np.linalg.norm(direction) - expanded_landmarks[idx] += direction * toplip_extension - - # Extend the bottom part (chin area) - chin_indices = [ - 11, - 12, - 13, - 14, - 15, - 16, - ] # Indices for landmarks 21, 22, 23, 24, 0, 8 - chin_extension = 2 * 0.2 # Adjust this factor to control the extension - for idx in chin_indices: - expanded_landmarks[idx][1] += ( - expanded_landmarks[idx][1] - center[1] - ) * chin_extension + # Removed specific top/chin extensions to preserve face shape # Convert back to integer coordinates expanded_landmarks = expanded_landmarks.astype(np.int32) @@ -165,7 +119,9 @@ def create_lower_mouth_mask( # Create the mask mask_roi = np.zeros((max_y - min_y, max_x - min_x), dtype=np.uint8) - cv2.fillPoly(mask_roi, [expanded_landmarks - [min_x, min_y]], 255) + # Shift polygon coordinates relative to the ROI's top-left corner + polygon_relative_to_roi = expanded_landmarks - [min_x, min_y] + cv2.fillPoly(mask_roi, [polygon_relative_to_roi], 255) # Apply Gaussian blur to soften the mask edges mask_roi = cv2.GaussianBlur(mask_roi, (15, 15), 5) @@ -178,8 +134,9 @@ def create_lower_mouth_mask( # Return the expanded lower lip polygon in original frame coordinates lower_lip_polygon = expanded_landmarks + mouth_box = (min_x, min_y, max_x, max_y) - return mask, mouth_cutout, (min_x, min_y, max_x, max_y), lower_lip_polygon + return mask, mouth_cutout, mouth_box, lower_lip_polygon def create_eyes_mask(face: Face, frame: Frame) -> (np.ndarray, np.ndarray, tuple, np.ndarray): mask = np.zeros(frame.shape[:2], dtype=np.uint8) @@ -606,4 +563,4 @@ def draw_mask_visualization( 1, ) - return vis_frame \ No newline at end of file + return vis_frame \ No newline at end of file diff --git a/modules/processors/frame/face_swapper.py b/modules/processors/frame/face_swapper.py index 427507f..40b3401 100644 --- a/modules/processors/frame/face_swapper.py +++ b/modules/processors/frame/face_swapper.py @@ -119,6 +119,12 @@ def swap_face(source_face: Face, target_face: Face, temp_frame: Frame) -> Frame: update_status("Face swapper model not loaded or failed to load. Skipping swap.", NAME) return temp_frame + # Safety check for faces + if source_face is None or target_face is None: + return temp_frame + if not hasattr(source_face, 'normed_embedding') or source_face.normed_embedding is None: + return temp_frame + # Store a copy of the original frame before swapping for opacity blending original_frame = temp_frame.copy() @@ -194,34 +200,34 @@ def swap_face(source_face: Face, target_face: Face, temp_frame: Frame) -> Frame: swapped_frame, target_face, mouth_mask_data ) - # --- Poisson Blending --- - if getattr(modules.globals, "poisson_blend", False): - face_mask = create_face_mask(target_face, temp_frame) - if face_mask is not None: - # Find bounding box of the mask - y_indices, x_indices = np.where(face_mask > 0) - if len(x_indices) > 0 and len(y_indices) > 0: - x_min, x_max = np.min(x_indices), np.max(x_indices) - y_min, y_max = np.min(y_indices), np.max(y_indices) - - # Calculate center - center = (int((x_min + x_max) / 2), int((y_min + y_max) / 2)) - - # Crop src and mask - src_crop = swapped_frame[y_min : y_max + 1, x_min : x_max + 1] - mask_crop = face_mask[y_min : y_max + 1, x_min : x_max + 1] - - try: - # Use original_frame as destination to blend the swapped face onto it - swapped_frame = cv2.seamlessClone( - src_crop, - original_frame, - mask_crop, - center, - cv2.NORMAL_CLONE, - ) - except Exception as e: - print(f"Poisson blending failed: {e}") + # --- Poisson Blending --- + if getattr(modules.globals, "poisson_blend", False): + face_mask = create_face_mask(target_face, temp_frame) + if face_mask is not None: + # Find bounding box of the mask + y_indices, x_indices = np.where(face_mask > 0) + if len(x_indices) > 0 and len(y_indices) > 0: + x_min, x_max = np.min(x_indices), np.max(x_indices) + y_min, y_max = np.min(y_indices), np.max(y_indices) + + # Calculate center + center = (int((x_min + x_max) / 2), int((y_min + y_max) / 2)) + + # Crop src and mask + src_crop = swapped_frame[y_min : y_max + 1, x_min : x_max + 1] + mask_crop = face_mask[y_min : y_max + 1, x_min : x_max + 1] + + try: + # Use original_frame as destination to blend the swapped face onto it + swapped_frame = cv2.seamlessClone( + src_crop, + original_frame, + mask_crop, + center, + cv2.NORMAL_CLONE, + ) + except Exception as e: + print(f"Poisson blending failed: {e}") # Apply opacity blend between the original frame and the swapped frame opacity = getattr(modules.globals, "opacity", 1.0) @@ -746,8 +752,9 @@ def create_lower_mouth_mask( return mask, mouth_cutout, mouth_box, lower_lip_polygon try: # Wrap main logic in try-except - # 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 - lower_lip_order = [65, 66, 62, 70, 69, 18, 19, 20, 21, 22, 23, 24, 0, 8, 7, 6, 5, 4, 3, 2, 65] # 21 points + # Use outer mouth landmarks (52-63) to capture the lips only + # This avoids including the chin/jawline, preserving the face shape from the swap + lower_lip_order = list(range(52, 64)) # Check if all indices are valid for the loaded landmarks (already partially done by < 106 check) if max(lower_lip_order) >= landmarks.shape[0]: @@ -771,31 +778,6 @@ def create_lower_mouth_mask( expansion_factor = 1 + mask_down_size expanded_landmarks = (lower_lip_landmarks - center) * expansion_factor + center - mask_size = getattr(modules.globals, "mask_size", 1.0) # Default 1.0 - toplip_extension = mask_size * 0.5 - - # Define toplip indices relative to lower_lip_order (safer) - toplip_local_indices = [0, 1, 2, 3, 4, 5, 19] # Indices in lower_lip_order for [65, 66, 62, 70, 69, 18, 2] - - for idx in toplip_local_indices: - if idx < len(expanded_landmarks): # Boundary check - direction = expanded_landmarks[idx] - center - norm = np.linalg.norm(direction) - if norm > 1e-6: # Avoid division by zero - direction_normalized = direction / norm - expanded_landmarks[idx] += direction_normalized * toplip_extension - - # Define chin indices relative to lower_lip_order - chin_local_indices = [9, 10, 11, 12, 13, 14] # Indices for [22, 23, 24, 0, 8, 7] - chin_extension = 2 * 0.2 - - for idx in chin_local_indices: - if idx < len(expanded_landmarks): # Boundary check - # Extend vertically based on distance from center y - y_diff = expanded_landmarks[idx][1] - center[1] - expanded_landmarks[idx][1] += y_diff * chin_extension - - # Ensure landmarks are finite after adjustments if not np.all(np.isfinite(expanded_landmarks)): # print("Warning: Non-finite values detected after expanding landmarks.") @@ -1094,13 +1076,43 @@ def create_face_mask(face: Face, frame: Frame) -> np.ndarray: landmarks_int = landmarks.astype(np.int32) # Use standard face outline landmarks (0-32) - face_outline_points = landmarks_int[0:33] # Points 0 to 32 cover chin and sides + # Use standard face outline (0-32) + face_outline = landmarks_int[0:33] + # Estimate forehead points to ensure mask covers the whole face (including forehead) + # This is critical for Poisson blending to work correctly on the forehead + eyebrows = landmarks_int[33:43] + if eyebrows.shape[0] > 0: + chin = landmarks_int[16] + eyebrow_center = np.mean(eyebrows, axis=0) + + # Vector from chin to eyebrows (upwards) + up_vector = eyebrow_center - chin + norm = np.linalg.norm(up_vector) + if norm > 0: + up_vector /= norm + + # Extend upwards by 1.0 of the chin-to-eyebrow distance (aggressive coverage) + # This ensures the mask covers the entire forehead for proper blending + forehead_offset = up_vector * (norm * 1.0) + + # Shift eyebrows up to create forehead points + forehead_points = eyebrows + forehead_offset + + # Expand the top points slightly outwards to cover forehead corners + # Calculate the center of the new top points + top_center = np.mean(forehead_points, axis=0) + + # Expand outwards by 20% + forehead_points = (forehead_points - top_center) * 1.2 + top_center + + # Combine outline and forehead points + face_outline = np.concatenate((face_outline, forehead_points.astype(np.int32)), axis=0) # Calculate convex hull of these points # Use try-except as convexHull can fail on degenerate input try: - hull = cv2.convexHull(full_face_poly.astype(np.float32)) # Use float for accuracy + hull = cv2.convexHull(face_outline.astype(np.float32)) # Use float for accuracy if hull is None or len(hull) < 3: # print("Warning: Convex hull calculation failed or returned too few points.") # Fallback: use bounding box of landmarks? Or just return empty mask?