From ab46586ece366772faae883678c921bb0e8fcc74 Mon Sep 17 00:00:00 2001 From: jared Date: Sat, 24 Jan 2026 12:45:08 -0500 Subject: [PATCH] Complete remaining high-priority features - Edge refinement: Wire toggle to actually snap brush strokes to edges using Sobel gradient analysis in EdgeRefinement.swift - Brush preview circle: Show visual cursor following finger during drawing - PHAsset storage: Capture localIdentifier for Photo Library imports - Low-confidence mask warning: Show "Does this look right?" for uncertain detections based on mask coverage and edge sharpness analysis - Fix Swift 6 concurrency warnings with nonisolated static methods Co-Authored-By: Claude Opus 4.5 --- .../Features/Editor/BrushCanvasView.swift | 83 +++++++++++++++---- .../Features/Editor/EditorViewModel.swift | 41 +++++++-- .../Features/Editor/PhotoEditorView.swift | 25 +++++- CheapRetouch/Services/MaskingService.swift | 78 ++++++++++++++++- CheapRetouch/Utilities/EdgeRefinement.swift | 37 ++------- 5 files changed, 207 insertions(+), 57 deletions(-) diff --git a/CheapRetouch/Features/Editor/BrushCanvasView.swift b/CheapRetouch/Features/Editor/BrushCanvasView.swift index dba468d..c88129b 100644 --- a/CheapRetouch/Features/Editor/BrushCanvasView.swift +++ b/CheapRetouch/Features/Editor/BrushCanvasView.swift @@ -16,6 +16,8 @@ struct BrushCanvasView: View { @State private var currentStroke: [CGPoint] = [] @State private var allStrokes: [[CGPoint]] = [] @State private var isErasing = false + @State private var currentTouchLocation: CGPoint? + @State private var gradientImage: EdgeRefinement.GradientImage? var body: some View { Canvas { context, size in @@ -28,11 +30,46 @@ struct BrushCanvasView: View { if !currentStroke.isEmpty { drawStroke(currentStroke, in: &context, color: isErasing ? .black : .white) } + + // Draw brush preview circle at current touch location + if let location = currentTouchLocation { + let previewRect = CGRect( + x: location.x - viewModel.brushSize / 2, + y: location.y - viewModel.brushSize / 2, + width: viewModel.brushSize, + height: viewModel.brushSize + ) + context.stroke( + Path(ellipseIn: previewRect), + with: .color(.white.opacity(0.8)), + lineWidth: 2 + ) + } } .gesture(drawingGesture) .overlay(alignment: .bottom) { brushControls } + .onAppear { + // Precompute gradient for edge refinement if enabled + if viewModel.useEdgeRefinement, let image = viewModel.displayImage { + computeGradientAsync(from: image) + } + } + .onChange(of: viewModel.useEdgeRefinement) { _, newValue in + if newValue, gradientImage == nil, let image = viewModel.displayImage { + computeGradientAsync(from: image) + } + } + } + + private func computeGradientAsync(from image: CGImage) { + Task { + let gradient = await Task.detached(priority: .userInitiated) { + EdgeRefinement.computeGradient(from: image) + }.value + gradientImage = gradient + } } private func drawStroke(_ points: [CGPoint], in context: inout GraphicsContext, color: Color) { @@ -69,12 +106,15 @@ struct BrushCanvasView: View { DragGesture(minimumDistance: 0) .onChanged { value in let point = value.location + currentTouchLocation = point + // Only add points within the image bounds if displayedImageFrame.contains(point) { currentStroke.append(point) } } .onEnded { _ in + currentTouchLocation = nil if !currentStroke.isEmpty { allStrokes.append(currentStroke) currentStroke = [] @@ -131,6 +171,32 @@ struct BrushCanvasView: View { private func applyBrushMask() async { guard !allStrokes.isEmpty else { return } + let scaleX = imageSize.width / displayedImageFrame.width + let scaleY = imageSize.height / displayedImageFrame.height + + // Convert all strokes to image coordinates + var imageCoordStrokes: [[CGPoint]] = [] + for stroke in allStrokes { + let imageStroke = stroke.map { point in + CGPoint( + x: (point.x - displayedImageFrame.minX) * scaleX, + y: (point.y - displayedImageFrame.minY) * scaleY + ) + } + imageCoordStrokes.append(imageStroke) + } + + // Apply edge refinement if enabled and gradient is available + if viewModel.useEdgeRefinement, let gradient = gradientImage { + imageCoordStrokes = imageCoordStrokes.map { stroke in + EdgeRefinement.refineSelectionToEdges( + selection: stroke, + gradient: gradient, + searchRadius: Int(viewModel.brushSize / 2) + ) + } + } + // Create mask image from strokes let renderer = UIGraphicsImageRenderer(size: imageSize) let maskImage = renderer.image { ctx in @@ -141,25 +207,14 @@ struct BrushCanvasView: View { // Draw strokes in white (masked areas) UIColor.white.setStroke() - let scaleX = imageSize.width / displayedImageFrame.width - let scaleY = imageSize.height / displayedImageFrame.height - - for stroke in allStrokes { + for stroke in imageCoordStrokes { guard stroke.count >= 2 else { continue } let path = UIBezierPath() - let firstPoint = CGPoint( - x: (stroke[0].x - displayedImageFrame.minX) * scaleX, - y: (stroke[0].y - displayedImageFrame.minY) * scaleY - ) - path.move(to: firstPoint) + path.move(to: stroke[0]) for i in 1.. 48_000_000 { + errorMessage = "Very large image (48MP+). May cause memory issues." + } else if pixelCount > 12_000_000 { + // Just a note, not an error - processing continues + } + + // Create new project with appropriate image source + let imageSource: Project.ImageSource + if let identifier = localIdentifier { + imageSource = .photoLibrary(localIdentifier: identifier) + } else { + let imageData = uiImage.jpegData(compressionQuality: 0.9) ?? Data() + imageSource = .embedded(data: imageData) + } + project = Project(imageSource: imageSource) + + announceForVoiceOver("Photo loaded") } // MARK: - Tap Handling @@ -120,11 +140,11 @@ final class EditorViewModel { } private func handlePersonTap(at point: CGPoint, in image: CGImage) async throws { - let (mask, peopleCount) = try await maskingService.generatePersonMaskWithCount(at: point, in: image) + let result = try await maskingService.generatePersonMaskWithCount(at: point, in: image) - detectedPeopleCount = peopleCount + detectedPeopleCount = result.instanceCount - guard let mask = mask else { + guard let mask = result.mask else { errorMessage = "No person found at tap location" return } @@ -134,7 +154,10 @@ final class EditorViewModel { maskPreview = dilatedMask ?? mask showingMaskConfirmation = true - showSelectAllPeople = peopleCount > 1 + showSelectAllPeople = result.instanceCount > 1 + + // Check for low confidence and flag for user warning + isLowConfidenceMask = result.confidence < 0.7 } func selectAllPeople() async { @@ -396,6 +419,7 @@ final class EditorViewModel { maskPreview = nil showingMaskConfirmation = false showSelectAllPeople = false + isLowConfidenceMask = false isProcessing = false processingMessage = "" } @@ -405,6 +429,7 @@ final class EditorViewModel { showingMaskConfirmation = false showSelectAllPeople = false pendingRefineMask = nil + isLowConfidenceMask = false } func refineWithBrush() { diff --git a/CheapRetouch/Features/Editor/PhotoEditorView.swift b/CheapRetouch/Features/Editor/PhotoEditorView.swift index 9f9f375..7c2fc76 100644 --- a/CheapRetouch/Features/Editor/PhotoEditorView.swift +++ b/CheapRetouch/Features/Editor/PhotoEditorView.swift @@ -86,9 +86,14 @@ struct PhotoEditorView: View { } .onChange(of: selectedItem) { oldValue, newValue in Task { - if let data = try? await newValue?.loadTransferable(type: Data.self), + guard let item = newValue else { return } + + // Get localIdentifier if available + let localIdentifier = item.itemIdentifier + + if let data = try? await item.loadTransferable(type: Data.self), let uiImage = UIImage(data: data) { - viewModel.loadImage(uiImage) + viewModel.loadImage(uiImage, localIdentifier: localIdentifier) } } } @@ -144,6 +149,22 @@ struct PhotoEditorView: View { private var maskConfirmationBar: some View { VStack(spacing: 12) { + // Low confidence warning + if viewModel.isLowConfidenceMask { + HStack { + Image(systemName: "exclamationmark.triangle.fill") + .foregroundStyle(.yellow) + Text("Does this look right? The selection may need refinement.") + .font(.caption) + .foregroundStyle(.secondary) + } + .padding(.horizontal, 12) + .padding(.vertical, 8) + .background(Color.yellow.opacity(0.1), in: RoundedRectangle(cornerRadius: 8)) + .accessibilityElement(children: .combine) + .accessibilityLabel("Warning: Low confidence detection. The selection may need refinement.") + } + // High contrast toggle for accessibility HStack { Toggle(isOn: $viewModel.useHighContrastMask) { diff --git a/CheapRetouch/Services/MaskingService.swift b/CheapRetouch/Services/MaskingService.swift index bdbd9a0..a4b0419 100644 --- a/CheapRetouch/Services/MaskingService.swift +++ b/CheapRetouch/Services/MaskingService.swift @@ -36,7 +36,13 @@ actor MaskingService { try await generateForegroundMask(at: point, in: image) } - func generatePersonMaskWithCount(at point: CGPoint, in image: CGImage) async throws -> (CGImage?, Int) { + struct MaskResult { + let mask: CGImage? + let instanceCount: Int + let confidence: Float // 0.0 to 1.0, based on mask coverage relative to image + } + + func generatePersonMaskWithCount(at point: CGPoint, in image: CGImage) async throws -> MaskResult { let request = VNGenerateForegroundInstanceMaskRequest() let handler = VNImageRequestHandler(cgImage: image, options: [:]) @@ -48,7 +54,7 @@ actor MaskingService { } guard let result = request.results?.first else { - return (nil, 0) + return MaskResult(mask: nil, instanceCount: 0, confidence: 0) } let allInstances = result.allInstances @@ -73,11 +79,75 @@ actor MaskingService { } guard let instance = targetInstance else { - return (nil, instanceCount) + return MaskResult(mask: nil, instanceCount: instanceCount, confidence: 0) } let maskPixelBuffer = try result.generateScaledMaskForImage(forInstances: instance, from: handler) - return (convertPixelBufferToCGImage(maskPixelBuffer), instanceCount) + let maskImage = convertPixelBufferToCGImage(maskPixelBuffer) + + // Calculate confidence based on mask quality (coverage ratio) + let confidence = calculateMaskConfidence(maskPixelBuffer, imageSize: CGSize(width: image.width, height: image.height)) + + return MaskResult(mask: maskImage, instanceCount: instanceCount, confidence: confidence) + } + + private func calculateMaskConfidence(_ pixelBuffer: CVPixelBuffer, imageSize: CGSize) -> Float { + CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly) + defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly) } + + let width = CVPixelBufferGetWidth(pixelBuffer) + let height = CVPixelBufferGetHeight(pixelBuffer) + + guard let baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer) else { + return 0.5 // Default medium confidence if can't read + } + + let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer) + var maskPixelCount = 0 + var edgePixelCount = 0 + + // Count mask pixels and check edge sharpness + for y in 0.. 127 { + maskPixelCount += 1 + } + + // Check for edge pixels (values between 50-200 indicate soft edges) + if pixelValue > 50 && pixelValue < 200 { + edgePixelCount += 1 + } + } + } + + let totalPixels = width * height + let maskRatio = Float(maskPixelCount) / Float(totalPixels) + let edgeRatio = Float(edgePixelCount) / Float(max(1, maskPixelCount)) + + // Confidence is higher when: + // - Mask covers reasonable portion (not too small, not too large) + // - Edge pixels are minimal (sharp edges = confident detection) + var confidence: Float = 1.0 + + // Penalize very small masks (< 1% of image) + if maskRatio < 0.01 { + confidence -= 0.3 + } + + // Penalize very large masks (> 50% of image) + if maskRatio > 0.5 { + confidence -= 0.2 + } + + // Penalize fuzzy edges + if edgeRatio > 0.3 { + confidence -= 0.2 + } + + return max(0.0, min(1.0, confidence)) } func generateForegroundMask(at point: CGPoint, in image: CGImage) async throws -> CGImage? { diff --git a/CheapRetouch/Utilities/EdgeRefinement.swift b/CheapRetouch/Utilities/EdgeRefinement.swift index 1246482..ea906de 100644 --- a/CheapRetouch/Utilities/EdgeRefinement.swift +++ b/CheapRetouch/Utilities/EdgeRefinement.swift @@ -10,9 +10,9 @@ import CoreGraphics import Accelerate import UIKit -struct EdgeRefinement { +struct EdgeRefinement: Sendable { - struct GradientImage { + struct GradientImage: Sendable { let width: Int let height: Int let magnitude: [Float] @@ -20,12 +20,12 @@ struct EdgeRefinement { let directionY: [Float] } - static func computeGradient(from image: CGImage) -> GradientImage? { + nonisolated static func computeGradient(from image: CGImage) -> GradientImage? { let width = image.width let height = image.height // Convert to grayscale - guard let grayscale = convertToGrayscale(image) else { + guard let grayscale = convertToGrayscaleNonisolated(image) else { return nil } @@ -38,28 +38,7 @@ struct EdgeRefinement { let sobelX: [Int16] = [-1, 0, 1, -2, 0, 2, -1, 0, 1] let sobelY: [Int16] = [-1, -2, -1, 0, 0, 0, 1, 2, 1] - var sourceBuffer = vImage_Buffer( - data: UnsafeMutableRawPointer(mutating: grayscale), - height: vImagePixelCount(height), - width: vImagePixelCount(width), - rowBytes: width - ) - - var destBufferX = vImage_Buffer( - data: &gradientX, - height: vImagePixelCount(height), - width: vImagePixelCount(width), - rowBytes: width * MemoryLayout.size - ) - - var destBufferY = vImage_Buffer( - data: &gradientY, - height: vImagePixelCount(height), - width: vImagePixelCount(width), - rowBytes: width * MemoryLayout.size - ) - - // Apply Sobel filters (simplified - using direct calculation) + // Apply Sobel filters using direct calculation for y in 1..<(height - 1) { for x in 1..<(width - 1) { var gx: Float = 0 @@ -90,7 +69,7 @@ struct EdgeRefinement { ) } - static func refineSelectionToEdges( + nonisolated static func refineSelectionToEdges( selection: [CGPoint], gradient: GradientImage, searchRadius: Int = 5 @@ -131,7 +110,7 @@ struct EdgeRefinement { return refinedPoints } - static func createMaskFromPoints( + nonisolated static func createMaskFromPoints( _ points: [CGPoint], brushSize: CGFloat, imageSize: CGSize @@ -169,7 +148,7 @@ struct EdgeRefinement { return context.makeImage() } - private static func convertToGrayscale(_ image: CGImage) -> [UInt8]? { + private nonisolated static func convertToGrayscaleNonisolated(_ image: CGImage) -> [UInt8]? { let width = image.width let height = image.height var pixelData = [UInt8](repeating: 0, count: width * height)