Logitech Muse for Apple Vision Pro Tracking, Action Buttons, and Haptics in RealityKit

AppModel.swift

import SwiftUI

@MainActor
@Observable
class AppModel {
    let immersiveSpaceID = "ImmersiveSpace"
    enum ImmersiveSpaceState {
        case closed
        case inTransition
        case open
    }
    var immersiveSpaceState = ImmersiveSpaceState.closed
}

ContentView.swift

import RealityKit
import SwiftUI

struct ContentView: View {
    @Environment(AppModel.self) private var appModel
    
    var body: some View {
        VStack(spacing: 24) {
            ToggleImmersiveSpaceButton()
            if appModel.immersiveSpaceState == .open {
                VStack(alignment: .leading, spacing: 12) {
                    HStack {
                        Circle().frame(width: 12).foregroundStyle(Color.white)
                        Text("Ready when white sphere appears")
                    }
                    
                    HStack {
                        Circle().frame(width: 12).foregroundStyle(Color.blue)
                        Text("Main button creates blue spheres")
                    }
                    
                    HStack {
                        Circle().frame(width: 12).foregroundStyle(Color.red)
                        Text("Secondary button creates red spheres")
                    }
                }
            } else {
                Text("Open immersive space to begin")
            }
        }
        .frame(width: 400, height: 400)
    }
}

struct ToggleImmersiveSpaceButton: View {
    @Environment(AppModel.self) private var appModel
    @Environment(\.dismissImmersiveSpace) private var dismissImmersiveSpace
    @Environment(\.openImmersiveSpace) private var openImmersiveSpace

    var body: some View {
        Button {
            Task { @MainActor in
                switch appModel.immersiveSpaceState {
                    case .open:
                        appModel.immersiveSpaceState = .inTransition
                        await dismissImmersiveSpace()
                    case .closed:
                        appModel.immersiveSpaceState = .inTransition
                        switch await openImmersiveSpace(id: appModel.immersiveSpaceID) {
                            case .opened:
                                break

                            case .userCancelled, .error:
                                fallthrough
                            @unknown default:
                                appModel.immersiveSpaceState = .closed
                        }

                    case .inTransition:
                        break
                }
            }
        } label: {
            Text(appModel.immersiveSpaceState == .open ? "Hide Immersive Space" : "Show Immersive Space")
        }
        .disabled(appModel.immersiveSpaceState == .inTransition)
        .animation(.none, value: 0)
        .fontWeight(.semibold)
    }
}

#Preview(windowStyle: .automatic) {
    ContentView()
        .environment(AppModel())
}

ImmersiveStylusExampleView.swift

import CoreHaptics
import GameController
import RealityKit
import SwiftUI

struct ImmersiveStylusExampleView: View {
    @State private var stylusManager = StylusManager()

    var body: some View {
        RealityView { content in
            let root = Entity()
            content.add(root)
            stylusManager.rootEntity = root
            await stylusManager.handleControllerSetup()
        }
        .task {
            // Don't forget to add the Accessory Tracking capability
            let configuration = SpatialTrackingSession.Configuration(tracking: [.accessory])
            let session = SpatialTrackingSession()
            await session.run(configuration)
        }
    }
}

@MainActor
final class StylusManager {
    var rootEntity: Entity?
    private var hapticEngines: [ObjectIdentifier: CHHapticEngine] = [:]
    private var hapticPlayers: [ObjectIdentifier: CHHapticPatternPlayer] = [:]

    func handleControllerSetup() async {
        // Existing connections
        let styluses = GCStylus.styli

        for stylus in styluses where stylus.productCategory == GCProductCategorySpatialStylus {
            try? await setupAccessory(stylus: stylus)
        }

        NotificationCenter.default.addObserver(
            forName: NSNotification.Name.GCStylusDidConnect, object: nil, queue: .main
        ) { [weak self] note in
            guard let self,
                  let stylus = note.object as? GCStylus,
                  stylus.productCategory == GCProductCategorySpatialStylus else { return }
            Task { @MainActor in
                try? await self.setupAccessory(stylus: stylus)
            }
        }
    }

    private func setupAccessory(stylus: GCStylus) async throws {
        guard let root = rootEntity else { return }
        let source = try await AnchoringComponent.AccessoryAnchoringSource(device: stylus)
        
        // List available locations (aim and origin appear to be possible)
        print("📍 Available locations: \(source.accessoryLocations)")

        guard let location = source.locationName(named: "aim") else { return }

        let anchor = AnchorEntity(
            .accessory(from: source, location: location),
            trackingMode: .predicted,
            physicsSimulation: .none
        )
        root.addChild(anchor)

        let key = ObjectIdentifier(stylus)

        // Setup haptics if available
        setupHaptics(for: stylus, key: key)
        setupStylusInputs(stylus: stylus, anchor: anchor, key: key)
        addStylusTipIndicator(to: anchor)
    }

    private func setupHaptics(for stylus: GCStylus, key: ObjectIdentifier) {
        guard let deviceHaptics = stylus.haptics else { return }
        
        // Create haptic engine
        let engine = deviceHaptics.createEngine(withLocality: .default)
        do {
            try engine?.start()
            hapticEngines[key] = engine
            
            // Create a simple "tap" pattern for button presses
            let pattern = try CHHapticPattern(events: [
                CHHapticEvent(eventType: .hapticTransient, parameters: [
                    CHHapticEventParameter(parameterID: .hapticIntensity, value: 0.8),
                    CHHapticEventParameter(parameterID: .hapticSharpness, value: 0.5)
                ], relativeTime: 0.0)
            ], parameters: [])
            
            let player = try engine?.makePlayer(with: pattern)
            hapticPlayers[key] = player
        } catch {
            print("❌ Failed to setup haptics: \(error)")
        }
    }
    
    private func playHaptic(for key: ObjectIdentifier) {
        guard let player = hapticPlayers[key] else { return }
        
        do {
            try player.start(atTime: CHHapticTimeImmediate)
        } catch {
            print("❌ Failed to play haptic: \(error)")
        }
    }

    private func addStylusTipIndicator(to anchor: AnchorEntity) {
        let tipSphere = ModelEntity(
            mesh: .generateSphere(radius: 0.003),
            materials: [SimpleMaterial(color: .white, isMetallic: false)]
        )
        anchor.addChild(tipSphere)
    }

    private func setupStylusInputs(stylus: GCStylus, anchor: AnchorEntity, key: ObjectIdentifier) {
        guard let input = stylus.input else { return }
        
        input.buttons[.stylusPrimaryButton]?.pressedInput.pressedDidChangeHandler = { [weak self] _, _, pressed in
            guard pressed, let self else { return }
            Task { @MainActor in
                self.playHaptic(for: key)
                self.spawnSphere(at: anchor, color: .systemBlue, radius: 0.02)
            }
        }
        
        input.buttons[.stylusSecondaryButton]?.pressedInput.pressedDidChangeHandler = { [weak self] _, _, pressed in
            guard pressed, let self else { return }
            Task { @MainActor in
                self.playHaptic(for: key)
                self.spawnSphere(at: anchor, color: .systemRed, radius: 0.01)
            }
        }
    }

    private func spawnSphere(at anchor: AnchorEntity, color: UIColor, radius: Float) {
        guard let root = rootEntity else { return }
        
        let worldTransform = anchor.transformMatrix(relativeTo: nil)
        let worldPosition = SIMD3<Float>(worldTransform.columns.3.x,
                                          worldTransform.columns.3.y,
                                          worldTransform.columns.3.z)
        
        let sphere = ModelEntity(
            mesh: .generateSphere(radius: radius),
            materials: [SimpleMaterial(color: color, isMetallic: false)]
        )
        sphere.position = worldPosition
        
        root.addChild(sphere)
    }
}

LogitechMusePlaygroundApp.swift

import SwiftUI

@main
struct LogitechMusePlaygroundApp: App {
    @State private var appModel = AppModel()
    
    var body: some Scene {
        WindowGroup {
            ContentView()
                .environment(appModel)
        }
        .windowResizability(.contentSize)

        ImmersiveSpace(id: appModel.immersiveSpaceID) {
            ImmersiveStylusExampleView()
                .onAppear { appModel.immersiveSpaceState = .open }
                .onDisappear { appModel.immersiveSpaceState = .closed }
        }
        .immersionStyle(selection: .constant(.mixed), in: .mixed)
     }
}

thanks for this - gave up on trying to get muse to work with unity and going with native.