PixelPilot: Instructions file for GitHub Copilot

Voice UI & Audio Feedback

Voice is the most natural interface. This file covers Web Speech API, voice commands, spoken feedback, and accessibility for voice-first users. Not just "add a microphone button" — proper voice UX.

---

CRITICAL RULES

1. Always visual fallback — Voice is enhancement, not requirement.
2. Permission before listening — Never auto-activate microphone.
3. Clear feedback — User must know when mic is hot.
4. Privacy first — Process locally when possible (no cloud for simple commands).
5. Respect prefers-reduced-motion — Voice waveform animations optional.

---

WEB SPEECH API SUPPORT

// Feature detection
const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
const SpeechSynthesis = window.speechSynthesis;

export const voiceSupport = {
  recognition: !!SpeechRecognition,
  synthesis: !!SpeechSynthesis,
  
  // Check if browser is actively blocking
  async checkPermission() {
    try {
      const result = await navigator.permissions.query({ name: 'microphone' });
      return result.state; // 'granted', 'denied', 'prompt'
    } catch {
      return 'unknown';
    }
  }
};

---

VOICE COMMAND SYSTEM

// useVoiceCommands.ts
import { useState, useEffect, useCallback, useRef } from 'react';

interface VoiceCommand {
  phrase: string | RegExp;
  action: (transcript: string, match?: RegExpMatchArray) => void;
  description: string; // For help/accessibility
}

interface UseVoiceCommandsOptions {
  commands: VoiceCommand[];
  continuous?: boolean;
  language?: string;
  onError?: (error: string) => void;
  onListeningChange?: (isListening: boolean) => void;
}

export function useVoiceCommands({
  commands,
  continuous = false,
  language = 'en-US',
  onError,
  onListeningChange,
}: UseVoiceCommandsOptions) {
  const [isListening, setIsListening] = useState(false);
  const [transcript, setTranscript] = useState('');
  const [confidence, setConfidence] = useState(0);
  const recognitionRef = useRef<SpeechRecognition | null>(null);

  useEffect(() => {
    const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
    if (!SpeechRecognition) return;

    const recognition = new SpeechRecognition();
    recognition.continuous = continuous;
    recognition.interimResults = true;
    recognition.lang = language;

    recognition.onresult = (event) => {
      const last = event.results[event.results.length - 1];
      const text = last[0].transcript.toLowerCase().trim();
      setTranscript(text);
      setConfidence(last[0].confidence);

      if (last.isFinal) {
        // Match against commands
        for (const cmd of commands) {
          if (typeof cmd.phrase === 'string') {
            if (text.includes(cmd.phrase.toLowerCase())) {
              cmd.action(text);
              break;
            }
          } else {
            const match = text.match(cmd.phrase);
            if (match) {
              cmd.action(text, match);
              break;
            }
          }
        }
      }
    };

    recognition.onerror = (event) => {
      onError?.(event.error);
      setIsListening(false);
    };

    recognition.onend = () => {
      setIsListening(false);
      onListeningChange?.(false);
    };

    recognitionRef.current = recognition;

    return () => {
      recognition.abort();
    };
  }, [commands, continuous, language, onError, onListeningChange]);

  const startListening = useCallback(async () => {
    if (!recognitionRef.current) {
      onError?.('Speech recognition not supported');
      return;
    }

    try {
      // Request microphone permission
      await navigator.mediaDevices.getUserMedia({ audio: true });
      recognitionRef.current.start();
      setIsListening(true);
      onListeningChange?.(true);
    } catch (err) {
      onError?.('Microphone permission denied');
    }
  }, [onError, onListeningChange]);

  const stopListening = useCallback(() => {
    recognitionRef.current?.stop();
    setIsListening(false);
    onListeningChange?.(false);
  }, [onListeningChange]);

  return {
    isListening,
    transcript,
    confidence,
    startListening,
    stopListening,
    isSupported: !!recognitionRef.current,
  };
}