fix: support kitty keyboard protocol for ctrl keybindings (#101)

Fixes #88

## Root cause

`MatchKey` treated `Ctrl+M` and the Enter key as the same raw byte
(`0x0d`). After editing the generated config, a `ctrl+m` binding could
shadow Enter, causing Return to toggle mode instead of executing the
selected command.

While investigating this, I also found that `MatchKey` only recognized
raw control bytes for `ctrl+<letter>` bindings. In Fish and terminals
that use the Kitty keyboard protocol (e.g. Kitty and Foot), those key
combinations may instead be sent as CSI sequences such as `\x1b[106;5u`,
preventing navigation bindings like `ctrl+j` / `ctrl+k` from being
matched.

## Changes

* `internal/config/keys.go`

* Reserve the Enter byte (`0x0d`) so it cannot be claimed by a `ctrl+m`
keybinding.
* Add `matchKittyCtrl()` to support Kitty keyboard protocol CSI
sequences for `ctrl+<letter>` bindings when the Ctrl modifier is
present.
* `root/wrapper.go`

* Extract `handleNavKey()` to remove duplicated navigation handling
logic.
* `internal/config/config_test.go`

* Add regression tests covering the Enter/`ctrl+m` fix and Kitty
keyboard protocol key sequences.

## Tested

* `go test ./...` — 201 passed
* Verified that Enter is no longer shadowed by `ctrl+m` after editing
the generated config.
* Verified that `ctrl+j` / `ctrl+k` navigation works correctly in Fish.

---------

Co-authored-by: verse91 <versedev.store@proton.me>
Co-authored-by: VERSE <sunnyone0901@gmail.com>

internal/config/config_test.go

@@ -289,6 +289,14 @@ func TestMatchKey(t *testing.T) {
 		{[]byte{0x09}, "tab", true, 1},
 		{[]byte{0x0d}, "enter", true, 1},
 		{[]byte{0x0d}, "ctrl+r", false, 0},
+		{[]byte("\x1b[106;4u"), "ctrl+j", true, 8},
+		{[]byte("\x1b[106;5u"), "ctrl+j", true, 8},
+		{[]byte("\x1b[107;4u"), "ctrl+k", true, 8},
+		{[]byte("\x1b[107;12u"), "ctrl+k", true, 9},
+		{[]byte("\x1b[106;1u"), "ctrl+j", false, 0},
+		{[]byte("\x1b[97;4u"), "ctrl+a", true, 7},
+		{[]byte("\x1b[106;4U"), "ctrl+j", false, 0},
+		{[]byte("\x1b[106u"), "ctrl+j", false, 0},
 	}
 
 	for _, tt := range tests {
@@ -298,3 +306,43 @@ func TestMatchKey(t *testing.T) {
 		}
 	}
 }
+
+// TestMatchKey_EnterReserved verifies that the Enter key (0x0d '\\r') can never
+// be claimed by another keybinding. In a raw terminal Ctrl+M and the
+// Enter/Return key are byte-identical (both 0x0d), so a "ctrl+m" keybinding
+// must not shadow Enter, otherwise line submission (the Enter key) breaks.
+func TestMatchKey_EnterReserved(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    []byte
+		expected string
+		matched  bool
+		consumed int
+	}{
+		{"ctrl+m must not match the Enter byte", []byte{0x0d}, "ctrl+m", false, 0},
+		{"ctrl+m must not match Ctrl+M as a generic binding", []byte{0x0d}, "ctrl+m", false, 0},
+		// other ctrl keys are still distinguishable from Enter and keep working
+		{"ctrl+n unaffected", []byte{0x0e}, "ctrl+n", true, 1},
+		{"ctrl+p unaffected", []byte{0x10}, "ctrl+p", true, 1},
+		{"ctrl+j (0x0a) remains a distinct binding", []byte{0x0a}, "ctrl+j", true, 1},
+	}
+
+	for _, tt := range tests {
+		m, c := MatchKey(tt.input, tt.expected)
+		if m != tt.matched || c != tt.consumed {
+			t.Errorf("%s: MatchKey(%v, %q) = (%v, %d); want (%v, %d)", tt.name, tt.input, tt.expected, m, c, tt.matched, tt.consumed)
+		}
+	}
+}
+
+// TestMatchKey_NavKeybindingsNoLongerHijackEnter is a regression guard: with a
+// user-configured navigation (or any) keybinding set to "ctrl+m", pressing the
+// Enter key must NOT be swallowed by that keybinding check.
+func TestMatchKey_NavKeybindingsNoLongerHijackEnter(t *testing.T) {
+	kb := []string{"ctrl+m", "ctrl+m", "<ctrl-m>", "CTRL+M"}
+	for _, expected := range kb {
+		if m, _ := MatchKey([]byte{0x0d}, expected); m {
+			t.Errorf("MatchKey(enter{0x0d}, %q) matched; Enter must remain reserved", expected)
+		}
+	}
+}

internal/config/keys.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"strconv"
 	"strings"
 )
 
@@ -29,9 +30,25 @@ func MatchKey(input []byte, expected string) (matched bool, consumed int) {
 		char := expected[5]
 		if char >= 'a' && char <= 'z' {
 			targetByte := char - 'a' + 1
+			// 0x0d ('m') is the Carriage Return byte. In a raw terminal the
+			// Enter/Return key and Ctrl+M both arrive as 0x0d, so they are
+			// indistinguishable. Matching a "ctrl+m" keybinding here would let
+			// it shadow the Enter key and break line submission (the wrapper
+			// checks keybindings before its enter handler). Reserve this byte
+			// for Enter so a "ctrl+m" keybinding can never hijack it.
+			if targetByte == 0x0d {
+				return false, 0
+			}
 			if input[0] == targetByte {
 				return true, 1
 			}
+		// Some terminals (foot, kitty, etc.) send kitty keyboard protocol
+		// escape sequences for Ctrl+letter instead of raw control bytes:
+		// CSI <keycode> ; <modifiers> <action>
+		// where modifiers include Ctrl=4 and action='u' means press.
+		if matched, consumed := matchKittyCtrl(input, int(char)); matched {
+			return matched, consumed
+		}
 		}
 	}
 
@@ -75,6 +92,51 @@ func MatchKey(input []byte, expected string) (matched bool, consumed int) {
 	return false, 0
 }
 
+// matchKittyCtrl matches the kitty keyboard protocol CSI sequence for Ctrl+<letter>.
+// Format: ESC [ <keycode> ; <modifiers> <action>
+// For Ctrl+<letter>: keycode is the ASCII code of the letter, modifiers must have bit 2 (value 4) set, action is 'u'.
+func matchKittyCtrl(input []byte, expectedASCII int) (matched bool, consumed int) {
+	if len(input) < 6 || input[0] != 0x1b || input[1] != '[' {
+		return false, 0
+	}
+
+	uIdx := -1
+	for i := 2; i < len(input); i++ {
+		if input[i] == 'u' {
+			uIdx = i
+			break
+		}
+	}
+	if uIdx == -1 {
+		return false, 0
+	}
+
+	body := string(input[2:uIdx])
+	parts := strings.Split(body, ";")
+	if len(parts) != 2 {
+		return false, 0
+	}
+
+	keycode, err := strconv.Atoi(parts[0])
+	if err != nil {
+		return false, 0
+	}
+	modifiers, err := strconv.Atoi(parts[1])
+	if err != nil {
+		return false, 0
+	}
+
+	if modifiers&4 == 0 {
+		return false, 0
+	}
+
+	if keycode != expectedASCII {
+		return false, 0
+	}
+
+	return true, uIdx + 1
+}
+
 // FormatKeyName takes a config key string like "ctrl+r" and formats it for UI display, e.g. "<Ctrl+R>".
 func FormatKeyName(key string) string {
 	parts := strings.Split(key, "+")

root/wrapper.go

@@ -128,6 +128,7 @@ func runWrapper() {
 	var bufferMu sync.Mutex
 	var userNavigated atomic.Bool
 	var renderMenuNow func()
+	var intercepted bool
 
 	r, w, err := os.Pipe() // pipe for ipc communication from shell to iris
 	if err != nil {
@@ -311,6 +312,92 @@ func runWrapper() {
 		return pgrp != shellPGID
 	}
 
+	// Shared handler for configured navigation keys (e.g. ctrl+j / ctrl+k).
+	// Moves the overlay cursor when visible, otherwise opens history/spec
+	// list and selects the next item in the requested direction.
+	handleNavKey := func(dir string) {
+		intercepted = true
+		userNavigated.Store(true)
+
+		if overlay.IsVisible() {
+			arrowDir := "down"
+			if dir == "up" {
+				arrowDir = "up"
+			}
+			moved, selectedCmd := overlay.MoveCursor(arrowDir)
+			if !moved {
+				return
+			}
+
+			bufferMu.Lock()
+			activeModeMu.RLock()
+			isHistMode := activeMode == "history"
+			activeModeMu.RUnlock()
+			var toWrite []byte
+			if isHistMode && selectedCmd != "" {
+				naiveBuffer = selectedCmd
+				cursorOffset = 0
+				toWrite = append([]byte{0x15}, selectedCmd...)
+			}
+			bufCopy := naiveBuffer
+			offsetCopy := cursorOffset
+			bufferMu.Unlock()
+
+			if len(toWrite) > 0 {
+				_, _ = ptmx.Write(toWrite)
+			}
+
+			var b strings.Builder
+			if !disableGhostText.Load() {
+				b.WriteString(overlay.RenderGhostText(bufCopy, true, offsetCopy == 0))
+			}
+			b.WriteString(overlay.Render())
+			writeStdout([]byte(b.String()))
+		} else {
+			activeModeMu.Lock()
+			if activeMode == "" {
+				activeMode = loadMode()
+			}
+			activeModeMu.Unlock()
+
+			activeModeMu.RLock()
+			currentMode := activeMode
+			activeModeMu.RUnlock()
+
+			bufferMu.Lock()
+			bufQuery := naiveBuffer
+			bufferMu.Unlock()
+
+			results := MergeResults(bufQuery, currentMode)
+			if len(results) > 0 {
+				limit := min(len(results), 100)
+				var historyList []spec.Suggestion
+
+				if dir == "up" {
+					for j := limit - 1; j >= 0; j-- {
+						historyList = append(historyList, results[j])
+					}
+				} else {
+					for j := range limit {
+						historyList = append(historyList, results[j])
+					}
+				}
+
+				selected := overlay.SetHistoryList(historyList, dir == "up")
+				if selected != "" {
+					bufferMu.Lock()
+					naiveBuffer = selected
+					cursorOffset = 0
+					bufferMu.Unlock()
+
+					userNavigated.Store(true)
+					writeStdout([]byte(overlay.Render()))
+					_, _ = ptmx.Write(append([]byte{0x15}, selected...))
+				}
+			}
+		}
+	}
+
 	// bridge pty output to actual stdout
 	go func() {
 		defer func() {
@@ -654,17 +741,14 @@ func runWrapper() {
 				continue
 			}
 
-			logger.Debugf("Stdin raw input: bytes=%q, hex=%x", inputSlice[:n], inputSlice[:n])
-
 			shouldOverlayDraw := false
 			for i := 0; i < n; i++ {
 				b := inputSlice[i]
-				intercepted := false
+				intercepted = false
 
 			if matched, consumed := config.MatchKey(inputSlice[i:], config.Get().Keybindings.ToggleMenu); matched {
 				intercepted = true
 				suggestionsEnabled = !suggestionsEnabled
-					logger.Debugf("Intercepted ToggleMenu, suggestionsEnabled=%v", suggestionsEnabled)
 				if !suggestionsEnabled {
 					writeStdout([]byte(overlay.ClearAndDisable()))
 				} else {
@@ -685,7 +769,6 @@ func runWrapper() {
 				}
 			saveMode(activeMode)
 			activeModeMu.Unlock()
-					logger.Debugf("Intercepted Ctrl+R, toggled mode to %q", activeMode)
 			if userNavigated.Load() {
 						bufferMu.Lock()
 						naiveBuffer = overlay.GetTypedQuery()
@@ -706,96 +789,15 @@ func runWrapper() {
 			}
 
 			if isNavUp || isNavDown {
-					if overlay.IsVisible() {
-						intercepted = true
-						userNavigated.Store(true)
-
 				arrowDir := "down"
 				if isNavUp {
 					arrowDir = "up"
 				}
-						moved, selectedCmd := overlay.MoveCursor(arrowDir)
+				handleNavKey(arrowDir)
-						if !moved {
 				i += navConsumed - 1
 				continue
 			}
 
-						bufferMu.Lock()
-						activeModeMu.RLock()
-						isHistMode := activeMode == "history"
-						activeModeMu.RUnlock()
-						var toWrite []byte
-						if isHistMode && selectedCmd != "" {
-							naiveBuffer = selectedCmd
-							cursorOffset = 0
-							toWrite = append([]byte{0x15}, selectedCmd...)
-						}
-						bufCopy := naiveBuffer
-						offsetCopy := cursorOffset
-						bufferMu.Unlock()
-
-						if len(toWrite) > 0 {
-							_, _ = ptmx.Write(toWrite)
-						}
-
-						var b strings.Builder
-						if !disableGhostText.Load() {
-							b.WriteString(overlay.RenderGhostText(bufCopy, true, offsetCopy == 0))
-						}
-						b.WriteString(overlay.Render())
-						writeStdout([]byte(b.String()))
-
-						i += navConsumed - 1
-						continue
-					} else {
-						// up/down arrow or navigation key when overlay is closed
-						intercepted = true
-						activeModeMu.Lock()
-						if activeMode == "" {
-							activeMode = loadMode()
-						}
-						activeModeMu.Unlock()
-
-						activeModeMu.RLock()
-						currentMode := activeMode
-						activeModeMu.RUnlock()
-
-						bufferMu.Lock()
-						bufQuery := naiveBuffer
-						bufferMu.Unlock()
-
-						results := MergeResults(bufQuery, currentMode)
-						if len(results) > 0 {
-							limit := min(len(results), 100)
-							var historyList []spec.Suggestion
-
-							if isNavUp {
-								for j := limit - 1; j >= 0; j-- {
-									historyList = append(historyList, results[j])
-								}
-							} else {
-								for j := range limit {
-									historyList = append(historyList, results[j])
-								}
-							}
-
-							selected := overlay.SetHistoryList(historyList, isNavUp)
-							if selected != "" {
-								bufferMu.Lock()
-								naiveBuffer = selected
-								cursorOffset = 0
-								bufferMu.Unlock()
-
-								userNavigated.Store(true)
-								writeStdout([]byte(overlay.Render()))
-								_, _ = ptmx.Write(append([]byte{0x15}, selected...))
-							}
-						}
-						i += navConsumed - 1
-						continue
-					}
-				}
-
 			if matched, consumed := config.MatchKey(inputSlice[i:], config.Get().Keybindings.SelectSuggestion); matched && config.Get().Keybindings.SelectSuggestion != "" {
 				if overlay.IsVisible() {
 					intercepted = true
@@ -830,7 +832,6 @@ func runWrapper() {
 
 			if b == 0x0d || b == 0x0a { // enter
 				intercepted = true
-					logger.Debugf("Intercepted Enter key")
 
 				if b == 0x0d && i+1 < n && inputSlice[i+1] == 0x0a {
 						i++ // consume trailing \n in \r\n to prevent matching ctrl+j
@@ -919,6 +920,22 @@ func runWrapper() {
 							continue
 						}
 					}
+
+					isNavUp, navConsumed := config.MatchKey(inputSlice[i:], config.Get().Keybindings.NavigateUp)
+					if !isNavUp {
+						isNavDown, navConsumed = config.MatchKey(inputSlice[i:], config.Get().Keybindings.NavigateDown)
+					}
+				if isNavUp || isNavDown {
+					intercepted = true
+					arrowDir := "down"
+					if isNavUp {
+						arrowDir = "up"
+					}
+					handleNavKey(arrowDir)
+						i += navConsumed - 1
+						continue
+					}
+
 					// handle escape sequences like arrow keys and functional shortcuts
 					// left/right arrow cursor tracking
 					isLeftRightArrow := false
@@ -963,7 +980,6 @@ func runWrapper() {
 
 						if len(ghostText) > 0 {
 							intercepted = true
-								logger.Debugf("Intercepted Right Arrow (accepted ghost text: %q)", ghostText)
 							bufferMu.Lock()
 								naiveBuffer += ghostText
 								cursorOffset = 0
@@ -993,7 +1009,6 @@ func runWrapper() {
 						writeStdout([]byte(overlay.ClearAndDisable()))
 						disableGhostText.Store(true)
 
-						// If it's a standalone ESC (n==1), don't clear the buffer because the user just wanted to hide the menu or enter vi-mode
 						isStandaloneEsc := n == 1 && b == '\033'
 						if !isLeftRightArrow && !isStandaloneEsc {
 							bufferMu.Lock()
@@ -1003,7 +1018,6 @@ func runWrapper() {
 						}
 
 						_, _ = ptmx.Write([]byte{b})
-						// skip remaining bytes of the escape sequence to avoid misinterpretation
 						for j := i + 1; j < n; j++ {
 							char := inputSlice[j]
 							_, _ = ptmx.Write([]byte{char})