Make the tray menu close, and its window come forward

Swing never grabs the pointer for a popup, so a click on the desktop or
another application is not seen and the menu stays up. The little window
the menu hangs off is focusable now and closes it when the focus goes
elsewhere, Escape closes it, and clicking the icon again toggles it.

Raising the window was ignored for the same reason it usually is: a
window manager refuses a raise from an application that does not have the
focus. The icon asks for _NET_ACTIVE_WINDOW instead — the request meant
for pagers and trays acting for the user — and falls back to Swing's
always-on-top shuffle when it cannot find the window. Windows are matched
by _NET_WM_PID and _NET_WM_NAME, since WM_NAME cannot carry the em dash
in our title.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

ts3-client/swing/src/main/java/com/ts3client/ui/TrayController.java

@@ -4,9 +4,11 @@ import com.ts3client.ui.tray.AwtTray;
 import com.ts3client.ui.tray.TrayBackend;
 import com.ts3client.ui.tray.X11Tray;
 
+import javax.swing.JComponent;
+import javax.swing.JDialog;
 import javax.swing.JMenuItem;
 import javax.swing.JPopupMenu;
-import javax.swing.JWindow;
+import javax.swing.KeyStroke;
 import javax.swing.SwingUtilities;
 import javax.swing.event.PopupMenuEvent;
 import javax.swing.event.PopupMenuListener;
@@ -14,6 +16,10 @@ import java.awt.Frame;
 import java.awt.Image;
 import java.awt.MenuItem;
 import java.awt.PopupMenu;
+import java.awt.Window;
+import java.awt.event.KeyEvent;
+import java.awt.event.WindowAdapter;
+import java.awt.event.WindowEvent;
 import java.util.function.IntFunction;
 
 /**
@@ -37,6 +43,10 @@ final class TrayController {
     private SelfState state;
     private String tooltip = "";
 
+    /** The context menu while it is on screen, and the window it hangs off. */
+    private JPopupMenu openMenu;
+    private JDialog openAnchor;
+
     TrayController(MainFrame frame, Runnable onQuit) {
         this.frame = frame;
         this.onQuit = onQuit;
@@ -116,12 +126,38 @@ final class TrayController {
     /**
      * The same menu for backends that have none — a Swing popup needs a component to
      * hang off, so it is given an empty window at the pointer.
+     *
+     * <p>Swing dismisses a popup from events it sees itself, and it sees none of the
+     * clicks that land on the desktop or another application, so the little window is
+     * made focusable and the menu closed as soon as it loses the focus. Escape and a
+     * second click on the icon close it too.
      */
     private void showMenu(int x, int y) {
-        JWindow anchor = new JWindow(frame);
+        if (openMenu != null) { // a second click on the icon: close it again
+            hideMenu();
+            return;
+        }
+        JDialog anchor = new JDialog(frame);
+        anchor.setUndecorated(true);
+        anchor.setFocusableWindowState(true);
+        anchor.setAlwaysOnTop(true);
         anchor.setLocation(x, y);
         anchor.setSize(1, 1);
         anchor.setVisible(true);
+        anchor.toFront();
+        anchor.requestFocus();
+        anchor.addWindowFocusListener(new WindowAdapter() {
+            @Override
+            public void windowLostFocus(WindowEvent e) {
+                // The menu's own popup window may take the focus instead; only the
+                // focus moving away from the menu altogether closes it.
+                if (!ownedBy(e.getOppositeWindow(), anchor)) hideMenu();
+            }
+        });
+
+        anchor.getRootPane().registerKeyboardAction(e -> hideMenu(),
+                KeyStroke.getKeyStroke(KeyEvent.VK_ESCAPE, 0),
+                JComponent.WHEN_IN_FOCUSED_WINDOW);
 
         JPopupMenu menu = new JPopupMenu();
         JMenuItem showItem = new JMenuItem("Show TS3J", Icons.app());
@@ -138,22 +174,58 @@ final class TrayController {
 
             @Override
             public void popupMenuWillBecomeInvisible(PopupMenuEvent e) {
-                anchor.dispose();
+                closed(anchor);
             }
 
             @Override
             public void popupMenuCanceled(PopupMenuEvent e) {
-                anchor.dispose();
+                closed(anchor);
             }
         });
+        openMenu = menu;
+        openAnchor = anchor;
         menu.show(anchor, 0, 0);
     }
 
-    /** Brings the window back from the taskbar or an iconified state. */
+    /** @return whether {@code window} is {@code owner} or one of its child windows */
+    private static boolean ownedBy(Window window, Window owner) {
+        for (Window w = window; w != null; w = w.getOwner()) {
+            if (w == owner) return true;
+        }
+        return false;
+    }
+
+    private void hideMenu() {
+        if (openMenu != null) openMenu.setVisible(false); // the listener cleans up
+        closed(openAnchor);
+    }
+
+    private void closed(JDialog anchor) {
+        openMenu = null;
+        openAnchor = null;
+        if (anchor != null) anchor.dispose();
+    }
+
+    /**
+     * Brings the window back from the taskbar or an iconified state.
+     *
+     * <p>A window manager ignores {@link Frame#toFront()} from an application that is
+     * not focused, which is every time the tray icon is used, so the backend is asked
+     * first — it can make the request the window manager does honour.
+     */
     private void show() {
+        hideMenu();
         frame.setVisible(true);
         frame.setExtendedState(frame.getExtendedState() & ~Frame.ICONIFIED);
+        // Deferred so the window is mapped before anything tries to raise it.
+        SwingUtilities.invokeLater(() -> {
+            if (tray != null && tray.activateWindow(frame.getTitle())) return;
+            // Whatever Swing can manage: asking to be on top for a moment gets the
+            // window in front even where a plain raise is refused.
+            frame.setAlwaysOnTop(true);
             frame.toFront();
             frame.requestFocus();
+            frame.setAlwaysOnTop(false);
+        });
     }
 }

ts3-client/swing/src/main/java/com/ts3client/ui/tray/TrayBackend.java

@@ -23,6 +23,19 @@ public interface TrayBackend {
     /** How this icon is being shown, for the client's own diagnostics. */
     String description();
 
+    /**
+     * Asks the window manager to bring a window of this application to the front.
+     * Swing cannot: a window manager ignores a raise from an application that does
+     * not have the focus, which is exactly the case when the tray icon is used.
+     *
+     * @param title the window's title
+     * @return {@code false} when this backend cannot ask, and the caller should try
+     * whatever Swing can do
+     */
+    default boolean activateWindow(String title) {
+        return false;
+    }
+
     /** What the user did with the icon. */
     interface Listener {
 

ts3-client/swing/src/main/java/com/ts3client/ui/tray/X11.java

@@ -47,6 +47,8 @@ final class X11 {
     static final long CW_COLORMAP = 1L << 13;
 
     static final long BUTTON_PRESS_MASK = 1L << 2;
+    static final long SUBSTRUCTURE_NOTIFY_MASK = 1L << 19;
+    static final long SUBSTRUCTURE_REDIRECT_MASK = 1L << 20;
     static final long EXPOSURE_MASK = 1L << 15;
     static final long STRUCTURE_NOTIFY_MASK = 1L << 17;
     static final long NO_EVENT_MASK = 0L;
@@ -63,6 +65,8 @@ final class X11 {
     static final long XA_VISUALID = 32;
 
     static final long VISUAL_ID_MASK = 0x1;
+    /** {@code AnyPropertyType}: read a property whatever its type. */
+    static final long ANY_PROPERTY_TYPE = 0;
 
     // ---- structure layout ----
 
@@ -160,6 +164,8 @@ final class X11 {
                 downcall("XSelectInput", FunctionDescriptor.of(INT, PTR, LONG, LONG));
         static final MethodHandle X_SEND_EVENT =
                 downcall("XSendEvent", FunctionDescriptor.of(INT, PTR, LONG, INT, LONG, PTR));
+        static final MethodHandle X_QUERY_TREE =
+                downcall("XQueryTree", FunctionDescriptor.of(INT, PTR, LONG, PTR, PTR, PTR, PTR));
         static final MethodHandle X_SET_WM_NORMAL_HINTS =
                 downcall("XSetWMNormalHints", FunctionDescriptor.ofVoid(PTR, LONG, PTR));
         static final MethodHandle X_RESIZE_WINDOW =
@@ -287,6 +293,59 @@ final class X11 {
                 data, bytes.length);
     }
 
+    /** @return the children of a window, oldest first */
+    static long[] children(MemorySegment display, Arena arena, long window) {
+        MemorySegment rootReturn = arena.allocate(LONG);
+        MemorySegment parentReturn = arena.allocate(LONG);
+        MemorySegment childrenReturn = arena.allocate(PTR);
+        MemorySegment countReturn = arena.allocate(INT);
+        int status = (int) call(Handles.X_QUERY_TREE, display, window, rootReturn, parentReturn,
+                childrenReturn, countReturn);
+        MemorySegment list = childrenReturn.get(PTR, 0);
+        if (status == 0 || list.equals(MemorySegment.NULL)) return new long[0];
+        int count = countReturn.get(INT, 0);
+        long[] ids = new long[count];
+        MemorySegment windows = list.reinterpret((long) count * Long.BYTES);
+        for (int i = 0; i < count; i++) ids[i] = windows.getAtIndex(LONG, i);
+        call(Handles.X_FREE, list);
+        return ids;
+    }
+
+    /**
+     * Reads a text property.
+     *
+     * @return its value as UTF-8, or {@code null} when the window has no such property
+     */
+    static String textProperty(MemorySegment display, Arena arena, long window, long property) {
+        MemorySegment actualType = arena.allocate(LONG);
+        MemorySegment actualFormat = arena.allocate(INT);
+        MemorySegment items = arena.allocate(LONG);
+        MemorySegment bytesAfter = arena.allocate(LONG);
+        MemorySegment data = arena.allocate(PTR);
+        int status = (int) call(Handles.X_GET_WINDOW_PROPERTY, display, window, property, 0L, 1024L,
+                0, ANY_PROPERTY_TYPE, actualType, actualFormat, items, bytesAfter, data);
+        MemorySegment value = data.get(PTR, 0);
+        if (status != SUCCESS || value.equals(MemorySegment.NULL)) return null;
+        long length = items.get(LONG, 0);
+        byte[] bytes = value.reinterpret(length).toArray(ValueLayout.JAVA_BYTE);
+        call(Handles.X_FREE, value);
+        return new String(bytes, StandardCharsets.UTF_8);
+    }
+
+    /** @return {@code true} when the window carries the property at all */
+    static boolean hasProperty(MemorySegment display, Arena arena, long window, long property) {
+        MemorySegment actualType = arena.allocate(LONG);
+        MemorySegment actualFormat = arena.allocate(INT);
+        MemorySegment items = arena.allocate(LONG);
+        MemorySegment bytesAfter = arena.allocate(LONG);
+        MemorySegment data = arena.allocate(PTR);
+        int status = (int) call(Handles.X_GET_WINDOW_PROPERTY, display, window, property, 0L, 0L, 0,
+                ANY_PROPERTY_TYPE, actualType, actualFormat, items, bytesAfter, data);
+        MemorySegment value = data.get(PTR, 0);
+        if (!value.equals(MemorySegment.NULL)) call(Handles.X_FREE, value);
+        return status == SUCCESS && actualType.get(LONG, 0) != 0;
+    }
+
     // ---- visuals ----
 
     /**

ts3-client/swing/src/main/java/com/ts3client/ui/tray/X11Tray.java

@@ -9,6 +9,7 @@ import java.lang.foreign.Arena;
 import java.lang.foreign.MemorySegment;
 import java.lang.foreign.ValueLayout;
 import java.util.function.IntFunction;
+import java.util.function.LongPredicate;
 
 /**
  * A tray icon docked into the panel's notification area by hand, over the
@@ -263,6 +264,77 @@ public final class X11Tray implements TrayBackend {
         }
     }
 
+    /**
+     * Sends the window manager an {@code _NET_ACTIVE_WINDOW} request for the named
+     * window — the EWMH way of saying "the user asked for this window", which is
+     * honoured where a plain raise is not.
+     */
+    @Override
+    public boolean activateWindow(String title) {
+        synchronized (this) {
+            if (!running) return false;
+            long target = findWindow(title);
+            if (target == 0) {
+                TrayLog.debug("no window named \"" + title + "\" to activate");
+                return false;
+            }
+            TrayLog.debug("activating window 0x" + Long.toHexString(target));
+
+            MemorySegment event = arena.allocate(X11.EVENT_SIZE);
+            event.fill((byte) 0);
+            event.set(ValueLayout.JAVA_INT, X11.EVENT_TYPE, X11.CLIENT_MESSAGE);
+            event.set(ValueLayout.JAVA_LONG, X11.EVENT_WINDOW, target);
+            event.set(ValueLayout.JAVA_LONG, X11.CLIENT_MESSAGE_TYPE,
+                    X11.internAtom(display, arena, "_NET_ACTIVE_WINDOW"));
+            event.set(ValueLayout.JAVA_INT, X11.CLIENT_MESSAGE_FORMAT, 32);
+            // Source indication 2: a pager or tray acting for the user, which window
+            // managers accept without their focus-stealing rules getting in the way.
+            event.set(ValueLayout.JAVA_LONG, X11.CLIENT_MESSAGE_DATA, 2);
+            X11.sendEvent(display, root,
+                    X11.SUBSTRUCTURE_NOTIFY_MASK | X11.SUBSTRUCTURE_REDIRECT_MASK, event);
+            X11.flush(display);
+            return true;
+        }
+    }
+
+    /**
+     * Finds one of this process' own top-level windows: the one with the given title
+     * for choice, else any window the window manager is managing for us.
+     *
+     * <p>Windows are matched by {@code _NET_WM_PID} rather than by name alone, and
+     * the title is read from {@code _NET_WM_NAME} because {@code WM_NAME} cannot
+     * carry anything outside Latin-1.
+     */
+    private long findWindow(String title) {
+        long pid = ProcessHandle.current().pid();
+        long pidAtom = X11.internAtom(display, arena, "_NET_WM_PID");
+        long nameAtom = X11.internAtom(display, arena, "_NET_WM_NAME");
+        long stateAtom = X11.internAtom(display, arena, "WM_STATE");
+
+        long found = findWindow(root, 0, w ->
+                X11.cardinalProperty(display, arena, w, pidAtom, X11.ANY_PROPERTY_TYPE) == pid
+                        && title.equals(X11.textProperty(display, arena, w, nameAtom)));
+        if (found != 0) return found;
+        return findWindow(root, 0, w ->
+                X11.cardinalProperty(display, arena, w, pidAtom, X11.ANY_PROPERTY_TYPE) == pid
+                        && X11.hasProperty(display, arena, w, stateAtom));
+    }
+
+    /**
+     * Walks the window tree below {@code parent}. Only the top few levels are looked
+     * at: a reparenting window manager puts client windows one or two frames below
+     * the root.
+     */
+    private long findWindow(long parent, int depth, LongPredicate match) {
+        if (depth > 3) return 0;
+        for (long child : X11.children(display, arena, parent)) {
+            if (match.test(child)) return child;
+            long found = findWindow(child, depth + 1, match);
+            if (found != 0) return found;
+        }
+        return 0;
+    }
+
     @Override
     public String description() {
         return "docked into the notification area, " + width + "x" + height + ", transparent";