Soundboard: WAV and Opus sounds sent with the microphone

Core: ClipMixer mixes clips into the capturing VoiceInput's frames (after
APM/VAD), transmitting past the gate while one plays; a mute still wins and
the clip runs out unheard. Sessions owns the mixer so sounds follow the
microphone between servers, and drops them when nobody captures. Soundboard
decodes (SoundFiles: wave or Ogg Opus, sniffed by content) off the caller's
thread, caches, and plays a local copy via the new stoppable
SoundPlayer.play(float[], volume). Sounds live in soundboard.properties.

Desktop: Self > Soundboard window (tile grid, click to play/stop, drop files
to add, edit/move/remove, sent and local volume), toolbar action, and hotkeys
Play Sound, Play/Stop Sound, Stop All Sounds.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-29 21:03:05 +00:00
co-authored by Claude Opus 5.5
parent d96339b72b
commit b33f1fd25a
26 changed files with 1426 additions and 22 deletions
@@ -0,0 +1,276 @@
package eu.lixko.ts3j.client.soundboard;
import eu.lixko.ts3j.client.audio.AudioCapture;
import eu.lixko.ts3j.client.audio.AudioDevice;
import eu.lixko.ts3j.client.audio.AudioIo;
import eu.lixko.ts3j.client.audio.AudioPlayback;
import eu.lixko.ts3j.client.audio.CaptureVoiceInput;
import eu.lixko.ts3j.client.audio.ClipMixer;
import eu.lixko.ts3j.client.audio.VoiceFormat;
import eu.lixko.ts3j.client.audio.desktop.NativeOpusCodec;
import eu.lixko.ts3j.client.audio.opus.OpusDecoder;
import eu.lixko.ts3j.client.config.Settings;
import eu.lixko.ts3j.client.sound.SoundPackConverter;
import eu.lixko.ts3j.client.sound.SoundPlayer;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Soundboard sounds, wave and Ogg Opus, go out through the real capture pipeline while the
* microphone itself is silent below the gate, and decode back to the sound that was played.
*/
class SoundboardTest {
private static final int RATE = 44_100;
@TempDir
File dir;
private final NativeOpusCodec opus = new NativeOpusCodec();
private final Settings settings = new Settings();
private final SilentMic mic = new SilentMic();
private final ClipMixer mixer = new ClipMixer();
private CaptureVoiceInput input;
private Soundboard soundboard;
private float[] tone;
@BeforeEach
void setUp() throws Exception {
settings.inputMode.set(Settings.InputMode.VOICE_ACTIVATION);
settings.vadMode.set(Settings.VadMode.VOLUME_GATE);
settings.soundboardLocalVolume.set(0.0);
input = new CaptureVoiceInput(settings, mic, opus);
input.setClips(mixer);
input.start();
mic.awaitFrameDone();
soundboard = new Soundboard(mixer, () -> true, opus, new SoundPlayer() {
@Override
public void play(File file, double volume) {
}
@Override
public void shutdown() {
}
}, settings);
tone = new float[RATE * 3 / 4 + 77];
for (int i = 0; i < tone.length; i++) {
tone[i] = (float) (0.4 * Math.sin(2 * Math.PI * 440 * i / RATE) * Math.min(1, i / 2000.0));
}
}
@AfterEach
void tearDown() {
input.stop();
soundboard.shutdown();
}
@Test
void waveSoundIsSentWhileTheGateIsShut() throws Exception {
File wave = writeWave(new File(dir, "tone.wav"), tone);
assertSent(new SoundboardSound("tone", wave.getAbsolutePath()));
}
@Test
void opusSoundIsSent() throws Exception {
File source = new File(dir, "wav/pack");
Files.createDirectories(source.toPath());
Files.writeString(new File(source, "settings.ini").toPath(), "[info]\nname = Test\n");
writeWave(new File(source, "tone.wav"), tone);
SoundPackConverter.convert(new File(dir, "wav"), new File(dir, "opus"), opus);
File opusFile = new File(dir, "opus/pack/tone.opus");
assertTrue(opusFile.isFile());
assertSent(new SoundboardSound("tone", opusFile.getAbsolutePath()));
}
@Test
void mutedMicrophoneSendsNothingButTheSoundRunsOut() throws Exception {
input.setMuted(true);
SoundboardSound sound = new SoundboardSound("tone", writeWave(new File(dir, "tone.wav"), tone).getAbsolutePath());
soundboard.play(sound).get(5, TimeUnit.SECONDS);
List<byte[]> packets = run(sound);
assertTrue(packets.isEmpty(), packets.size() + " packets sent while muted");
assertFalse(soundboard.isPlaying(sound));
}
@Test
void unreadableFileFailsThePlay() throws Exception {
File junk = new File(dir, "junk.wav");
Files.writeString(junk.toPath(), "not a sound");
ExecutionException e = assertThrows(ExecutionException.class,
() -> soundboard.play(new SoundboardSound("junk", junk.getAbsolutePath())).get(5, TimeUnit.SECONDS));
assertInstanceOf(SoundboardException.class, e.getCause());
assertFalse(soundboard.isPlaying());
}
private void assertSent(SoundboardSound sound) throws Exception {
List<Boolean> talk = new ArrayList<>();
input.setTalkListener(talk::add);
soundboard.play(sound).get(5, TimeUnit.SECONDS);
assertTrue(soundboard.isPlaying(sound));
List<byte[]> packets = run(sound);
float[] expected = resample(tone);
int frames = (expected.length + VoiceFormat.FRAME_SIZE - 1) / VoiceFormat.FRAME_SIZE;
assertEquals(frames, packets.size(), "one packet per 20 ms of sound");
assertEquals(List.of(true, false), talk, "talking for the sound's length only");
float[] decoded = decode(packets);
double best = 0;
for (int lag = 0; lag < 1200; lag++) best = Math.max(best, correlation(expected, decoded, lag));
assertTrue(best > 0.95, "correlation " + best);
}
/** Steps the microphone frame by frame until the sound is over, collecting what would be sent. */
private List<byte[]> run(SoundboardSound sound) throws InterruptedException {
List<byte[]> packets = new ArrayList<>();
for (int i = 0; i < 500 && (soundboard.isPlaying(sound) || input.isReady()); i++) {
mic.captureFrame();
byte[] p;
while ((p = input.provide()).length > 0) packets.add(p);
}
// Once the sound is over the gate is shut again, so ts3j ends the stream.
mic.captureFrame();
assertFalse(input.isReady());
return packets;
}
private float[] decode(List<byte[]> packets) {
float[] out = new float[packets.size() * VoiceFormat.FRAME_SIZE];
float[] frame = new float[VoiceFormat.FRAME_SIZE];
try (OpusDecoder decoder = opus.createDecoder(VoiceFormat.SAMPLE_RATE, VoiceFormat.FRAME_SIZE, 1)) {
int at = 0;
for (byte[] p : packets) {
int n = decoder.decode(p, frame);
System.arraycopy(frame, 0, out, at, n);
at += n;
}
}
return out;
}
private static double correlation(float[] a, float[] b, int lag) {
double dot = 0, ea = 0, eb = 0;
for (int i = 0; i < a.length && i + lag < b.length; i++) {
dot += a[i] * b[i + lag];
ea += a[i] * a[i];
eb += b[i + lag] * b[i + lag];
}
return ea == 0 || eb == 0 ? 0 : dot / Math.sqrt(ea * eb);
}
private static float[] resample(float[] in) {
double ratio = VoiceFormat.SAMPLE_RATE / (double) RATE;
float[] out = new float[(int) (in.length * ratio)];
for (int i = 0; i < out.length; i++) {
double position = i / ratio;
int index = (int) position;
float a = in[Math.min(index, in.length - 1)];
float b = in[Math.min(index + 1, in.length - 1)];
out[i] = (float) (a + (b - a) * (position - index));
}
return out;
}
private static File writeWave(File file, float[] samples) throws IOException {
ByteBuffer b = ByteBuffer.allocate(44 + samples.length * 2).order(ByteOrder.LITTLE_ENDIAN);
b.put("RIFF".getBytes()).putInt(36 + samples.length * 2).put("WAVE".getBytes());
b.put("fmt ".getBytes()).putInt(16).putShort((short) 1).putShort((short) 1)
.putInt(RATE).putInt(RATE * 2).putShort((short) 2).putShort((short) 16);
b.put("data".getBytes()).putInt(samples.length * 2);
for (float s : samples) b.putShort((short) Math.round(s * 32767));
Files.write(file.toPath(), b.array());
return file;
}
/** A mono microphone that captures silence one frame at a time, when the test says so. */
private static final class SilentMic implements AudioIo {
private final LinkedBlockingQueue<Boolean> frames = new LinkedBlockingQueue<>();
private final Semaphore done = new Semaphore(0);
/** Lets one frame through and waits until the capture loop has dealt with it. */
void captureFrame() throws InterruptedException {
frames.put(true);
awaitFrameDone();
}
void awaitFrameDone() throws InterruptedException {
assertTrue(done.tryAcquire(5, TimeUnit.SECONDS), "capture loop stalled");
}
@Override
public List<AudioDevice> inputDevices() {
return List.of();
}
@Override
public List<AudioDevice> outputDevices() {
return List.of();
}
@Override
public AudioCapture openCapture(String deviceId, int preferredChannels) {
return new AudioCapture() {
private volatile boolean open = true;
@Override
public int channels() {
return 1;
}
@Override
public void start() {
}
@Override
public int read(byte[] buffer, int offset, int length) {
// Asking for the next frame means the last one is through the pipeline.
done.release();
try {
while (open) {
if (frames.poll(50, TimeUnit.MILLISECONDS) != null) {
Arrays.fill(buffer, offset, offset + length, (byte) 0);
return length;
}
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
return 0;
}
@Override
public void close() {
open = false;
}
};
}
@Override
public AudioPlayback openPlayback(String deviceId, int preferredChannels) {
throw new UnsupportedOperationException();
}
}
}