Soundboard: WAV and Opus sounds sent with the microphone
Core: ClipMixer mixes clips into the capturing VoiceInput's frames (after APM/VAD), transmitting past the gate while one plays; a mute still wins and the clip runs out unheard. Sessions owns the mixer so sounds follow the microphone between servers, and drops them when nobody captures. Soundboard decodes (SoundFiles: wave or Ogg Opus, sniffed by content) off the caller's thread, caches, and plays a local copy via the new stoppable SoundPlayer.play(float[], volume). Sounds live in soundboard.properties. Desktop: Self > Soundboard window (tile grid, click to play/stop, drop files to add, edit/move/remove, sent and local volume), toolbar action, and hotkeys Play Sound, Play/Stop Sound, Stop All Sounds. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,276 @@
|
||||
package eu.lixko.ts3j.client.soundboard;
|
||||
|
||||
import eu.lixko.ts3j.client.audio.AudioCapture;
|
||||
import eu.lixko.ts3j.client.audio.AudioDevice;
|
||||
import eu.lixko.ts3j.client.audio.AudioIo;
|
||||
import eu.lixko.ts3j.client.audio.AudioPlayback;
|
||||
import eu.lixko.ts3j.client.audio.CaptureVoiceInput;
|
||||
import eu.lixko.ts3j.client.audio.ClipMixer;
|
||||
import eu.lixko.ts3j.client.audio.VoiceFormat;
|
||||
import eu.lixko.ts3j.client.audio.desktop.NativeOpusCodec;
|
||||
import eu.lixko.ts3j.client.audio.opus.OpusDecoder;
|
||||
import eu.lixko.ts3j.client.config.Settings;
|
||||
import eu.lixko.ts3j.client.sound.SoundPackConverter;
|
||||
import eu.lixko.ts3j.client.sound.SoundPlayer;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
/**
|
||||
* Soundboard sounds, wave and Ogg Opus, go out through the real capture pipeline while the
|
||||
* microphone itself is silent below the gate, and decode back to the sound that was played.
|
||||
*/
|
||||
class SoundboardTest {
|
||||
|
||||
private static final int RATE = 44_100;
|
||||
|
||||
@TempDir
|
||||
File dir;
|
||||
|
||||
private final NativeOpusCodec opus = new NativeOpusCodec();
|
||||
private final Settings settings = new Settings();
|
||||
private final SilentMic mic = new SilentMic();
|
||||
private final ClipMixer mixer = new ClipMixer();
|
||||
private CaptureVoiceInput input;
|
||||
private Soundboard soundboard;
|
||||
private float[] tone;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
settings.inputMode.set(Settings.InputMode.VOICE_ACTIVATION);
|
||||
settings.vadMode.set(Settings.VadMode.VOLUME_GATE);
|
||||
settings.soundboardLocalVolume.set(0.0);
|
||||
input = new CaptureVoiceInput(settings, mic, opus);
|
||||
input.setClips(mixer);
|
||||
input.start();
|
||||
mic.awaitFrameDone();
|
||||
soundboard = new Soundboard(mixer, () -> true, opus, new SoundPlayer() {
|
||||
@Override
|
||||
public void play(File file, double volume) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void shutdown() {
|
||||
}
|
||||
}, settings);
|
||||
|
||||
tone = new float[RATE * 3 / 4 + 77];
|
||||
for (int i = 0; i < tone.length; i++) {
|
||||
tone[i] = (float) (0.4 * Math.sin(2 * Math.PI * 440 * i / RATE) * Math.min(1, i / 2000.0));
|
||||
}
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void tearDown() {
|
||||
input.stop();
|
||||
soundboard.shutdown();
|
||||
}
|
||||
|
||||
@Test
|
||||
void waveSoundIsSentWhileTheGateIsShut() throws Exception {
|
||||
File wave = writeWave(new File(dir, "tone.wav"), tone);
|
||||
assertSent(new SoundboardSound("tone", wave.getAbsolutePath()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void opusSoundIsSent() throws Exception {
|
||||
File source = new File(dir, "wav/pack");
|
||||
Files.createDirectories(source.toPath());
|
||||
Files.writeString(new File(source, "settings.ini").toPath(), "[info]\nname = Test\n");
|
||||
writeWave(new File(source, "tone.wav"), tone);
|
||||
SoundPackConverter.convert(new File(dir, "wav"), new File(dir, "opus"), opus);
|
||||
File opusFile = new File(dir, "opus/pack/tone.opus");
|
||||
assertTrue(opusFile.isFile());
|
||||
assertSent(new SoundboardSound("tone", opusFile.getAbsolutePath()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void mutedMicrophoneSendsNothingButTheSoundRunsOut() throws Exception {
|
||||
input.setMuted(true);
|
||||
SoundboardSound sound = new SoundboardSound("tone", writeWave(new File(dir, "tone.wav"), tone).getAbsolutePath());
|
||||
soundboard.play(sound).get(5, TimeUnit.SECONDS);
|
||||
List<byte[]> packets = run(sound);
|
||||
assertTrue(packets.isEmpty(), packets.size() + " packets sent while muted");
|
||||
assertFalse(soundboard.isPlaying(sound));
|
||||
}
|
||||
|
||||
@Test
|
||||
void unreadableFileFailsThePlay() throws Exception {
|
||||
File junk = new File(dir, "junk.wav");
|
||||
Files.writeString(junk.toPath(), "not a sound");
|
||||
ExecutionException e = assertThrows(ExecutionException.class,
|
||||
() -> soundboard.play(new SoundboardSound("junk", junk.getAbsolutePath())).get(5, TimeUnit.SECONDS));
|
||||
assertInstanceOf(SoundboardException.class, e.getCause());
|
||||
assertFalse(soundboard.isPlaying());
|
||||
}
|
||||
|
||||
private void assertSent(SoundboardSound sound) throws Exception {
|
||||
List<Boolean> talk = new ArrayList<>();
|
||||
input.setTalkListener(talk::add);
|
||||
soundboard.play(sound).get(5, TimeUnit.SECONDS);
|
||||
assertTrue(soundboard.isPlaying(sound));
|
||||
List<byte[]> packets = run(sound);
|
||||
|
||||
float[] expected = resample(tone);
|
||||
int frames = (expected.length + VoiceFormat.FRAME_SIZE - 1) / VoiceFormat.FRAME_SIZE;
|
||||
assertEquals(frames, packets.size(), "one packet per 20 ms of sound");
|
||||
assertEquals(List.of(true, false), talk, "talking for the sound's length only");
|
||||
|
||||
float[] decoded = decode(packets);
|
||||
double best = 0;
|
||||
for (int lag = 0; lag < 1200; lag++) best = Math.max(best, correlation(expected, decoded, lag));
|
||||
assertTrue(best > 0.95, "correlation " + best);
|
||||
}
|
||||
|
||||
/** Steps the microphone frame by frame until the sound is over, collecting what would be sent. */
|
||||
private List<byte[]> run(SoundboardSound sound) throws InterruptedException {
|
||||
List<byte[]> packets = new ArrayList<>();
|
||||
for (int i = 0; i < 500 && (soundboard.isPlaying(sound) || input.isReady()); i++) {
|
||||
mic.captureFrame();
|
||||
byte[] p;
|
||||
while ((p = input.provide()).length > 0) packets.add(p);
|
||||
}
|
||||
// Once the sound is over the gate is shut again, so ts3j ends the stream.
|
||||
mic.captureFrame();
|
||||
assertFalse(input.isReady());
|
||||
return packets;
|
||||
}
|
||||
|
||||
private float[] decode(List<byte[]> packets) {
|
||||
float[] out = new float[packets.size() * VoiceFormat.FRAME_SIZE];
|
||||
float[] frame = new float[VoiceFormat.FRAME_SIZE];
|
||||
try (OpusDecoder decoder = opus.createDecoder(VoiceFormat.SAMPLE_RATE, VoiceFormat.FRAME_SIZE, 1)) {
|
||||
int at = 0;
|
||||
for (byte[] p : packets) {
|
||||
int n = decoder.decode(p, frame);
|
||||
System.arraycopy(frame, 0, out, at, n);
|
||||
at += n;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static double correlation(float[] a, float[] b, int lag) {
|
||||
double dot = 0, ea = 0, eb = 0;
|
||||
for (int i = 0; i < a.length && i + lag < b.length; i++) {
|
||||
dot += a[i] * b[i + lag];
|
||||
ea += a[i] * a[i];
|
||||
eb += b[i + lag] * b[i + lag];
|
||||
}
|
||||
return ea == 0 || eb == 0 ? 0 : dot / Math.sqrt(ea * eb);
|
||||
}
|
||||
|
||||
private static float[] resample(float[] in) {
|
||||
double ratio = VoiceFormat.SAMPLE_RATE / (double) RATE;
|
||||
float[] out = new float[(int) (in.length * ratio)];
|
||||
for (int i = 0; i < out.length; i++) {
|
||||
double position = i / ratio;
|
||||
int index = (int) position;
|
||||
float a = in[Math.min(index, in.length - 1)];
|
||||
float b = in[Math.min(index + 1, in.length - 1)];
|
||||
out[i] = (float) (a + (b - a) * (position - index));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private static File writeWave(File file, float[] samples) throws IOException {
|
||||
ByteBuffer b = ByteBuffer.allocate(44 + samples.length * 2).order(ByteOrder.LITTLE_ENDIAN);
|
||||
b.put("RIFF".getBytes()).putInt(36 + samples.length * 2).put("WAVE".getBytes());
|
||||
b.put("fmt ".getBytes()).putInt(16).putShort((short) 1).putShort((short) 1)
|
||||
.putInt(RATE).putInt(RATE * 2).putShort((short) 2).putShort((short) 16);
|
||||
b.put("data".getBytes()).putInt(samples.length * 2);
|
||||
for (float s : samples) b.putShort((short) Math.round(s * 32767));
|
||||
Files.write(file.toPath(), b.array());
|
||||
return file;
|
||||
}
|
||||
|
||||
/** A mono microphone that captures silence one frame at a time, when the test says so. */
|
||||
private static final class SilentMic implements AudioIo {
|
||||
|
||||
private final LinkedBlockingQueue<Boolean> frames = new LinkedBlockingQueue<>();
|
||||
private final Semaphore done = new Semaphore(0);
|
||||
|
||||
/** Lets one frame through and waits until the capture loop has dealt with it. */
|
||||
void captureFrame() throws InterruptedException {
|
||||
frames.put(true);
|
||||
awaitFrameDone();
|
||||
}
|
||||
|
||||
void awaitFrameDone() throws InterruptedException {
|
||||
assertTrue(done.tryAcquire(5, TimeUnit.SECONDS), "capture loop stalled");
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AudioDevice> inputDevices() {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AudioDevice> outputDevices() {
|
||||
return List.of();
|
||||
}
|
||||
|
||||
@Override
|
||||
public AudioCapture openCapture(String deviceId, int preferredChannels) {
|
||||
return new AudioCapture() {
|
||||
private volatile boolean open = true;
|
||||
|
||||
@Override
|
||||
public int channels() {
|
||||
return 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void start() {
|
||||
}
|
||||
|
||||
@Override
|
||||
public int read(byte[] buffer, int offset, int length) {
|
||||
// Asking for the next frame means the last one is through the pipeline.
|
||||
done.release();
|
||||
try {
|
||||
while (open) {
|
||||
if (frames.poll(50, TimeUnit.MILLISECONDS) != null) {
|
||||
Arrays.fill(buffer, offset, offset + length, (byte) 0);
|
||||
return length;
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
open = false;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public AudioPlayback openPlayback(String deviceId, int preferredChannels) {
|
||||
throw new UnsupportedOperationException();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user