diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/StreamingVoiceOutput.java b/ts3-client/core/src/main/java/com/ts3client/audio/StreamingVoiceOutput.java index ff97e80..82be5c4 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/StreamingVoiceOutput.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/StreamingVoiceOutput.java @@ -49,6 +49,7 @@ public final class StreamingVoiceOutput implements VoiceOutput { /** Notified (clientId, talking) on an audio thread when a speaker starts/stops. */ private volatile BiConsumer talkListener; + private volatile VoiceTap voiceTap; /** Catches talk bursts whose end packet never arrived, and paces per-speaker lines. */ private final ScheduledExecutorService clock = Executors.newSingleThreadScheduledExecutor(r -> { @@ -89,6 +90,12 @@ public final class StreamingVoiceOutput implements VoiceOutput { this.talkListener = l; } + @Override + public void setVoiceTap(VoiceTap tap) { + this.voiceTap = tap; + for (VoiceStream s : streams.values()) s.setTap(tap); + } + @Override public void setMasterVolume(double v) { this.masterVolume = Math.max(0, Math.min(2.0, v)); @@ -156,7 +163,11 @@ public final class StreamingVoiceOutput implements VoiceOutput { if (deafened) return; if (mutedClients.contains(clientId)) return; - VoiceStream stream = streams.computeIfAbsent(clientId, id -> new VoiceStream(id, opus, this::fireTalk)); + VoiceStream stream = streams.computeIfAbsent(clientId, id -> { + VoiceStream created = new VoiceStream(id, opus, this::fireTalk); + created.setTap(voiceTap); + return created; + }); if (stream.offer(packetId, codec, data)) playout.started(stream); } diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceOutput.java b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceOutput.java index cb76ebb..06c7c6d 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceOutput.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceOutput.java @@ -40,4 +40,7 @@ public interface VoiceOutput { /** Receives remote speaker talk-state transitions as (clientId, talking). */ void setTalkListener(BiConsumer listener); + + /** Receives every speaker's voice as it is played, e.g. for recording; null detaches it. */ + void setVoiceTap(VoiceTap tap); } diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java index 9b272b8..dac4489 100644 --- a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java +++ b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceStream.java @@ -33,6 +33,7 @@ final class VoiceStream { static final int MAX_FRAME = 5760; private static final int FRAME = VoiceFormat.FRAME_SIZE; + private static final float[] SILENT_FRAME = new float[FRAME]; private static final int FRAME_TICKS = JitterBuffer.TEAMSPEAK_TICKS_PER_FRAME; /** Pulls in a row with nothing held before a burst whose end marker got lost is given up. */ private static final int EMPTY_PULLS_TO_END = 10; @@ -77,6 +78,9 @@ final class VoiceStream { private OpusDecoder decoder; private int decoderChannels; private boolean closed; + /** The played frame in mono and at full volume, for the {@link #tap}. */ + private final float[] tapped = new float[FRAME]; + private volatile VoiceTap tap; private volatile boolean talking; private volatile long lastPacketNanos; @@ -87,6 +91,10 @@ final class VoiceStream { this.talkListener = talkListener; } + void setTap(VoiceTap tap) { + this.tap = tap; + } + private static int channelsFor(CodecType codec) { return codec == CodecType.OPUS_MUSIC ? 2 : 1; } @@ -209,29 +217,40 @@ final class VoiceStream { int outChannels, double gain) { switch (source) { case SURPLUS -> { - emit(surplus, surplusOffset, surplusTake, surplusChannels, out, outChannels, gain); + play(surplus, surplusOffset, surplusTake, surplusChannels, out, outChannels, gain); surplusOffset += surplusTake; return surplusTake; } case SILENCE -> { - java.util.Arrays.fill(out, 0, FRAME * outChannels, 0f); + play(SILENT_FRAME, 0, FRAME, 1, out, outChannels, gain); return FRAME; } case CONCEAL -> { int frames = decoderFor(decoderChannels == 0 ? packetChannels : decoderChannels).conceal(pcm, FRAME); - emit(pcm, 0, frames, decoderChannels, out, outChannels, gain); + play(pcm, 0, frames, decoderChannels, out, outChannels, gain); return frames; } default -> { int frames = decoderFor(packetChannels).decode(packet, pcm); int played = Math.min(frames, FRAME); - emit(pcm, 0, played, packetChannels, out, outChannels, gain); + play(pcm, 0, played, packetChannels, out, outChannels, gain); if (frames > played) keepSurplus(played, frames - played, packetChannels); return played; } } } + /** Hands a frame to the output and, if attached, the tap. */ + private void play(float[] src, int fromFrame, int frames, int channels, float[] out, int outChannels, + double gain) { + emit(src, fromFrame, frames, channels, out, outChannels, gain); + VoiceTap t = tap; + if (t == null || frames <= 0) return; + int n = Math.min(frames, FRAME); + emit(src, fromFrame, n, channels, tapped, 1, 1.0); + t.onVoice(clientId, tapped, n); + } + /** Holds back what a packet decoded to beyond one frame, for the following pulls. */ private void keepSurplus(int from, int frames, int channels) { System.arraycopy(pcm, from * channels, surplus, 0, frames * channels); diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/VoiceTap.java b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceTap.java new file mode 100644 index 0000000..3ca4dfa --- /dev/null +++ b/ts3-client/core/src/main/java/com/ts3client/audio/VoiceTap.java @@ -0,0 +1,15 @@ +package com.ts3client.audio; + +/** Receives each speaker's voice as it is played, before any volume is applied. */ +@FunctionalInterface +public interface VoiceTap { + + /** + * Called from an audio thread with one played frame; do not block. The array is reused + * between calls. + * + * @param mono samples in [-1, 1] + * @param frames how many samples, at most one 20 ms frame + */ + void onVoice(int clientId, float[] mono, int frames); +} diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/ogg/OggPageWriter.java b/ts3-client/core/src/main/java/com/ts3client/audio/ogg/OggPageWriter.java new file mode 100644 index 0000000..d665a95 --- /dev/null +++ b/ts3-client/core/src/main/java/com/ts3client/audio/ogg/OggPageWriter.java @@ -0,0 +1,113 @@ +package com.ts3client.audio.ogg; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; + +/** + * Frames the packets of one logical bitstream into Ogg pages (RFC 3533). Packets too long + * for one page run on into the next; a page is closed once it holds about + * {@link #TARGET_PAGE_BYTES}, or on {@link #flush}. + */ +public final class OggPageWriter { + + private static final int TARGET_PAGE_BYTES = 4096; + private static final int MAX_SEGMENTS = 255; + + private static final int FLAG_CONTINUED = 0x01; + private static final int FLAG_FIRST = 0x02; + private static final int FLAG_LAST = 0x04; + + private final OutputStream out; + private final int serial; + private final byte[] lacing = new byte[MAX_SEGMENTS]; + private final ByteArrayOutputStream data = new ByteArrayOutputStream(); + private int segments; + private int sequence; + /** Whether the open page starts with the rest of a packet from the page before. */ + private boolean continued; + /** Granule position of the last packet completed on the open page, -1 if none. */ + private long granule = -1; + private long lastGranule; + + public OggPageWriter(OutputStream out, int serial) { + this.out = out; + this.serial = serial; + } + + /** @param granule the stream position at the end of this packet */ + public void write(byte[] packet, long granule) throws IOException { + int offset = 0; + while (true) { + if (segments == MAX_SEGMENTS) closePage(0, true); + int take = Math.min(255, packet.length - offset); + lacing[segments++] = (byte) take; + data.write(packet, offset, take); + offset += take; + // A packet ends with a segment shorter than 255 bytes, if need be an empty one. + if (take < 255) break; + } + this.granule = granule; + this.lastGranule = granule; + if (data.size() >= TARGET_PAGE_BYTES) flush(); + } + + /** Closes the open page, so the next packet starts a new one (as the Opus headers need). */ + public void flush() throws IOException { + if (segments > 0) closePage(0, false); + } + + /** Writes out the stream's last page. */ + public void finish() throws IOException { + closePage(FLAG_LAST, false); + out.flush(); + } + + private void closePage(int flags, boolean packetContinues) throws IOException { + if (continued) flags |= FLAG_CONTINUED; + if (sequence == 0) flags |= FLAG_FIRST; + long pageGranule = granule >= 0 ? granule : (flags & FLAG_LAST) != 0 ? lastGranule : -1; + + ByteBuffer header = ByteBuffer.allocate(27 + segments).order(ByteOrder.LITTLE_ENDIAN); + header.put("OggS".getBytes(StandardCharsets.US_ASCII)); + header.put((byte) 0); + header.put((byte) flags); + header.putLong(pageGranule); + header.putInt(serial); + header.putInt(sequence++); + header.putInt(0); // checksum, filled in below + header.put((byte) segments); + header.put(lacing, 0, segments); + + byte[] page = new byte[header.capacity() + data.size()]; + System.arraycopy(header.array(), 0, page, 0, header.capacity()); + System.arraycopy(data.toByteArray(), 0, page, header.capacity(), data.size()); + ByteBuffer.wrap(page).order(ByteOrder.LITTLE_ENDIAN).putInt(22, crc(page)); + out.write(page); + + segments = 0; + data.reset(); + granule = -1; + continued = packetContinues; + } + + /** Ogg's CRC-32: polynomial 0x04c11db7, not reflected, starting from 0. */ + private static int crc(byte[] page) { + int crc = 0; + for (byte b : page) crc = (crc << 8) ^ CRC_TABLE[((crc >>> 24) ^ b) & 0xff]; + return crc; + } + + private static final int[] CRC_TABLE = new int[256]; + + static { + for (int i = 0; i < 256; i++) { + int r = i << 24; + for (int j = 0; j < 8; j++) r = (r & 0x80000000) != 0 ? (r << 1) ^ 0x04c11db7 : r << 1; + CRC_TABLE[i] = r; + } + } +} diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/ogg/OpusHeaders.java b/ts3-client/core/src/main/java/com/ts3client/audio/ogg/OpusHeaders.java new file mode 100644 index 0000000..5549095 --- /dev/null +++ b/ts3-client/core/src/main/java/com/ts3client/audio/ogg/OpusHeaders.java @@ -0,0 +1,78 @@ +package com.ts3client.audio.ogg; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.List; + +/** The two header packets that open an Ogg Opus stream (RFC 7845, section 5). */ +public final class OpusHeaders { + + /** Channel mapping family 255 caps a stream at this many channels. */ + public static final int MAX_DISCRETE_CHANNELS = 255; + /** + * libopus' encoder delay at 48 kHz for 20 ms frames, which the file tells the decoder + * to drop (the SPI does not expose {@code OPUS_GET_LOOKAHEAD}). + */ + public static final int LIBOPUS_PRE_SKIP = 312; + + private OpusHeaders() { + } + + /** A single mono stream (mapping family 0). */ + public static byte[] mono(int preSkip, int inputRate) { + return head(1, preSkip, inputRate, 0).array(); + } + + /** + * {@code channels} unrelated channels, each its own uncoupled mono stream in channel + * order (mapping family 255): every packet is a multistream packet, see + * {@link SelfDelimitedOpus#join}. + */ + public static byte[] discrete(int channels, int preSkip, int inputRate) { + if (channels < 1 || channels > MAX_DISCRETE_CHANNELS) { + throw new IllegalArgumentException("channels " + channels); + } + ByteBuffer b = head(channels, preSkip, inputRate, 255); + b.put((byte) channels); // streams + b.put((byte) 0); // coupled streams + for (int c = 0; c < channels; c++) b.put((byte) c); + return b.array(); + } + + private static ByteBuffer head(int channels, int preSkip, int inputRate, int family) { + ByteBuffer b = ByteBuffer.allocate(19 + (family == 0 ? 0 : 2 + channels)).order(ByteOrder.LITTLE_ENDIAN); + b.put("OpusHead".getBytes(StandardCharsets.US_ASCII)); + b.put((byte) 1); // version + b.put((byte) channels); + b.putShort((short) preSkip); + b.putInt(inputRate); + b.putShort((short) 0); // output gain + b.put((byte) family); + return b; + } + + /** @param comments {@code NAME=value} user comments */ + public static byte[] tags(String vendor, List comments) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.writeBytes("OpusTags".getBytes(StandardCharsets.US_ASCII)); + writeString(out, vendor); + writeInt(out, comments.size()); + for (String comment : comments) writeString(out, comment); + return out.toByteArray(); + } + + private static void writeString(ByteArrayOutputStream out, String s) { + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + writeInt(out, bytes.length); + out.writeBytes(bytes); + } + + private static void writeInt(ByteArrayOutputStream out, int v) { + out.write(v); + out.write(v >>> 8); + out.write(v >>> 16); + out.write(v >>> 24); + } +} diff --git a/ts3-client/core/src/main/java/com/ts3client/audio/ogg/SelfDelimitedOpus.java b/ts3-client/core/src/main/java/com/ts3client/audio/ogg/SelfDelimitedOpus.java new file mode 100644 index 0000000..0d12662 --- /dev/null +++ b/ts3-client/core/src/main/java/com/ts3client/audio/ogg/SelfDelimitedOpus.java @@ -0,0 +1,104 @@ +package com.ts3client.audio.ogg; + +import java.io.ByteArrayOutputStream; +import java.util.List; + +/** + * Opus' self-delimiting packet framing (RFC 6716, appendix B), which multistream packets + * use to put several streams' packets back to back. + */ +public final class SelfDelimitedOpus { + + private SelfDelimitedOpus() { + } + + /** + * Joins one packet per stream into a multistream packet: every packet but the last is + * self-delimited, the last is kept as it is. + */ + public static byte[] join(List packets) { + if (packets.size() == 1) return packets.get(0); + int size = 0; + for (byte[] p : packets) size += p.length + 2; + ByteArrayOutputStream out = new ByteArrayOutputStream(size); + for (int i = 0; i < packets.size() - 1; i++) out.writeBytes(selfDelimited(packets.get(i))); + out.writeBytes(packets.get(packets.size() - 1)); + return out.toByteArray(); + } + + /** Re-frames a packet so its length can be told from the packet itself. */ + public static byte[] selfDelimited(byte[] packet) { + if (packet.length == 0) throw new IllegalArgumentException("empty Opus packet"); + int code = packet[0] & 0x03; + // The extra length goes right after the packet's own header: the TOC byte, plus for + // code 1 and 2 nothing more, for code 2 the first frame's length, and for code 3 the + // frame count, padding length and (VBR) all but the last frame's lengths. + int header; + int lastFrame; + switch (code) { + case 0 -> { + header = 1; + lastFrame = packet.length - 1; + } + case 1 -> { + header = 1; + lastFrame = (packet.length - 1) / 2; + } + case 2 -> { + int[] first = readLength(packet, 1); + header = 1 + first[1]; + lastFrame = packet.length - header - first[0]; + } + default -> { + if (packet.length < 2) throw new IllegalArgumentException("truncated Opus packet"); + int count = packet[1] & 0x3f; + boolean vbr = (packet[1] & 0x80) != 0; + boolean padded = (packet[1] & 0x40) != 0; + int pos = 2; + int padding = 0; + if (padded) { + int b; + do { + b = packet[pos++] & 0xff; + padding += b == 255 ? 254 : b; + } while (b == 255); + } + int payload; + if (vbr) { + int framesBefore = 0; + for (int i = 0; i < count - 1; i++) { + int[] length = readLength(packet, pos); + framesBefore += length[0]; + pos += length[1]; + } + payload = packet.length - pos - padding; + lastFrame = payload - framesBefore; + } else { + payload = packet.length - pos - padding; + lastFrame = payload / count; + } + header = pos; + } + } + if (lastFrame < 0) throw new IllegalArgumentException("malformed Opus packet"); + byte[] length = encodeLength(lastFrame); + byte[] out = new byte[packet.length + length.length]; + System.arraycopy(packet, 0, out, 0, header); + System.arraycopy(length, 0, out, header, length.length); + System.arraycopy(packet, header, out, header + length.length, packet.length - header); + return out; + } + + /** @return the frame length at {@code pos} and how many bytes it took */ + private static int[] readLength(byte[] packet, int pos) { + int b = packet[pos] & 0xff; + if (b < 252) return new int[]{b, 1}; + return new int[]{b + 4 * (packet[pos + 1] & 0xff), 2}; + } + + private static byte[] encodeLength(int length) { + if (length < 252) return new byte[]{(byte) length}; + int first = 252 + (length & 3); + return new byte[]{(byte) first, (byte) ((length - first) / 4)}; + } +} diff --git a/ts3-client/core/src/main/java/com/ts3client/config/Settings.java b/ts3-client/core/src/main/java/com/ts3client/config/Settings.java index 251271f..2237715 100644 --- a/ts3-client/core/src/main/java/com/ts3client/config/Settings.java +++ b/ts3-client/core/src/main/java/com/ts3client/config/Settings.java @@ -5,6 +5,8 @@ import com.ts3client.sound.NotificationSettings; import java.io.File; import java.io.FileInputStream; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Properties; /** @@ -199,6 +201,8 @@ public final class Settings { public boolean chatLogs = true; /** Where chat logs go; empty for the TeamSpeak 3 client's own folder, so both share them. */ public String chatLogDir = ""; + /** The folder recordings were last saved to; empty for {@code ~/Recordings}. */ + public String recordingDir = ""; /** Whether descriptions show their pictures, fetched from the web, or only links to them. */ public boolean descriptionPictures = true; /** Ctrl+wheel zoom of the chat's text and pictures, 1.0 being 100 %. */ @@ -234,6 +238,12 @@ public final class Settings { return new File(identityFile); } + public Path recordingFolder() { + return recordingDir.isBlank() + ? Paths.get(System.getProperty("user.home", "."), "Recordings") + : Paths.get(recordingDir); + } + /** {@link #outputVolume} scaled by the Master Volume slider. */ public double effectiveOutputVolume() { return outputVolume * masterVolume; @@ -299,6 +309,7 @@ public final class Settings { whisperPolicy = parseEnum(WhisperPolicy.class, props.getProperty("whisperPolicy"), whisperPolicy); chatLogs = parseB(props.getProperty("chatLogs"), chatLogs); chatLogDir = props.getProperty("chatLogDir", chatLogDir); + recordingDir = props.getProperty("recordingDir", recordingDir); } private void writeToProps() { @@ -356,6 +367,7 @@ public final class Settings { props.setProperty("whisperPolicy", whisperPolicy.name()); props.setProperty("chatLogs", Boolean.toString(chatLogs)); props.setProperty("chatLogDir", chatLogDir); + props.setProperty("recordingDir", recordingDir); } private static > E parseEnum(Class type, String v, E def) { diff --git a/ts3-client/core/src/main/java/com/ts3client/net/TeamspeakConnection.java b/ts3-client/core/src/main/java/com/ts3client/net/TeamspeakConnection.java index 40f174a..8598b06 100644 --- a/ts3-client/core/src/main/java/com/ts3client/net/TeamspeakConnection.java +++ b/ts3-client/core/src/main/java/com/ts3client/net/TeamspeakConnection.java @@ -27,6 +27,7 @@ import com.ts3client.net.filetransfer.FileTransferManager; import com.ts3client.net.ban.BanEntry; import com.ts3client.net.ban.BanRule; import com.ts3client.net.filetransfer.RemoteFile; +import com.ts3client.recording.VoiceRecorder; import com.ts3client.sound.SoundEvent; import com.ts3client.sound.SoundNotifier; import com.ts3client.text.TsLink; @@ -34,12 +35,14 @@ import com.ts3client.text.TsLink; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -115,6 +118,8 @@ public final class TeamspeakConnection implements TS3Listener { private volatile LocalIdentity identity; private volatile FileTransferManager fileTransfers; private volatile ChatLogStore chatLogs; + /** The recording in progress; guarded by this connection. */ + private VoiceRecorder recorder; /** * Runs ts3j's event callbacks. Supplying it ourselves (rather than leaving ts3j to make * its own) is what lets {@link #awaitEventsProcessed()} tell when everything the server @@ -561,6 +566,12 @@ public final class TeamspeakConnection implements TS3Listener { } private synchronized void safeCleanup() { + // Finishes the files; there is no server left to tell. + VoiceRecorder rec = recorder; + if (rec != null) { + detachRecorder(); + rec.stop(); + } VoiceInput mic = microphone; VoiceOutput out = playback; LocalTeamspeakClientSocket sock = client; @@ -873,6 +884,82 @@ public final class TeamspeakConnection implements TS3Listener { "Setting phonetic nickname failed"); } + // ---- recording ---- + + /** + * Starts recording what we hear on this server, our own voice included, and tells the + * server so: other clients see that they are being recorded. + */ + public synchronized VoiceRecorder startRecording(VoiceRecorder.Mode mode, Path file) throws IOException { + VoiceOutput out = playback; + VoiceInput mic = microphone; + if (!connected || out == null || mic == null) throw new IllegalStateException("Not connected"); + if (recorder != null) throw new IllegalStateException("Already recording"); + + Map names = new HashMap<>(); + for (ClientEntry c : model.allClients()) { + if (!c.uniqueId.isEmpty()) names.putIfAbsent(c.uniqueId, c.nickname); + } + VoiceRecorder rec = VoiceRecorder.start(mode, file, audio.opus(), names); + recorder = rec; + out.setVoiceTap((clientId, pcm, frames) -> { + ClientEntry c = model.getClient(clientId); + if (c != null) rec.offer(c.uniqueId, c.nickname, pcm, 1, frames); + }); + mic.setMonitorListener((pcm, channels) -> { + ClientEntry self = model.getClient(selfClientId); + if (self != null) rec.offer(getSelfUniqueId(), self.nickname, pcm, channels, pcm.length / channels); + }); + // A recording that fails to write ends by itself. + rec.whenDone().whenComplete((files, error) -> { + if (error != null) end(rec, true); + }); + publishRecording(true); + return rec; + } + + /** + * Stops the recording, if any; its files are finished in the background. + * + * @return the recording's files once written, or null if nothing was recording + */ + public CompletableFuture> stopRecording() { + return stopRecording(true); + } + + /** @param announce whether to tell the server; pointless when about to leave it */ + public CompletableFuture> stopRecording(boolean announce) { + VoiceRecorder rec = getRecorder(); + return rec != null && end(rec, announce) ? rec.stop() : null; + } + + /** Ends {@code rec} if it is still the one recording. */ + private boolean end(VoiceRecorder rec, boolean announce) { + synchronized (this) { + if (recorder != rec) return false; + detachRecorder(); + } + if (announce && connected) publishRecording(false); + return true; + } + + public synchronized VoiceRecorder getRecorder() { + return recorder; + } + + private synchronized void detachRecorder() { + recorder = null; + VoiceOutput out = playback; + VoiceInput mic = microphone; + if (out != null) out.setVoiceTap(null); + if (mic != null) mic.setMonitorListener(null); + } + + private void publishRecording(boolean recording) { + selfUpdate(cmd -> cmd.add(new CommandSingleParameter("client_is_recording", recording ? "1" : "0")), + "Recording status update failed"); + } + /** * Applies a change of our own state to the local model. The server sends * {@code notifyclientupdated} to the other clients only, so without this diff --git a/ts3-client/core/src/main/java/com/ts3client/recording/MixdownSink.java b/ts3-client/core/src/main/java/com/ts3client/recording/MixdownSink.java new file mode 100644 index 0000000..a9d6fd4 --- /dev/null +++ b/ts3-client/core/src/main/java/com/ts3client/recording/MixdownSink.java @@ -0,0 +1,85 @@ +package com.ts3client.recording; + +import com.ts3client.audio.VoiceFormat; +import com.ts3client.audio.ogg.OpusHeaders; +import com.ts3client.audio.opus.OpusCodec; +import com.ts3client.audio.opus.OpusEncoder; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.TreeMap; + +/** Mixes every speaker down onto one mono track, written to the file as it goes. */ +final class MixdownSink implements RecordingSink { + + private static final int BITRATE = 64_000; + /** How long a frame waits for the other speakers' frames of the same moment before it is mixed. */ + private static final int MIX_DELAY_FRAMES = 25; + + private final Path file; + private final OpusEncoder encoder; + private final OggOpusStream stream; + /** Frames not yet mixed, by their place on the timeline. */ + private final TreeMap pending = new TreeMap<>(); + private long mixed; + + MixdownSink(Path file, OpusCodec opus) throws IOException { + this.file = file; + this.encoder = RecordingEncoders.mono(opus, BITRATE); + try { + this.stream = new OggOpusStream(file, OpusHeaders.mono(OggOpusStream.PRE_SKIP, VoiceFormat.SAMPLE_RATE), + OpusHeaders.tags(opus.version(), List.of())); + } catch (IOException e) { + encoder.close(); + throw e; + } + } + + @Override + public void write(Track track, long index, float[] frame) { + if (index < mixed) return; // too late to be mixed in + pending.merge(index, frame, (sum, more) -> { + for (int i = 0; i < sum.length; i++) sum[i] += more[i]; + return sum; + }); + } + + @Override + public void advance(long now) throws IOException { + mixUpTo(now - MIX_DELAY_FRAMES); + } + + private void mixUpTo(long end) throws IOException { + for (; mixed < end; mixed++) { + float[] frame = pending.remove(mixed); + if (frame == null) { + frame = RecordingEncoders.SILENT_FRAME; + } else { + for (int i = 0; i < frame.length; i++) frame[i] = Math.max(-1f, Math.min(1f, frame[i])); + } + stream.write(encoder.encode(frame)); + } + } + + @Override + public List finish(long end) throws IOException { + long frames = pending.isEmpty() ? end : Math.max(end, pending.lastKey() + 1); + frames = Math.max(frames, mixed); + mixUpTo(frames); + // One more frame brings out what the encoder's delay still holds. + stream.write(encoder.encode(RecordingEncoders.SILENT_FRAME)); + stream.finish(frames); + encoder.close(); + return List.of(file); + } + + @Override + public void abort() { + try { + stream.close(); + } catch (IOException ignored) { + } + encoder.close(); + } +} diff --git a/ts3-client/core/src/main/java/com/ts3client/recording/MultitrackSink.java b/ts3-client/core/src/main/java/com/ts3client/recording/MultitrackSink.java new file mode 100644 index 0000000..eaac326 --- /dev/null +++ b/ts3-client/core/src/main/java/com/ts3client/recording/MultitrackSink.java @@ -0,0 +1,242 @@ +package com.ts3client.recording; + +import com.ts3client.audio.VoiceFormat; +import com.ts3client.audio.ogg.OpusHeaders; +import com.ts3client.audio.ogg.SelfDelimitedOpus; +import com.ts3client.audio.opus.OpusCodec; +import com.ts3client.audio.opus.OpusEncoder; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Gives every speaker a channel of their own: one uncoupled mono Opus stream each, up to + * {@link OpusHeaders#MAX_DISCRETE_CHANNELS} to a file. How many channels a file has goes in + * its header, so tracks are encoded into spool files as the recording runs and only put + * together into the Ogg files once it stops. A track is silent from the start of the + * recording until its speaker first speaks. + */ +final class MultitrackSink implements RecordingSink { + + private static final int BITRATE = 48_000; + /** Pauses up to this long are encoded as they are; longer ones are filled with a stock silent packet. */ + private static final int ENCODED_PAUSE_FRAMES = 50; + /** + * Frames of silence a fresh encoder encodes before its first speech: its first packets + * after the stock silence come out faded in. + */ + private static final int PRIMING_FRAMES = 5; + + private final Path file; + private final OpusCodec opus; + private final Path spoolDir; + private final byte[] silence; + /** In the order speakers first spoke, which is the order of the channels. */ + private final Map spools = new LinkedHashMap<>(); + + MultitrackSink(Path file, OpusCodec opus) throws IOException { + this.file = file; + this.opus = opus; + this.silence = RecordingEncoders.silentPacket(opus, BITRATE); + Path dir = file.toAbsolutePath().getParent(); + this.spoolDir = Files.createTempDirectory(dir, "." + file.getFileName() + "-"); + } + + @Override + public void write(Track track, long index, float[] frame) throws IOException { + Spool spool = spools.get(track); + if (spool == null) { + spool = new Spool(Math.max(0, index - PRIMING_FRAMES), spoolDir.resolve(spools.size() + ".spool")); + spools.put(track, spool); + } + spool.write(index, frame); + } + + @Override + public void advance(long now) { + } + + @Override + public List finish(long end) throws IOException { + long frames = end; + for (Spool spool : spools.values()) frames = Math.max(frames, spool.next); + for (Spool spool : spools.values()) spool.close(); + // Every track reaches one frame past the end, the encoder's delay. + long total = frames + 1; + + List tracks = new ArrayList<>(spools.keySet()); + List files = new ArrayList<>(); + int perFile = OpusHeaders.MAX_DISCRETE_CHANNELS; + // Even nobody speaking leaves a file behind, as long as the recording ran. + for (int first = 0; first < Math.max(1, tracks.size()); first += perFile) { + Path path = first == 0 ? file : numbered(file, first / perFile + 1); + mux(tracks.subList(first, Math.min(tracks.size(), first + perFile)), path, total, frames); + files.add(path); + } + deleteSpools(); + return files; + } + + /** {@code name.ogg} becomes {@code name-2.ogg}. */ + private static Path numbered(Path file, int number) { + String name = file.getFileName().toString(); + int dot = name.lastIndexOf('.'); + String numbered = dot > 0 + ? name.substring(0, dot) + "-" + number + name.substring(dot) + : name + "-" + number; + return file.resolveSibling(numbered); + } + + private void mux(List tracks, Path path, long total, long frames) throws IOException { + int channels = Math.max(1, tracks.size()); + byte[] head = channels == 1 + ? OpusHeaders.mono(OggOpusStream.PRE_SKIP, VoiceFormat.SAMPLE_RATE) + : OpusHeaders.discrete(channels, OggOpusStream.PRE_SKIP, VoiceFormat.SAMPLE_RATE); + List comments = new ArrayList<>(); + for (int i = 0; i < tracks.size(); i++) { + comments.add("CHANNEL" + (i + 1) + "_NAME=" + tracks.get(i).name); + comments.add("CHANNEL" + (i + 1) + "_UID=" + tracks.get(i).speakerId); + } + + List readers = new ArrayList<>(); + try (OggOpusStream stream = new OggOpusStream(path, head, OpusHeaders.tags(opus.version(), comments))) { + for (Track track : tracks) readers.add(new SpoolReader(spools.get(track))); + List packets = new ArrayList<>(channels); + for (long f = 0; f < total; f++) { + packets.clear(); + for (SpoolReader reader : readers) packets.add(reader.packetAt(f)); + if (packets.isEmpty()) packets.add(silence); + stream.write(SelfDelimitedOpus.join(packets)); + } + stream.finish(frames); + } finally { + for (SpoolReader reader : readers) reader.close(); + } + } + + @Override + public void abort() { + for (Spool spool : spools.values()) { + try { + spool.close(); + } catch (IOException ignored) { + } + } + deleteSpools(); + } + + private void deleteSpools() { + for (Spool spool : spools.values()) { + try { + Files.deleteIfExists(spool.path); + } catch (IOException ignored) { + } + } + try { + Files.deleteIfExists(spoolDir); + } catch (IOException ignored) { + } + } + + /** + * One track's packets from its first frame on. Each entry is a packet's length and bytes, + * or a negative count of frames of silence. + */ + private final class Spool { + final long start; + final Path path; + private final DataOutputStream out; + /** The frame after the last one written. */ + long next; + /** Encodes the current talk burst; a new burst after a long pause gets a fresh one. */ + private OpusEncoder encoder; + + Spool(long start, Path path) throws IOException { + this.start = start; + this.next = start; + this.path = path; + this.out = new DataOutputStream(new BufferedOutputStream(Files.newOutputStream(path), 1 << 14)); + } + + void write(long index, float[] frame) throws IOException { + if (index - next > ENCODED_PAUSE_FRAMES) { + endBurst(); + int silent = (int) (index - next - PRIMING_FRAMES); + out.writeInt(-silent); + next += silent; + } + while (next < index) packet(encoder().encode(RecordingEncoders.SILENT_FRAME)); + packet(encoder().encode(frame)); + } + + private OpusEncoder encoder() { + if (encoder == null) encoder = RecordingEncoders.mono(opus, BITRATE); + return encoder; + } + + private void packet(byte[] packet) throws IOException { + out.writeInt(packet.length); + out.write(packet); + next++; + } + + /** Lets the encoder play out the audio its delay still holds. */ + private void endBurst() throws IOException { + if (encoder == null) return; + packet(encoder.encode(RecordingEncoders.SILENT_FRAME)); + encoder.close(); + encoder = null; + } + + void close() throws IOException { + try { + endBurst(); + } finally { + out.close(); + } + } + } + + private final class SpoolReader implements AutoCloseable { + private final Spool spool; + private final DataInputStream in; + private int silentFrames; + + SpoolReader(Spool spool) throws IOException { + this.spool = spool; + this.in = new DataInputStream(new BufferedInputStream(Files.newInputStream(spool.path), 1 << 14)); + } + + /** Must be asked for every frame in turn. */ + byte[] packetAt(long frame) throws IOException { + if (frame < spool.start || frame >= spool.next) return silence; + if (silentFrames > 0) { + silentFrames--; + return silence; + } + int length = in.readInt(); + if (length < 0) { + silentFrames = -length - 1; + return silence; + } + return in.readNBytes(length); + } + + @Override + public void close() { + try { + in.close(); + } catch (IOException ignored) { + } + } + } +} diff --git a/ts3-client/core/src/main/java/com/ts3client/recording/OggOpusStream.java b/ts3-client/core/src/main/java/com/ts3client/recording/OggOpusStream.java new file mode 100644 index 0000000..3147065 --- /dev/null +++ b/ts3-client/core/src/main/java/com/ts3client/recording/OggOpusStream.java @@ -0,0 +1,55 @@ +package com.ts3client.recording; + +import com.ts3client.audio.VoiceFormat; +import com.ts3client.audio.ogg.OggPageWriter; +import com.ts3client.audio.ogg.OpusHeaders; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.ThreadLocalRandom; + +/** An Ogg Opus file written a 20 ms packet at a time. */ +final class OggOpusStream implements AutoCloseable { + + static final int PRE_SKIP = OpusHeaders.LIBOPUS_PRE_SKIP; + + private final OutputStream file; + private final OggPageWriter ogg; + /** The latest packet, held back until it is known whether it ends the stream. */ + private byte[] held; + private long packets; + + OggOpusStream(Path path, byte[] head, byte[] tags) throws IOException { + file = new BufferedOutputStream(Files.newOutputStream(path), 1 << 16); + ogg = new OggPageWriter(file, ThreadLocalRandom.current().nextInt()); + ogg.write(head, 0); + ogg.flush(); + ogg.write(tags, 0); + ogg.flush(); + } + + void write(byte[] packet) throws IOException { + if (held != null) ogg.write(held, packets * VoiceFormat.FRAME_SIZE); + held = packet; + packets++; + } + + /** + * Ends the stream after {@code frames} frames of audio; the packets written must reach + * past them by the encoder's delay. + */ + void finish(long frames) throws IOException { + if (held != null) ogg.write(held, PRE_SKIP + frames * VoiceFormat.FRAME_SIZE); + held = null; + ogg.finish(); + file.close(); + } + + @Override + public void close() throws IOException { + file.close(); + } +} diff --git a/ts3-client/core/src/main/java/com/ts3client/recording/RecordingEncoders.java b/ts3-client/core/src/main/java/com/ts3client/recording/RecordingEncoders.java new file mode 100644 index 0000000..31c7e2a --- /dev/null +++ b/ts3-client/core/src/main/java/com/ts3client/recording/RecordingEncoders.java @@ -0,0 +1,33 @@ +package com.ts3client.recording; + +import com.ts3client.audio.VoiceFormat; +import com.ts3client.audio.opus.OpusCodec; +import com.ts3client.audio.opus.OpusEncoder; + +/** The Opus encoders a recording's tracks are encoded with. */ +final class RecordingEncoders { + + static final float[] SILENT_FRAME = new float[VoiceFormat.FRAME_SIZE]; + + private RecordingEncoders() { + } + + static OpusEncoder mono(OpusCodec opus, int bitrate) { + OpusEncoder encoder = opus.createEncoder(VoiceFormat.SAMPLE_RATE, VoiceFormat.FRAME_SIZE, 1, + OpusCodec.Application.AUDIO); + encoder.setBitrate(bitrate); + encoder.setVbr(true); + encoder.setComplexity(10); + return encoder; + } + + /** A packet that decodes to a frame of silence, to stand in where a track has nothing. */ + static byte[] silentPacket(OpusCodec opus, int bitrate) { + try (OpusEncoder encoder = mono(opus, bitrate)) { + byte[] packet = null; + // Past the first frames, which still carry the encoder's start-up. + for (int i = 0; i < 4; i++) packet = encoder.encode(SILENT_FRAME); + return packet; + } + } +} diff --git a/ts3-client/core/src/main/java/com/ts3client/recording/RecordingSink.java b/ts3-client/core/src/main/java/com/ts3client/recording/RecordingSink.java new file mode 100644 index 0000000..b49cd39 --- /dev/null +++ b/ts3-client/core/src/main/java/com/ts3client/recording/RecordingSink.java @@ -0,0 +1,26 @@ +package com.ts3client.recording; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +/** Where a recording's frames are encoded and stored; driven from the recorder's one thread. */ +interface RecordingSink { + + /** @param index the frame's place on the timeline, past every earlier frame of the track */ + void write(Track track, long index, float[] frame) throws IOException; + + /** The clock reached frame {@code now}; frames well behind it may be finalised. */ + void advance(long now) throws IOException; + + /** + * Writes everything out, ending the recording at frame {@code end} or after the last + * frame written, whichever is later. + * + * @return the files written + */ + List finish(long end) throws IOException; + + /** Gives up after a failure, releasing what is held. */ + void abort(); +} diff --git a/ts3-client/core/src/main/java/com/ts3client/recording/Track.java b/ts3-client/core/src/main/java/com/ts3client/recording/Track.java new file mode 100644 index 0000000..b5db749 --- /dev/null +++ b/ts3-client/core/src/main/java/com/ts3client/recording/Track.java @@ -0,0 +1,49 @@ +package com.ts3client.recording; + +import java.util.concurrent.TimeUnit; + +/** + * One speaker in a recording: where each of their frames goes on the recording's timeline. + * + *

Frames arrive as they are played, 20 ms apart within a talk burst, so a burst is + * laid down back to back from where its first frame arrived; scheduling jitter then never + * tears holes into it. Only when the speaker's clock drifts too far from the recording's + * is the burst moved back into place. + */ +final class Track { + + /** A pause longer than this starts a new talk burst, placed afresh. */ + private static final long BURST_GAP_NANOS = TimeUnit.MILLISECONDS.toNanos(60); + /** How many frames a burst may run behind or ahead of the clock before it is moved. */ + private static final int MAX_DRIFT_FRAMES = 5; + + final String speakerId; + final String name; + + private long next = -1; + private long lastNanos; + + Track(String speakerId, String name) { + this.speakerId = speakerId; + this.name = name; + } + + /** + * @param due the frame the clock says was playing when the frame arrived + * @return where the frame goes, never before an earlier one; -1 to drop it, for a burst + * that has run too far ahead + */ + long place(long due, long nanos) { + long index; + if (next < 0 || nanos - lastNanos > BURST_GAP_NANOS) { + index = Math.max(next, due); + } else if (next - due > MAX_DRIFT_FRAMES) { + return -1; + } else { + index = due - next > MAX_DRIFT_FRAMES ? due : next; + } + next = index + 1; + lastNanos = nanos; + return index; + } +} diff --git a/ts3-client/core/src/main/java/com/ts3client/recording/VoiceRecorder.java b/ts3-client/core/src/main/java/com/ts3client/recording/VoiceRecorder.java new file mode 100644 index 0000000..69fd4f5 --- /dev/null +++ b/ts3-client/core/src/main/java/com/ts3client/recording/VoiceRecorder.java @@ -0,0 +1,171 @@ +package com.ts3client.recording; + +import com.ts3client.audio.VoiceFormat; +import com.ts3client.audio.opus.OpusCodec; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.LongSupplier; + +/** + * Records voice into Ogg Opus: everyone mixed down onto one mono track, or every speaker + * on a channel of their own. Speakers are told apart by their unique id, so somebody who + * reconnects keeps their channel. + * + *

Frames are handed in as they are played and encoded on a thread of the recorder's + * own; {@link #offer} never blocks, so it may be called from audio threads. + */ +public final class VoiceRecorder { + + public enum Mode { + /** All speakers mixed down to mono. */ + MIXED, + /** A channel per speaker, as many files as it takes. */ + MULTITRACK + } + + private static final int FRAME = VoiceFormat.FRAME_SIZE; + private static final long FRAME_NANOS = TimeUnit.SECONDS.toNanos(FRAME) / VoiceFormat.SAMPLE_RATE; + /** How often the writer looks at the clock while nobody speaks. */ + private static final long IDLE_POLL_MILLIS = 100; + + private record Frame(String speakerId, String nickname, long nanos, float[] mono) { + } + + private record Stop(long nanos) { + } + + private final Mode mode; + private final Path file; + private final Map namesAtStart; + private final RecordingSink sink; + private final LongSupplier clock; + private final long startNanos; + private final LinkedBlockingQueue queue = new LinkedBlockingQueue<>(); + private final AtomicBoolean stopped = new AtomicBoolean(); + private final CompletableFuture> done = new CompletableFuture<>(); + /** Touched only by the writer thread. */ + private final Map tracks = new HashMap<>(); + + /** + * Starts recording into {@code file}; a multitrack recording with more speakers than one + * file holds continues into {@code name-2.ogg} and so on. + * + * @param namesAtStart nicknames by unique id of everyone present now, which name their + * channels; later arrivals are named as they first speak + */ + public static VoiceRecorder start(Mode mode, Path file, OpusCodec opus, Map namesAtStart) + throws IOException { + return start(mode, file, opus, namesAtStart, System::nanoTime); + } + + /** @param clock nanoseconds, as {@link System#nanoTime} counts them */ + static VoiceRecorder start(Mode mode, Path file, OpusCodec opus, Map namesAtStart, + LongSupplier clock) throws IOException { + RecordingSink sink = mode == Mode.MIXED ? new MixdownSink(file, opus) : new MultitrackSink(file, opus); + VoiceRecorder recorder = new VoiceRecorder(mode, file, namesAtStart, sink, clock); + Thread writer = new Thread(recorder::run, "ts3j-recorder"); + writer.setDaemon(true); + writer.start(); + return recorder; + } + + private VoiceRecorder(Mode mode, Path file, Map namesAtStart, RecordingSink sink, + LongSupplier clock) { + this.clock = clock; + this.startNanos = clock.getAsLong(); + this.mode = mode; + this.file = file; + this.namesAtStart = Map.copyOf(namesAtStart); + this.sink = sink; + } + + public Mode mode() { + return mode; + } + + public Path file() { + return file; + } + + /** + * Takes a frame a speaker is being heard with. Copied, so the caller may reuse it. + * + * @param pcm interleaved samples in [-1, 1] + * @param frames samples per channel, at most one 20 ms frame + */ + public void offer(String speakerId, String nickname, float[] pcm, int channels, int frames) { + if (stopped.get() || frames <= 0 || speakerId == null || speakerId.isEmpty()) return; + long now = clock.getAsLong(); + float[] mono = new float[FRAME]; + int n = Math.min(frames, FRAME); + if (channels == 1) { + System.arraycopy(pcm, 0, mono, 0, n); + } else { + for (int i = 0; i < n; i++) { + float sum = 0; + for (int c = 0; c < channels; c++) sum += pcm[i * channels + c]; + mono[i] = sum / channels; + } + } + queue.add(new Frame(speakerId, nickname, now, mono)); + } + + /** + * Stops recording. The files are finished on the recorder's thread, which for a long + * multitrack recording takes a moment. + * + * @return the files written, once they are; failed if the recording could not be written + */ + public CompletableFuture> stop() { + if (stopped.compareAndSet(false, true)) queue.add(new Stop(clock.getAsLong())); + return done; + } + + /** Completes when the recording ends, whether stopped or failed. */ + public CompletableFuture> whenDone() { + return done; + } + + private void run() { + try { + while (true) { + Object item = queue.poll(IDLE_POLL_MILLIS, TimeUnit.MILLISECONDS); + if (item instanceof Stop stop) { + done.complete(sink.finish(frameAt(stop.nanos()))); + return; + } + // Frames are queued in the order they were played, so a frame's own time is + // as far as the clock may be trusted while more are still queued behind it. + if (item instanceof Frame frame) { + place(frame); + sink.advance(frameAt(frame.nanos())); + } else { + sink.advance(frameAt(clock.getAsLong())); + } + } + } catch (Throwable e) { + stopped.set(true); + sink.abort(); + done.completeExceptionally(e); + } + } + + private void place(Frame frame) throws IOException { + Track track = tracks.computeIfAbsent(frame.speakerId(), id -> + new Track(id, namesAtStart.getOrDefault(id, frame.nickname()))); + long index = track.place(frameAt(frame.nanos()), frame.nanos()); + if (index >= 0) sink.write(track, index, frame.mono()); + } + + private long frameAt(long nanos) { + return Math.max(0, (nanos - startNanos) / FRAME_NANOS); + } +} diff --git a/ts3-client/core/src/main/java/com/ts3client/session/ServerSession.java b/ts3-client/core/src/main/java/com/ts3client/session/ServerSession.java index a8adb6d..348dff3 100644 --- a/ts3-client/core/src/main/java/com/ts3client/session/ServerSession.java +++ b/ts3-client/core/src/main/java/com/ts3client/session/ServerSession.java @@ -10,10 +10,18 @@ import com.ts3client.net.ClientEntry; import com.ts3client.net.ConnectionListener; import com.ts3client.net.ServerModel; import com.ts3client.net.TeamspeakConnection; +import com.ts3client.recording.VoiceRecorder; import com.ts3client.sound.SoundNotifier; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; /** * One server connection and the local client's state on it, independent of any UI: the @@ -203,9 +211,17 @@ public final class ServerSession { if (isBusy()) conn.disconnect(); } - /** Synchronous teardown for shutdown paths, so the server sees us leave. */ + /** Synchronous teardown for shutdown paths, so the server sees us leave and recordings are written. */ public void shutdown() { + CompletableFuture> recording = conn.stopRecording(false); if (isBusy()) conn.disconnectBlocking("Leaving"); + if (recording != null) { + try { + recording.join(); + } catch (RuntimeException ignored) { + // Already reported. + } + } } /** @@ -356,6 +372,59 @@ public final class ServerSession { return conn.getModel().getClient(conn.getSelfClientId()); } + // ---- recording ---- + + public boolean isRecording() { + return conn.getRecorder() != null; + } + + /** The recording in progress, or null. */ + public VoiceRecorder.Mode recordingMode() { + VoiceRecorder rec = conn.getRecorder(); + return rec == null ? null : rec.mode(); + } + + /** A file name for a recording started now, unique within {@code dir}. */ + public Path recordingFile(Path dir) { + String server = title.replaceAll("[\\\\/:*?\"<>|\\p{Cntrl}]", "_").strip(); + String name = (server.isEmpty() ? "recording" : server) + " " + + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH-mm-ss")); + Path file = dir.resolve(name + ".ogg"); + for (int n = 2; Files.exists(file); n++) file = dir.resolve(name + " (" + n + ").ogg"); + return file; + } + + public void startRecording(VoiceRecorder.Mode mode, Path file) { + if (!conn.isConnected() || isRecording()) return; + VoiceRecorder rec; + try { + Path dir = file.toAbsolutePath().getParent(); + if (dir != null) Files.createDirectories(dir); + rec = conn.startRecording(mode, file); + } catch (IOException | RuntimeException e) { + listener.onSystemMessage("Failed to start recording: " + e.getMessage()); + return; + } + boolean multitrack = mode == VoiceRecorder.Mode.MULTITRACK; + listener.onSystemMessage((multitrack ? "Started multitrack recording: " : "Started recording: ") + file); + rec.whenDone().whenComplete((files, error) -> { + if (error != null) { + Throwable cause = error.getCause() != null ? error.getCause() : error; + listener.onSystemMessage("Recording failed: " + cause.getMessage()); + } else { + listener.onSystemMessage((multitrack ? "Stopped multitrack recording: " : "Stopped recording: ") + + files.stream().map(Path::toString).collect(Collectors.joining(", "))); + } + listener.onSessionChanged(); + }); + listener.onSessionChanged(); + } + + /** Stops the recording; its files are finished in the background, then reported. */ + public void stopRecording() { + if (conn.stopRecording() != null) listener.onSessionChanged(); + } + // ---- chat ---- /** diff --git a/ts3-client/core/src/main/java/com/ts3client/sound/OggOpusFile.java b/ts3-client/core/src/main/java/com/ts3client/sound/OggOpusFile.java index 68456b8..55e8cb8 100644 --- a/ts3-client/core/src/main/java/com/ts3client/sound/OggOpusFile.java +++ b/ts3-client/core/src/main/java/com/ts3client/sound/OggOpusFile.java @@ -1,6 +1,8 @@ package com.ts3client.sound; import com.ts3client.audio.VoiceFormat; +import com.ts3client.audio.ogg.OggPageWriter; +import com.ts3client.audio.ogg.OpusHeaders; import com.ts3client.audio.opus.OpusCodec; import com.ts3client.audio.opus.OpusDecoder; import com.ts3client.audio.opus.OpusEncoder; @@ -27,16 +29,7 @@ public final class OggOpusFile { /** Opus always decodes at 48 kHz; 120 ms is the longest packet. */ private static final int MAX_PACKET_SAMPLES = 5760; private static final int FRAME_SAMPLES = VoiceFormat.FRAME_SIZE; - /** - * libopus' encoder delay at 48 kHz for 20 ms frames, which the file tells the - * decoder to drop (the SPI does not expose {@code OPUS_GET_LOOKAHEAD}). - */ - private static final int PRE_SKIP = 312; - /** Packets per page: a page holds up to 255 segments, and short sounds are a few pages anyway. */ - private static final int PACKETS_PER_PAGE = 50; - - private static final int FLAG_FIRST = 0x02; - private static final int FLAG_LAST = 0x04; + private static final int PRE_SKIP = OpusHeaders.LIBOPUS_PRE_SKIP; private OggOpusFile() { } @@ -91,79 +84,26 @@ public final class OggOpusFile { audio.add(encoder.encode(frame)); } - int serial = file.getName().hashCode(); ByteArrayOutputStream out = new ByteArrayOutputStream(); - int sequence = 0; - writePage(out, FLAG_FIRST, 0, serial, sequence++, List.of(opusHead(inputRate))); - writePage(out, 0, 0, serial, sequence++, List.of(opusTags())); - for (int first = 0; first < audio.size(); first += PACKETS_PER_PAGE) { - int last = Math.min(audio.size(), first + PACKETS_PER_PAGE); - boolean end = last == audio.size(); - long granule = end ? total : (long) last * FRAME_SAMPLES; - writePage(out, end ? FLAG_LAST : 0, granule, serial, sequence++, audio.subList(first, last)); + OggPageWriter ogg = new OggPageWriter(out, file.getName().hashCode()); + ogg.write(OpusHeaders.mono(PRE_SKIP, inputRate), 0); + ogg.flush(); + ogg.write(OpusHeaders.tags("ts3j", List.of()), 0); + ogg.flush(); + for (int i = 0; i < audio.size(); i++) { + // The last packet's position marks where the sound ends, short of the padding. + boolean end = i == audio.size() - 1; + ogg.write(audio.get(i), end ? total : (long) (i + 1) * FRAME_SAMPLES); } + ogg.finish(); Files.write(file.toPath(), out.toByteArray()); } - private static byte[] opusHead(int inputRate) { - ByteBuffer b = ByteBuffer.allocate(19).order(ByteOrder.LITTLE_ENDIAN); - b.put("OpusHead".getBytes(StandardCharsets.US_ASCII)); - b.put((byte) 1); // version - b.put((byte) 1); // channels - b.putShort((short) PRE_SKIP); - b.putInt(inputRate); - b.putShort((short) 0); // output gain - b.put((byte) 0); // channel mapping family: mono/stereo - return b.array(); - } - - private static byte[] opusTags() { - byte[] vendor = "ts3j".getBytes(StandardCharsets.UTF_8); - ByteBuffer b = ByteBuffer.allocate(8 + 4 + vendor.length + 4).order(ByteOrder.LITTLE_ENDIAN); - b.put("OpusTags".getBytes(StandardCharsets.US_ASCII)); - b.putInt(vendor.length).put(vendor); - b.putInt(0); // no user comments - return b.array(); - } - // ---- Ogg framing ---- private record Page(long granule, byte[] segments, byte[] data) { } - private static void writePage(ByteArrayOutputStream out, int flags, long granule, int serial, int sequence, - List packets) { - ByteArrayOutputStream lacing = new ByteArrayOutputStream(); - ByteArrayOutputStream data = new ByteArrayOutputStream(); - for (byte[] packet : packets) { - // A packet is split into 255-byte segments and ends with a shorter one (possibly empty). - int left = packet.length; - while (left >= 255) { - lacing.write(255); - left -= 255; - } - lacing.write(left); - data.write(packet, 0, packet.length); - } - ByteBuffer header = ByteBuffer.allocate(27 + lacing.size()).order(ByteOrder.LITTLE_ENDIAN); - header.put("OggS".getBytes(StandardCharsets.US_ASCII)); - header.put((byte) 0); - header.put((byte) flags); - header.putLong(granule); - header.putInt(serial); - header.putInt(sequence); - header.putInt(0); // checksum, filled in below - header.put((byte) lacing.size()); - header.put(lacing.toByteArray()); - - byte[] page = new byte[header.capacity() + data.size()]; - System.arraycopy(header.array(), 0, page, 0, header.capacity()); - System.arraycopy(data.toByteArray(), 0, page, header.capacity(), data.size()); - int crc = crc(page); - ByteBuffer.wrap(page).order(ByteOrder.LITTLE_ENDIAN).putInt(22, crc); - out.write(page, 0, page.length); - } - private static List pages(byte[] file) throws IOException { List pages = new ArrayList<>(); ByteBuffer in = ByteBuffer.wrap(file).order(ByteOrder.LITTLE_ENDIAN); @@ -205,14 +145,4 @@ public final class OggOpusFile { } return packets; } - - /** Ogg's CRC-32: polynomial 0x04c11db7, not reflected, starting from 0. */ - private static int crc(byte[] data) { - int crc = 0; - for (byte b : data) { - crc ^= (b & 0xff) << 24; - for (int i = 0; i < 8; i++) crc = (crc & 0x80000000) != 0 ? (crc << 1) ^ 0x04c11db7 : crc << 1; - } - return crc; - } } diff --git a/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java b/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java index 4d197bc..49e98f4 100644 --- a/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java +++ b/ts3-client/core/src/test/java/com/ts3client/audio/VoiceStreamTest.java @@ -44,6 +44,20 @@ class VoiceStreamTest { assertEquals(List.of(true), talk); } + @Test + void tapHearsTheSpeakerInMonoAtFullVolume() { + List tapped = new ArrayList<>(); + stream.setTap((clientId, mono, frames) -> { + assertEquals(7, clientId); + assertEquals(VoiceFormat.FRAME_SIZE, frames); + tapped.add(mono[frames - 1]); + }); + stream.offer(0, CodecType.OPUS_MUSIC, FakeOpus.packet(40)); + stream.pull(out, 2, 0.5); + assertEquals(0.2f, out[1]); + assertEquals(List.of(0.4f), tapped); + } + @Test void concealsALostPacket() { offer(0, 1); diff --git a/ts3-client/desktop/src/test/java/com/ts3client/recording/LibOpusMultistream.java b/ts3-client/desktop/src/test/java/com/ts3client/recording/LibOpusMultistream.java new file mode 100644 index 0000000..b7ab270 --- /dev/null +++ b/ts3-client/desktop/src/test/java/com/ts3client/recording/LibOpusMultistream.java @@ -0,0 +1,59 @@ +package com.ts3client.recording; + +import java.lang.foreign.Arena; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SymbolLookup; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; + +/** libopus' own multistream decoder, to check our multistream packets against. */ +final class LibOpusMultistream implements AutoCloseable { + + private static final Linker LINKER = Linker.nativeLinker(); + private static final SymbolLookup LIB = SymbolLookup.libraryLookup("libopus.so.0", Arena.global()); + private static final MethodHandle CREATE = LINKER.downcallHandle(LIB.find("opus_multistream_decoder_create").orElseThrow(), + FunctionDescriptor.of(ValueLayout.ADDRESS, ValueLayout.JAVA_INT, ValueLayout.JAVA_INT, ValueLayout.JAVA_INT, + ValueLayout.JAVA_INT, ValueLayout.ADDRESS, ValueLayout.ADDRESS)); + private static final MethodHandle DECODE = LINKER.downcallHandle(LIB.find("opus_multistream_decode_float").orElseThrow(), + FunctionDescriptor.of(ValueLayout.JAVA_INT, ValueLayout.ADDRESS, ValueLayout.ADDRESS, ValueLayout.JAVA_INT, + ValueLayout.ADDRESS, ValueLayout.JAVA_INT, ValueLayout.JAVA_INT)); + private static final MethodHandle DESTROY = LINKER.downcallHandle(LIB.find("opus_multistream_decoder_destroy").orElseThrow(), + FunctionDescriptor.ofVoid(ValueLayout.ADDRESS)); + + private static final int MAX_FRAME = 5760; + + private final Arena arena = Arena.ofConfined(); + private final MemorySegment decoder; + private final MemorySegment pcm; + private final MemorySegment data = arena.allocate(1 << 20); + final int channels; + + LibOpusMultistream(int channels, int streams, int coupled, byte[] mapping) throws Throwable { + this.channels = channels; + MemorySegment map = arena.allocate(mapping.length); + map.copyFrom(MemorySegment.ofArray(mapping)); + MemorySegment error = arena.allocate(ValueLayout.JAVA_INT); + decoder = (MemorySegment) CREATE.invokeExact(48_000, channels, streams, coupled, map, error); + if (error.get(ValueLayout.JAVA_INT, 0) != 0) throw new IllegalStateException("create: " + error.get(ValueLayout.JAVA_INT, 0)); + pcm = arena.allocate(ValueLayout.JAVA_FLOAT, (long) MAX_FRAME * channels); + } + + /** @return interleaved samples, or throws if libopus rejects the packet */ + float[] decode(byte[] packet) throws Throwable { + MemorySegment.copy(packet, 0, data, ValueLayout.JAVA_BYTE, 0, packet.length); + int n = (int) DECODE.invokeExact(decoder, data, packet.length, pcm, MAX_FRAME, 0); + if (n < 0) throw new IllegalStateException("opus_multistream_decode_float: " + n); + return pcm.asSlice(0, (long) n * channels * 4).toArray(ValueLayout.JAVA_FLOAT); + } + + @Override + public void close() { + try { + DESTROY.invokeExact(decoder); + } catch (Throwable ignored) { + } + arena.close(); + } +} diff --git a/ts3-client/desktop/src/test/java/com/ts3client/recording/VoiceRecorderTest.java b/ts3-client/desktop/src/test/java/com/ts3client/recording/VoiceRecorderTest.java new file mode 100644 index 0000000..61cf7d7 --- /dev/null +++ b/ts3-client/desktop/src/test/java/com/ts3client/recording/VoiceRecorderTest.java @@ -0,0 +1,313 @@ +package com.ts3client.recording; + +import com.ts3client.audio.VoiceFormat; +import com.ts3client.audio.desktop.NativeOpusCodec; +import com.ts3client.audio.ogg.SelfDelimitedOpus; +import com.ts3client.audio.opus.OpusEncoder; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Recordings decode, through libopus' own multistream decoder, to every speaker where and + * when they spoke. + */ +class VoiceRecorderTest { + + private static final int FRAME = VoiceFormat.FRAME_SIZE; + private static final long FRAME_NANOS = TimeUnit.MILLISECONDS.toNanos(20); + /** Frames the simulated recordings run for. */ + private static final int LENGTH = 250; + + @TempDir + Path dir; + + private final NativeOpusCodec opus = new NativeOpusCodec(); + + /** Who speaks when, in frames, at which pitch. */ + private record Burst(String uid, String nickname, int from, int to, double hz) { + } + + private static final List CONVERSATION = List.of( + new Burst("uidA", "Alice (renamed)", 50, 100, 440), + new Burst("uidB", "Bob", 150, 175, 660), + // After a pause long enough for a fresh encoder. + new Burst("uidA", "Alice (renamed)", 200, 210, 440)); + + @Test + void multitrackPutsEverySpeakerOnTheirOwnChannelFromTheStart() throws Throwable { + Path file = dir.resolve("multi.ogg"); + List files = record(VoiceRecorder.Mode.MULTITRACK, file, Map.of("uidA", "Alice"), CONVERSATION); + assertEquals(List.of(file), files); + + Decoded d = decode(file); + assertEquals(2, d.channels); + assertTrue(d.comments.containsAll(List.of("CHANNEL1_NAME=Alice", "CHANNEL1_UID=uidA", + "CHANNEL2_NAME=Bob", "CHANNEL2_UID=uidB")), d.comments.toString()); + assertEquals(LENGTH * FRAME, d.frames()); + + assertTone(d, 0, 50, 100, 440); + assertSilent(d, 0, 0, 49); + assertSilent(d, 0, 102, 199); + assertTone(d, 0, 200, 210, 440); + assertTone(d, 1, 150, 175, 660); + assertStartsAt(d, 0, 50); + assertStartsAt(d, 0, 200); + assertStartsAt(d, 1, 150); + assertSilent(d, 1, 0, 149); + assertSilent(d, 1, 177, LENGTH); + assertTrue(Files.list(dir).noneMatch(p -> p.getFileName().toString().startsWith(".")), "spool left behind"); + } + + @Test + void mixedPutsEveryoneOnOneMonoTrack() throws Throwable { + Path file = dir.resolve("mixed.ogg"); + record(VoiceRecorder.Mode.MIXED, file, Map.of(), CONVERSATION); + + Decoded d = decode(file); + assertEquals(1, d.channels); + assertEquals(LENGTH * FRAME, d.frames()); + assertTone(d, 0, 50, 100, 440); + assertTone(d, 0, 150, 175, 660); + assertSilent(d, 0, 0, 49); + assertSilent(d, 0, 102, 149); + } + + @Test + void moreSpeakersThanAFileHoldsContinueInTheNextFile() throws Throwable { + List crowd = new ArrayList<>(); + for (int i = 0; i < 300; i++) crowd.add(new Burst("uid" + i, "Client " + i, i % 200, i % 200 + 5, 440)); + crowd.sort((a, b) -> Integer.compare(a.from, b.from)); + Path file = dir.resolve("crowd.ogg"); + List files = record(VoiceRecorder.Mode.MULTITRACK, file, Map.of(), crowd); + assertEquals(List.of(file, dir.resolve("crowd-2.ogg")), files); + + Decoded first = decode(files.get(0)); + Decoded second = decode(files.get(1)); + assertEquals(255, first.channels); + assertEquals(45, second.channels); + assertEquals(LENGTH * FRAME, first.frames()); + assertEquals(LENGTH * FRAME, second.frames()); + // Channels go in the order speakers first spoke. + assertTone(first, 0, 0, 5, 440); + // Two speakers a frame start until frame 100, one after: the 256th first spoke at 155. + assertTone(second, 0, 155, 160, 440); + assertStartsAt(second, 0, 155); + assertSilent(second, 0, 0, 154); + } + + @Test + void multiFramePacketsAreSplitWhereLibopusSplitsThem() throws Throwable { + byte[][] f = new byte[6][]; + try (OpusEncoder encoder = RecordingEncoders.mono(opus, 48_000)) { + for (int i = 0; i < f.length; i++) f[i] = encoder.encode(tone(440 + 50 * i, i)); + } + byte[] padded = code3(f[0], f[1], 3); + byte[] code2 = cat(new byte[]{(byte) ((f[2][0] & 0xfc) | 2), (byte) (f[2].length - 1)}, + Arrays.copyOfRange(f[2], 1, f[2].length), Arrays.copyOfRange(f[3], 1, f[3].length)); + byte[] last = code3(f[4], f[5], 0); + byte[] joined = SelfDelimitedOpus.join(List.of(padded, code2, last)); + + try (LibOpusMultistream decoder = new LibOpusMultistream(3, 3, 0, new byte[]{0, 1, 2})) { + assertEquals(3 * 2 * FRAME, decoder.decode(joined).length); + } + } + + /** Two single-frame packets of one configuration as one VBR code 3 packet. */ + private static byte[] code3(byte[] a, byte[] b, int padding) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write((a[0] & 0xfc) | 3); + out.write(0x80 | (padding > 0 ? 0x40 : 0) | 2); + if (padding > 0) out.write(padding); + out.write(a.length - 1); + out.write(a, 1, a.length - 1); + out.write(b, 1, b.length - 1); + out.write(new byte[padding], 0, padding); + return out.toByteArray(); + } + + private static byte[] cat(byte[]... parts) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (byte[] p : parts) out.writeBytes(p); + return out.toByteArray(); + } + + // ---- simulation ---- + + private List record(VoiceRecorder.Mode mode, Path file, Map names, List bursts) + throws Exception { + AtomicLong clock = new AtomicLong(1_000_000_000L); + long start = clock.get(); + VoiceRecorder recorder = VoiceRecorder.start(mode, file, opus, names, clock::get); + for (int f = 0; f < LENGTH; f++) { + for (Burst burst : bursts) { + if (f < burst.from || f >= burst.to) continue; + // Played a little late, as audio threads are. + clock.set(start + f * FRAME_NANOS + (f * 7919L % 5) * 1_000_000L); + recorder.offer(burst.uid, burst.nickname, tone(burst.hz, f), 1, FRAME); + } + } + clock.set(start + LENGTH * FRAME_NANOS); + List files = recorder.stop().get(60, TimeUnit.SECONDS); + String dump = System.getProperty("recording.dump"); + if (dump != null) { + for (Path p : files) Files.copy(p, Path.of(dump).resolve(p.getFileName()), StandardCopyOption.REPLACE_EXISTING); + } + return files; + } + + private static float[] tone(double hz, int frame) { + float[] pcm = new float[FRAME]; + for (int i = 0; i < FRAME; i++) { + pcm[i] = (float) (0.5 * Math.sin(2 * Math.PI * hz * ((long) frame * FRAME + i) / VoiceFormat.SAMPLE_RATE)); + } + return pcm; + } + + // ---- checks ---- + + /** A tone of {@code hz} over the frames, give or take the edges. */ + private static void assertTone(Decoded d, int channel, int from, int to, double hz) { + int start = (from + 1) * FRAME; + int end = (to - 1) * FRAME; + double energy = 0; + double sin = 0; + double cos = 0; + for (int i = start; i < end; i++) { + double x = d.sample(i, channel); + double phase = 2 * Math.PI * hz * i / VoiceFormat.SAMPLE_RATE; + sin += x * Math.sin(phase); + cos += x * Math.cos(phase); + energy += x * x; + } + double amplitude = Math.sqrt(sin * sin + cos * cos) * 2 / (end - start); + double rms = Math.sqrt(energy / (end - start)); + assertTrue(amplitude > 0.4, "channel " + channel + " frames " + from + "-" + to + ": amplitude at " + hz + " Hz " + amplitude); + assertTrue(rms > 0.3 && rms < 0.42, "channel " + channel + " frames " + from + "-" + to + ": rms " + rms); + } + + /** Sound sets in right at the frame, not faded in. */ + private static void assertStartsAt(Decoded d, int channel, int frame) { + int onset = frame * FRAME - FRAME; + while (Math.abs(d.sample(onset, channel)) < 0.05) onset++; + assertEquals(frame * FRAME, onset, 48.0, "channel " + channel + " onset"); + } + + private static void assertSilent(Decoded d, int channel, int from, int to) { + double peak = 0; + for (int i = from * FRAME; i < Math.min(to * FRAME, d.frames()); i++) peak = Math.max(peak, Math.abs(d.sample(i, channel))); + assertTrue(peak < 0.01, "channel " + channel + " frames " + from + "-" + to + ": peak " + peak); + } + + // ---- decoding ---- + + private record Decoded(int channels, List comments, float[] pcm) { + int frames() { + return pcm.length / channels; + } + + double sample(int i, int channel) { + return pcm[i * channels + channel]; + } + } + + private static Decoded decode(Path file) throws Throwable { + List packets = new ArrayList<>(); + long granule = readOgg(Files.readAllBytes(file), packets); + + ByteBuffer head = ByteBuffer.wrap(packets.get(0)).order(ByteOrder.LITTLE_ENDIAN); + assertEquals("OpusHead", new String(packets.get(0), 0, 8, StandardCharsets.US_ASCII)); + int channels = head.get(9) & 0xff; + int preSkip = head.getShort(10) & 0xffff; + int family = head.get(18) & 0xff; + int streams = 1; + int coupled = 0; + byte[] mapping = {0}; + if (family != 0) { + assertEquals(255, family); + streams = head.get(19) & 0xff; + coupled = head.get(20) & 0xff; + mapping = Arrays.copyOfRange(packets.get(0), 21, 21 + channels); + } + + ByteBuffer tags = ByteBuffer.wrap(packets.get(1)).order(ByteOrder.LITTLE_ENDIAN); + tags.position(8); + int vendor = tags.getInt(); + tags.position(tags.position() + vendor); + int count = tags.getInt(); + List comments = new ArrayList<>(); + for (int i = 0; i < count; i++) { + byte[] s = new byte[tags.getInt()]; + tags.get(s); + comments.add(new String(s, StandardCharsets.UTF_8)); + } + + float[] pcm = new float[0]; + int length = 0; + try (LibOpusMultistream decoder = new LibOpusMultistream(channels, streams, coupled, mapping)) { + for (byte[] packet : packets.subList(2, packets.size())) { + float[] out = decoder.decode(packet); + if (length + out.length > pcm.length) pcm = Arrays.copyOf(pcm, Math.max(pcm.length * 2, length + out.length)); + System.arraycopy(out, 0, pcm, length, out.length); + length += out.length; + } + } + assertTrue(granule * channels <= length, "granule past the audio"); + return new Decoded(channels, comments, Arrays.copyOfRange(pcm, preSkip * channels, (int) granule * channels)); + } + + /** Reads every packet, checking the pages' framing; returns the last page's granule position. */ + private static long readOgg(byte[] file, List packets) { + ByteBuffer in = ByteBuffer.wrap(file).order(ByteOrder.LITTLE_ENDIAN); + ByteArrayOutputStream packet = new ByteArrayOutputStream(); + long granule = -1; + int expectedSequence = 0; + boolean sawLast = false; + while (in.hasRemaining()) { + int start = in.position(); + assertEquals(0x5367674f, in.getInt(), "page sync"); + in.get(); + int flags = in.get() & 0xff; + granule = in.getLong(); + in.getInt(); + assertEquals(expectedSequence++, in.getInt(), "page sequence"); + in.getInt(); + assertEquals(packet.size() > 0, (flags & 1) != 0, "continuation flag"); + assertEquals(start == 0, (flags & 2) != 0, "first page flag"); + sawLast = (flags & 4) != 0; + int segments = in.get() & 0xff; + byte[] lacing = new byte[segments]; + in.get(lacing); + for (byte l : lacing) { + int size = l & 0xff; + byte[] data = new byte[size]; + in.get(data); + packet.writeBytes(data); + if (size < 255) { + packets.add(packet.toByteArray()); + packet.reset(); + } + } + } + assertTrue(sawLast, "last page flag"); + return granule; + } +} diff --git a/ts3-client/swing/src/main/java/com/ts3client/ui/HotkeyActions.java b/ts3-client/swing/src/main/java/com/ts3client/ui/HotkeyActions.java index d2adcdb..e3b0f94 100644 --- a/ts3-client/swing/src/main/java/com/ts3client/ui/HotkeyActions.java +++ b/ts3-client/swing/src/main/java/com/ts3client/ui/HotkeyActions.java @@ -93,6 +93,12 @@ final class HotkeyActions implements HotkeyEngine.Handler { case NICKNAME_CHANGE -> frame.changeNickname(hotkey.argument); + case RECORDING_START -> startRecording(targets, false); + case RECORDING_START_MULTITRACK -> startRecording(targets, true); + case RECORDING_STOP -> { + for (ServerTab tab : targets) tab.session().stopRecording(); + } + case FILEBROWSER -> frame.browseCurrentChannel(); case SKIN_RELOAD -> frame.reloadSkin(); case BRING_TO_FRONT -> frame.bringToFront(); @@ -104,6 +110,10 @@ final class HotkeyActions implements HotkeyEngine.Handler { } } + private void startRecording(List targets, boolean multitrack) { + for (ServerTab tab : targets) frame.recordings().startNow(tab.session(), multitrack); + } + private void setMicMuted(List targets, boolean muted) { for (ServerTab tab : targets) tab.session().setMicMuted(muted); frame.refreshAfterHotkey(); diff --git a/ts3-client/swing/src/main/java/com/ts3client/ui/MainFrame.java b/ts3-client/swing/src/main/java/com/ts3client/ui/MainFrame.java index 7ea3804..9b2bf94 100644 --- a/ts3-client/swing/src/main/java/com/ts3client/ui/MainFrame.java +++ b/ts3-client/swing/src/main/java/com/ts3client/ui/MainFrame.java @@ -59,6 +59,7 @@ import java.util.concurrent.atomic.AtomicBoolean; public final class MainFrame extends JFrame implements ServerTabPane.Listener { private final Settings settings; + private final RecordingActions recordings; private final Bookmarks bookmarks = Bookmarks.load(); private final AwayMessages awayMessages = AwayMessages.load(); private final IdentityStore identities; @@ -105,6 +106,7 @@ public final class MainFrame extends JFrame implements ServerTabPane.Listener { public MainFrame(Settings settings) { super("TS3J — TeamSpeak 3 Java Client"); this.settings = settings; + this.recordings = new RecordingActions(this, settings); this.chatZoom = new TextZoom(settings.chatZoom, z -> { settings.chatZoom = z; settings.save(); @@ -164,7 +166,9 @@ public final class MainFrame extends JFrame implements ServerTabPane.Listener { if (selected != null) selected.chat().openFind(); } }); - statusBar = new StatusBar(); + statusBar = new StatusBar(() -> { + if (selected != null) selected.session().stopRecording(); + }); statusBar.setVisible(settings.showStatusBar); add(statusBar, BorderLayout.SOUTH); @@ -298,6 +302,21 @@ public final class MainFrame extends JFrame implements ServerTabPane.Listener { showContacts(null); } + @Override + public boolean isRecording() { + return selected != null && selected.session().isRecording(); + } + + @Override + public void onStartRecording(boolean multitrack) { + if (selected != null) recordings.choose(selected.session(), multitrack); + } + + @Override + public void onStopRecording() { + if (selected != null) selected.session().stopRecording(); + } + @Override public boolean canViewBanList() { return selected != null && selected.session().canViewBanList(); @@ -588,6 +607,10 @@ public final class MainFrame extends JFrame implements ServerTabPane.Listener { return out; } + RecordingActions recordings() { + return recordings; + } + ServerTab selectedTab() { return selected; } @@ -1004,6 +1027,7 @@ public final class MainFrame extends JFrame implements ServerTabPane.Listener { toolbar.refresh(connected, anyConnected, micMuted, deaf, active, away); toolbar.showHostButton(hostButtonOf(selected)); menuBar.refresh(connected, anyConnected, micMuted, deaf, active, away, commander); + statusBar.showRecording(connected && selected.session().isRecording()); updateTray(); } diff --git a/ts3-client/swing/src/main/java/com/ts3client/ui/MainMenuBar.java b/ts3-client/swing/src/main/java/com/ts3client/ui/MainMenuBar.java index 3d3ae4e..5ac152a 100644 --- a/ts3-client/swing/src/main/java/com/ts3client/ui/MainMenuBar.java +++ b/ts3-client/swing/src/main/java/com/ts3client/ui/MainMenuBar.java @@ -76,6 +76,13 @@ final class MainMenuBar extends JMenuBar { void onShowContacts(); + /** Whether the current tab is recording; asked as the Tools menu opens. */ + boolean isRecording(); + + void onStartRecording(boolean multitrack); + + void onStopRecording(); + /** Whether the current server lets us read its ban list; asked as the Tools menu opens. */ boolean canViewBanList(); @@ -125,6 +132,9 @@ final class MainMenuBar extends JMenuBar { private JMenuItem phoneticItem; private JMenuItem avatarItem; private JMenuItem deleteAvatarItem; + private JMenuItem startRecordingItem; + private JMenuItem startMultitrackItem; + private JMenuItem stopRecordingItem; private JMenuItem banListItem; private JMenuItem complaintsItem; private JMenuItem serverLogItem; @@ -271,6 +281,12 @@ final class MainMenuBar extends JMenuBar { JMenuItem contactsItem = new JMenuItem("Contacts…", Icons.ofAny("CONTACT", "EDIT_FRIEND_FOE_STATUS")); contactsItem.setAccelerator(KeyStroke.getKeyStroke("control shift O")); contactsItem.addActionListener(e -> listener.onShowContacts()); + startRecordingItem = new JMenuItem("Start Recording", Icons.of("RECORDING_START")); + startRecordingItem.addActionListener(e -> listener.onStartRecording(false)); + startMultitrackItem = new JMenuItem("Start Multitrack Recording", Icons.of("RECORDING_START")); + startMultitrackItem.addActionListener(e -> listener.onStartRecording(true)); + stopRecordingItem = new JMenuItem("Stop Recording", Icons.of("RECORDING_STOP")); + stopRecordingItem.addActionListener(e -> listener.onStopRecording()); banListItem = new JMenuItem("Ban List", Icons.of("BAN_LIST")); banListItem.setAccelerator(KeyStroke.getKeyStroke("control shift B")); // The shortcut works while the item is hidden, so it asks for itself. @@ -294,6 +310,10 @@ final class MainMenuBar extends JMenuBar { tools.add(identitiesItem); tools.add(contactsItem); tools.addSeparator(); + tools.add(startRecordingItem); + tools.add(startMultitrackItem); + tools.add(stopRecordingItem); + tools.addSeparator(); tools.add(banListItem); tools.add(complaintsItem); tools.add(serverLogItem); @@ -311,6 +331,10 @@ final class MainMenuBar extends JMenuBar { complaintsItem.setVisible(complaints); serverLogItem.setVisible(serverLog); adminSeparator.setVisible(bans || complaints || serverLog); + boolean recording = listener.isRecording(); + startRecordingItem.setVisible(!recording); + startMultitrackItem.setVisible(!recording); + stopRecordingItem.setVisible(recording); } @Override @@ -350,6 +374,8 @@ final class MainMenuBar extends JMenuBar { phoneticItem.setEnabled(connected); avatarItem.setEnabled(connected); deleteAvatarItem.setEnabled(connected); + startRecordingItem.setEnabled(connected); + startMultitrackItem.setEnabled(connected); banListItem.setEnabled(connected); complaintsItem.setEnabled(connected); serverLogItem.setEnabled(connected); diff --git a/ts3-client/swing/src/main/java/com/ts3client/ui/RecordingActions.java b/ts3-client/swing/src/main/java/com/ts3client/ui/RecordingActions.java new file mode 100644 index 0000000..638b8e0 --- /dev/null +++ b/ts3-client/swing/src/main/java/com/ts3client/ui/RecordingActions.java @@ -0,0 +1,51 @@ +package com.ts3client.ui; + +import com.ts3client.config.Settings; +import com.ts3client.recording.VoiceRecorder; +import com.ts3client.session.ServerSession; + +import javax.swing.JFileChooser; +import javax.swing.filechooser.FileNameExtensionFilter; +import java.awt.Component; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** Starts recordings, into a file the user picks or, from a hotkey, the folder last used. */ +final class RecordingActions { + + private final Component owner; + private final Settings settings; + + RecordingActions(Component owner, Settings settings) { + this.owner = owner; + this.settings = settings; + } + + /** Asks where to save, then starts recording. */ + void choose(ServerSession session, boolean multitrack) { + if (!session.isConnected() || session.isRecording()) return; + Path dir = settings.recordingFolder(); + if (!Files.isDirectory(dir)) dir = Paths.get(System.getProperty("user.home", ".")); + JFileChooser chooser = new JFileChooser(dir.toFile()); + chooser.setDialogTitle(multitrack ? "Start Multitrack Recording" : "Start Recording"); + chooser.setFileFilter(new FileNameExtensionFilter("Ogg Opus (*.ogg)", "ogg")); + chooser.setSelectedFile(session.recordingFile(dir).toFile()); + if (chooser.showSaveDialog(owner) != JFileChooser.APPROVE_OPTION) return; + File file = chooser.getSelectedFile(); + if (!file.getName().toLowerCase().endsWith(".ogg")) file = new File(file.getParentFile(), file.getName() + ".ogg"); + settings.recordingDir = file.getParentFile().getAbsolutePath(); + settings.save(); + session.startRecording(mode(multitrack), file.toPath()); + } + + void startNow(ServerSession session, boolean multitrack) { + if (!session.isConnected() || session.isRecording()) return; + session.startRecording(mode(multitrack), session.recordingFile(settings.recordingFolder())); + } + + private static VoiceRecorder.Mode mode(boolean multitrack) { + return multitrack ? VoiceRecorder.Mode.MULTITRACK : VoiceRecorder.Mode.MIXED; + } +} diff --git a/ts3-client/swing/src/main/java/com/ts3client/ui/StatusBar.java b/ts3-client/swing/src/main/java/com/ts3client/ui/StatusBar.java index 5e303c2..4b56aec 100644 --- a/ts3-client/swing/src/main/java/com/ts3client/ui/StatusBar.java +++ b/ts3-client/swing/src/main/java/com/ts3client/ui/StatusBar.java @@ -1,26 +1,54 @@ package com.ts3client.ui; import com.formdev.flatlaf.util.ScaledEmptyBorder; +import com.formdev.flatlaf.util.UIScale; import javax.swing.JLabel; import javax.swing.JPanel; import java.awt.BorderLayout; +import java.awt.Cursor; +import java.awt.FlowLayout; +import java.awt.event.MouseAdapter; +import java.awt.event.MouseEvent; -/** The strip at the bottom of the window: connection status on the left, codec info on the right. */ +/** + * The strip at the bottom of the window: connection status on the left, codec info on the + * right, and while the server on screen is being recorded, a label that stops it. + */ final class StatusBar extends JPanel { private final JLabel statusLabel = new JLabel("Not connected"); private final JLabel codecLabel = new JLabel(); + private final JLabel recordingLabel = new JLabel("Recording", Icons.of("RECORDING_STOP"), JLabel.LEADING); - StatusBar() { + StatusBar(Runnable onStopRecording) { super(new BorderLayout()); setBackground(Theme.statusBg()); setBorder(new ScaledEmptyBorder(3, 8, 3, 8)); statusLabel.setFont(Theme.uiFont()); codecLabel.setFont(Theme.uiFont()); codecLabel.setForeground(Theme.chatSystem()); - add(statusLabel, BorderLayout.WEST); - add(codecLabel, BorderLayout.EAST); + recordingLabel.setFont(Theme.uiFont()); + recordingLabel.setToolTipText("Stop Recording"); + recordingLabel.setCursor(Cursor.getPredefinedCursor(Cursor.HAND_CURSOR)); + recordingLabel.setVisible(false); + recordingLabel.addMouseListener(new MouseAdapter() { + @Override + public void mouseClicked(MouseEvent e) { + onStopRecording.run(); + } + }); + JPanel east = new JPanel(new FlowLayout(FlowLayout.TRAILING, UIScale.scale(12), 0)); + east.setOpaque(false); + east.add(recordingLabel); + east.add(codecLabel); + // In the centre, a long status is cut short rather than drawn over the labels beside it. + add(statusLabel, BorderLayout.CENTER); + add(east, BorderLayout.EAST); + } + + void showRecording(boolean recording) { + recordingLabel.setVisible(recording); } void setStatus(String text) {