From 9ee0902087345df112ff88f94848d052f9eec0c9 Mon Sep 17 00:00:00 2001 From: pedro Date: Wed, 10 May 2017 17:37:13 +0200 Subject: [PATCH] fix rtsp audio bug and more --- .idea/modules.xml | 1 - .../customexample/RtspActivity.java | 2 +- .../java/com/pedro/builder/RtspBuilder.java | 10 +- .../com/pedro/encoder/audio/AudioEncoder.java | 12 +- .../input/audio/MicrophoneManager.java | 286 ++++++++++-------- .../input/video/Camera1ApiManager.java | 25 +- .../simplertmp/DefaultRtmpPublisher.java | 9 +- .../faucamp/simplertmp/RtmpPublisher.java | 7 - .../faucamp/simplertmp/io/RtmpConnection.java | 51 ++-- .../main/java/net/ossrs/rtmp/SrsFlvMuxer.java | 59 +--- .../com/pedro/rtsp/rtp/packets/AccPacket.java | 45 +-- .../pedro/rtsp/rtp/packets/H264Packet.java | 30 +- .../pedro/rtsp/rtp/sockets/BaseRtpSocket.java | 11 +- .../main/java/com/pedro/rtsp/rtsp/Body.java | 5 +- .../java/com/pedro/rtsp/rtsp/RtspClient.java | 24 +- 15 files changed, 303 insertions(+), 274 deletions(-) diff --git a/.idea/modules.xml b/.idea/modules.xml index a4c372b87..163ab22f4 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -4,7 +4,6 @@ - diff --git a/app/src/main/java/com/pedro/rtmpstreamer/customexample/RtspActivity.java b/app/src/main/java/com/pedro/rtmpstreamer/customexample/RtspActivity.java index d4f4d4f08..295773c15 100644 --- a/app/src/main/java/com/pedro/rtmpstreamer/customexample/RtspActivity.java +++ b/app/src/main/java/com/pedro/rtmpstreamer/customexample/RtspActivity.java @@ -113,7 +113,7 @@ public void onDrawerClosed(View view) { etVideoBitrate.setText("2500"); etFps.setText("30"); etAudioBitrate.setText("128"); - etSampleRate.setText("44100"); + etSampleRate.setText("16000"); etWowzaUser = (EditText) navigationView.getMenu().findItem(R.id.et_wowza_user).getActionView(); etWowzaPassword = (EditText) navigationView.getMenu().findItem(R.id.et_wowza_password).getActionView(); diff --git a/builder/src/main/java/com/pedro/builder/RtspBuilder.java b/builder/src/main/java/com/pedro/builder/RtspBuilder.java index 400f04a0f..fb212a83a 100644 --- a/builder/src/main/java/com/pedro/builder/RtspBuilder.java +++ b/builder/src/main/java/com/pedro/builder/RtspBuilder.java @@ -4,7 +4,6 @@ import android.hardware.Camera; import android.media.MediaCodec; import android.os.Build; -import android.util.Base64; import android.view.SurfaceView; import com.pedro.encoder.audio.AudioEncoder; import com.pedro.encoder.audio.GetAccData; @@ -64,6 +63,7 @@ public boolean prepareVideo(int width, int height, int fps, int bitrate, int rot public boolean prepareAudio(int bitrate, int sampleRate, boolean isStereo) { rtspClient.setSampleRate(sampleRate); + rtspClient.setIsStereo(isStereo); microphoneManager.createMicrophone(sampleRate, isStereo); return audioEncoder.prepareAudioEncoder(bitrate, sampleRate, isStereo); } @@ -73,7 +73,10 @@ public boolean prepareVideo() { return videoEncoder.prepareVideoEncoder(); } + //set 16000hz sample rate because 44100 produce desynchronization audio in rtsp public boolean prepareAudio() { + microphoneManager.setSampleRate(16000); + audioEncoder.setSampleRate(16000); microphoneManager.createMicrophone(); rtspClient.setSampleRate(microphoneManager.getSampleRate()); return audioEncoder.prepareAudioEncoder(); @@ -168,10 +171,7 @@ public void onSPSandPPS(ByteBuffer sps, ByteBuffer pps) { byte[] mPPS = new byte[pps.capacity() - 4]; pps.position(4); pps.get(mPPS, 0, mPPS.length); - - String sSPS = Base64.encodeToString(mSPS, 0, mSPS.length, Base64.NO_WRAP); - String sPPS = Base64.encodeToString(mPPS, 0, mPPS.length, Base64.NO_WRAP); - rtspClient.setSPSandPPS(sSPS, sPPS); + rtspClient.setSPSandPPS(mPPS, mSPS); rtspClient.connect(); } diff --git a/encoder/src/main/java/com/pedro/encoder/audio/AudioEncoder.java b/encoder/src/main/java/com/pedro/encoder/audio/AudioEncoder.java index 914d6b82e..2a803b778 100644 --- a/encoder/src/main/java/com/pedro/encoder/audio/AudioEncoder.java +++ b/encoder/src/main/java/com/pedro/encoder/audio/AudioEncoder.java @@ -1,13 +1,13 @@ package com.pedro.encoder.audio; import android.media.MediaCodec; +import android.media.MediaCodecInfo; import android.media.MediaFormat; import android.os.Build; import android.support.annotation.RequiresApi; import android.util.Log; - import com.pedro.encoder.input.audio.GetMicrophoneData; - +import com.pedro.encoder.input.audio.MicrophoneManager; import java.io.IOException; import java.nio.ByteBuffer; @@ -45,7 +45,9 @@ public boolean prepareAudioEncoder(int bitRate, int sampleRate, boolean isStereo int a = (isStereo) ? 2 : 1; MediaFormat audioFormat = MediaFormat.createAudioFormat(mime, sampleRate, a); audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitRate); - audioFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 0); + audioFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, MicrophoneManager.BUFFER_SIZE); + audioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, + MediaCodecInfo.CodecProfileLevel.AACObjectLC); audioEncoder.configure(audioFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); running = false; return true; @@ -148,6 +150,10 @@ private void getDataFromEncoder(byte[] data, int size) { } } + public void setSampleRate(int sampleRate) { + this.sampleRate = sampleRate; + } + public boolean isRunning() { return running; } diff --git a/encoder/src/main/java/com/pedro/encoder/input/audio/MicrophoneManager.java b/encoder/src/main/java/com/pedro/encoder/input/audio/MicrophoneManager.java index ec80da791..bb2fb5ac1 100644 --- a/encoder/src/main/java/com/pedro/encoder/input/audio/MicrophoneManager.java +++ b/encoder/src/main/java/com/pedro/encoder/input/audio/MicrophoneManager.java @@ -3,8 +3,9 @@ import android.media.AudioFormat; import android.media.AudioRecord; import android.media.MediaRecorder; +import android.media.audiofx.AcousticEchoCanceler; +import android.media.audiofx.AutomaticGainControl; import android.util.Log; - import com.pedro.encoder.audio.DataTaken; /** @@ -13,139 +14,174 @@ public class MicrophoneManager { - private final String TAG = "MicrophoneManager"; - private AudioRecord audioRecord; - private GetMicrophoneData getMicrophoneData; - private byte[] pcmBuffer = new byte[4096]; - private byte[] pcmBufferMuted = new byte[11]; - private boolean running = false; - - //default parameters for microphone - private int sampleRate = 44100; //hz - private int audioFormat = AudioFormat.ENCODING_PCM_16BIT; - private int channel = AudioFormat.CHANNEL_IN_STEREO; - private boolean muted = false; - - public MicrophoneManager(GetMicrophoneData getMicrophoneData) { - this.getMicrophoneData = getMicrophoneData; + private final String TAG = "MicrophoneManager"; + public static final int BUFFER_SIZE = 4096; + private AudioRecord audioRecord; + private GetMicrophoneData getMicrophoneData; + private byte[] pcmBuffer = new byte[BUFFER_SIZE]; + private byte[] pcmBufferMuted = new byte[11]; + private boolean running = false; + + //default parameters for microphone + private int sampleRate = 44100; //hz + private int audioFormat = AudioFormat.ENCODING_PCM_16BIT; + private int channel = AudioFormat.CHANNEL_IN_STEREO; + private boolean muted = false; + //echo canceler + private AcousticEchoCanceler acousticEchoCanceler; + private AutomaticGainControl automaticGainControl; + + public MicrophoneManager(GetMicrophoneData getMicrophoneData) { + this.getMicrophoneData = getMicrophoneData; + } + + /** + * Create audio record + */ + public void createMicrophone() { + createMicrophone(sampleRate, true); + Log.i(TAG, "Microphone created, 44100hz, Stereo"); + } + + /** + * Create audio record with params + */ + public void createMicrophone(int sampleRate, boolean isStereo) { + this.sampleRate = sampleRate; + if (!isStereo) channel = AudioFormat.CHANNEL_IN_MONO; + audioRecord = + new AudioRecord(MediaRecorder.AudioSource.DEFAULT, sampleRate, channel, audioFormat, + getPcmBufferSize() * 4); + enableAudioEchoCanceler(audioRecord.getAudioSessionId()); + String chl = (isStereo) ? "Stereo" : "Mono"; + Log.i(TAG, "Microphone created, " + sampleRate + "hz, " + chl); + } + + private void enableAudioEchoCanceler(int id) { + if (AcousticEchoCanceler.isAvailable() && acousticEchoCanceler == null) { + acousticEchoCanceler = AcousticEchoCanceler.create(id); + acousticEchoCanceler.setEnabled(true); } - - /** - * Create audio record - */ - public void createMicrophone() { - createMicrophone(sampleRate, true); - Log.i(TAG, "Microphone created, 44100hz, Stereo"); + if (AutomaticGainControl.isAvailable() && automaticGainControl == null) { + automaticGainControl = AutomaticGainControl.create(id); + automaticGainControl.setEnabled(true); } + } - /** - * Create audio record with params - */ - public void createMicrophone(int sampleRate, boolean isStereo) { - this.sampleRate = sampleRate; - if (!isStereo) channel = AudioFormat.CHANNEL_IN_MONO; - audioRecord = new AudioRecord(MediaRecorder.AudioSource.DEFAULT, sampleRate, channel, - audioFormat, getPcmBufferSize() * 4); - String chl = (isStereo) ? "Stereo" : "Mono"; - Log.i(TAG, "Microphone created, " + sampleRate + "hz, " + chl); + private void disableAudioEchoCanceler(){ + if(acousticEchoCanceler != null){ + acousticEchoCanceler.setEnabled(false); + acousticEchoCanceler.release(); + acousticEchoCanceler = null; } - - /** - * Start record and get data - */ - public void start() { - init(); - new Thread(new Runnable() { - @Override - public void run() { - android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO); - while (running && !Thread.interrupted()) { - DataTaken dataTaken = read(); - if (dataTaken != null) { - getMicrophoneData.inputPcmData(dataTaken.getPcmBuffer(), dataTaken.getSize()); - } else { - running = false; - } - } - } - }).start(); + if(automaticGainControl != null){ + automaticGainControl.setEnabled(false); + automaticGainControl.release(); + automaticGainControl = null; } - - private void init() { - if (audioRecord != null) { - audioRecord.startRecording(); - running = true; - Log.i(TAG, "Microphone started"); - } else { - Log.e(TAG, "Error starting, microphone was stopped or not created, " + - "use createMicrophone() before start()"); + } + /** + * Start record and get data + */ + public void start() { + init(); + new Thread(new Runnable() { + @Override + public void run() { + android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO); + while (running && !Thread.interrupted()) { + DataTaken dataTaken = read(); + if (dataTaken != null) { + getMicrophoneData.inputPcmData(dataTaken.getPcmBuffer(), dataTaken.getSize()); + } else { + running = false; + } } + } + }).start(); + } + + private void init() { + if (audioRecord != null) { + audioRecord.startRecording(); + running = true; + Log.i(TAG, "Microphone started"); + } else { + Log.e(TAG, "Error starting, microphone was stopped or not created, " + + "use createMicrophone() before start()"); } - - public void mute(){ - muted = true; - } - - public void unMute(){ - muted = false; - } - - public boolean isMuted(){ - return muted; + } + + public void mute() { + muted = true; + } + + public void unMute() { + muted = false; + } + + public boolean isMuted() { + return muted; + } + + /** + * @return Object with size and PCM buffer data + */ + private DataTaken read() { + int size; + if (muted) { + size = audioRecord.read(pcmBufferMuted, 0, pcmBufferMuted.length); + } else { + size = audioRecord.read(pcmBuffer, 0, pcmBuffer.length); } - /** - * @return Object with size and PCM buffer data - */ - private DataTaken read() { - int size; - if(muted){ - size = audioRecord.read(pcmBufferMuted, 0, pcmBufferMuted.length); - } else { - size = audioRecord.read(pcmBuffer, 0, pcmBuffer.length); - } - if (size <= 0) { - return null; - } - return new DataTaken(pcmBuffer, size); + if (size <= 0) { + return null; } - - /** - * Stop and release microphone - */ - public void stop() { - running = false; - if (audioRecord != null) { - audioRecord.setRecordPositionUpdateListener(null); - audioRecord.stop(); - audioRecord.release(); - audioRecord = null; - } - Log.i(TAG, "Microphone stopped"); - } - - /** - * Get PCM buffer size - */ - private int getPcmBufferSize() { - int pcmBufSize = AudioRecord.getMinBufferSize(sampleRate, channel, - AudioFormat.ENCODING_PCM_16BIT) + 8191; - return pcmBufSize - (pcmBufSize % 8192); - } - - public int getSampleRate() { - return sampleRate; - } - - public int getAudioFormat() { - return audioFormat; - } - - public int getChannel() { - return channel; - } - - public boolean isRunning() { - return running; + return new DataTaken(pcmBuffer, size); + } + + /** + * Stop and release microphone + */ + public void stop() { + running = false; + if (audioRecord != null) { + audioRecord.setRecordPositionUpdateListener(null); + audioRecord.stop(); + audioRecord.release(); + audioRecord = null; } + disableAudioEchoCanceler(); + Log.i(TAG, "Microphone stopped"); + } + + /** + * Get PCM buffer size + */ + private int getPcmBufferSize() { + int pcmBufSize = + AudioRecord.getMinBufferSize(sampleRate, channel, AudioFormat.ENCODING_PCM_16BIT) + 8191; + return pcmBufSize - (pcmBufSize % 8192); + } + + public int getSampleRate() { + return sampleRate; + } + + public void setSampleRate(int sampleRate) { + this.sampleRate = sampleRate; + } + + public int getAudioFormat() { + return audioFormat; + } + + public int getChannel() { + return channel; + } + + public boolean isRunning() { + return running; + } } diff --git a/encoder/src/main/java/com/pedro/encoder/input/video/Camera1ApiManager.java b/encoder/src/main/java/com/pedro/encoder/input/video/Camera1ApiManager.java index a1acf9437..e7ca01bd8 100644 --- a/encoder/src/main/java/com/pedro/encoder/input/video/Camera1ApiManager.java +++ b/encoder/src/main/java/com/pedro/encoder/input/video/Camera1ApiManager.java @@ -4,13 +4,10 @@ import android.hardware.Camera; import android.opengl.GLES20; import android.util.Log; -import android.view.Surface; import android.view.SurfaceHolder; import android.view.SurfaceView; - import java.io.IOException; import java.util.List; - import javax.microedition.khronos.egl.EGL10; import javax.microedition.khronos.egl.EGLConfig; import javax.microedition.khronos.egl.EGLContext; @@ -50,7 +47,7 @@ public class Camera1ApiManager implements Camera.PreviewCallback { public Camera1ApiManager(SurfaceView surfaceView, GetCameraData getCameraData) { this.surfaceView = surfaceView; this.getCameraData = getCameraData; - if(surfaceView.getContext().getResources().getConfiguration().orientation == 1){ + if (surfaceView.getContext().getResources().getConfiguration().orientation == 1) { orientation = 90; } cameraSelect = selectCamera(); @@ -71,7 +68,7 @@ public void start() { if (camera == null) { try { camera = Camera.open(cameraSelect); - if(!checkCanOpen()){ + if (!checkCanOpen()) { throw new CameraOpenException("This camera resolution cant be opened"); } Camera.Parameters parameters = camera.getParameters(); @@ -191,13 +188,13 @@ public void onPreviewFrame(byte[] data, Camera camera) { * Example: 842094169 -> YV12, 17 -> NV21 */ public List getCameraPreviewImageFormatSupported() { - if(camera != null) { + if (camera != null) { List formats = camera.getParameters().getSupportedPreviewFormats(); for (Integer i : formats) { Log.i(TAG, "camera format supported: " + i); } return formats; - } else{ + } else { return null; } } @@ -206,13 +203,13 @@ public List getCameraPreviewImageFormatSupported() { * call if after start() */ public List getPreviewSize() { - if(camera != null) { + if (camera != null) { List previewSizes = camera.getParameters().getSupportedPreviewSizes(); for (Camera.Size size : previewSizes) { Log.i(TAG, size.width + "X" + size.height); } return previewSizes; - } else{ + } else { camera = Camera.open(cameraSelect); List previewSizes = camera.getParameters().getSupportedPreviewSizes(); camera.release(); @@ -225,7 +222,7 @@ public List getPreviewSize() { } public void setEffect(EffectManager effect) { - if(camera != null) { + if (camera != null) { Camera.Parameters parameters = camera.getParameters(); parameters.setColorEffect(effect.getEffect()); try { @@ -237,7 +234,7 @@ public void setEffect(EffectManager effect) { } } - public void switchCamera() throws CameraOpenException{ + public void switchCamera() throws CameraOpenException { if (camera != null) { int number = Camera.getNumberOfCameras(); for (int i = 0; i < number; i++) { @@ -251,9 +248,9 @@ public void switchCamera() throws CameraOpenException{ } } - private boolean checkCanOpen(){ - for(Camera.Size size : getPreviewSize()){ - if(size.width == width && size.height == height){ + private boolean checkCanOpen() { + for (Camera.Size size : getPreviewSize()) { + if (size.width == width && size.height == height) { return true; } } diff --git a/rtmp/src/main/java/com/github/faucamp/simplertmp/DefaultRtmpPublisher.java b/rtmp/src/main/java/com/github/faucamp/simplertmp/DefaultRtmpPublisher.java index 6608f1101..3daffc792 100644 --- a/rtmp/src/main/java/com/github/faucamp/simplertmp/DefaultRtmpPublisher.java +++ b/rtmp/src/main/java/com/github/faucamp/simplertmp/DefaultRtmpPublisher.java @@ -1,9 +1,7 @@ package com.github.faucamp.simplertmp; -import java.io.InputStream; -import java.util.concurrent.atomic.AtomicInteger; - import com.github.faucamp.simplertmp.io.RtmpConnection; +import java.io.InputStream; import net.ossrs.rtmp.ConnectCheckerRtmp; /** @@ -48,11 +46,6 @@ public void publishAudioData(byte[] data, int size, int dts) { rtmpConnection.publishAudioData(data, size, dts); } - @Override - public AtomicInteger getVideoFrameCacheNumber() { - return rtmpConnection.getVideoFrameCacheNumber(); - } - @Override public void setVideoResolution(int width, int height) { rtmpConnection.setVideoResolution(width, height); diff --git a/rtmp/src/main/java/com/github/faucamp/simplertmp/RtmpPublisher.java b/rtmp/src/main/java/com/github/faucamp/simplertmp/RtmpPublisher.java index 5354143e2..bad9ee04f 100644 --- a/rtmp/src/main/java/com/github/faucamp/simplertmp/RtmpPublisher.java +++ b/rtmp/src/main/java/com/github/faucamp/simplertmp/RtmpPublisher.java @@ -1,7 +1,5 @@ package com.github.faucamp.simplertmp; -import java.util.concurrent.atomic.AtomicInteger; - /** * Simple RTMP publisher, using vanilla Java networking (no NIO) * This was created primarily to address a NIO bug in Android 2.2 when @@ -52,11 +50,6 @@ public interface RtmpPublisher { */ void publishAudioData(byte[] data, int size, int dts); - /** - * obtain video frame number cached in publisher - */ - AtomicInteger getVideoFrameCacheNumber(); - /** * set video resolution * diff --git a/rtmp/src/main/java/com/github/faucamp/simplertmp/io/RtmpConnection.java b/rtmp/src/main/java/com/github/faucamp/simplertmp/io/RtmpConnection.java index 4b4dbf44a..8eb9f9f91 100644 --- a/rtmp/src/main/java/com/github/faucamp/simplertmp/io/RtmpConnection.java +++ b/rtmp/src/main/java/com/github/faucamp/simplertmp/io/RtmpConnection.java @@ -1,42 +1,36 @@ package com.github.faucamp.simplertmp.io; +import android.util.Log; +import com.github.faucamp.simplertmp.RtmpPublisher; import com.github.faucamp.simplertmp.Util; +import com.github.faucamp.simplertmp.amf.AmfMap; +import com.github.faucamp.simplertmp.amf.AmfNull; +import com.github.faucamp.simplertmp.amf.AmfNumber; +import com.github.faucamp.simplertmp.amf.AmfObject; +import com.github.faucamp.simplertmp.amf.AmfString; +import com.github.faucamp.simplertmp.packets.Abort; +import com.github.faucamp.simplertmp.packets.Audio; +import com.github.faucamp.simplertmp.packets.Command; +import com.github.faucamp.simplertmp.packets.Data; +import com.github.faucamp.simplertmp.packets.Handshake; +import com.github.faucamp.simplertmp.packets.RtmpPacket; +import com.github.faucamp.simplertmp.packets.SetPeerBandwidth; +import com.github.faucamp.simplertmp.packets.UserControl; +import com.github.faucamp.simplertmp.packets.Video; +import com.github.faucamp.simplertmp.packets.WindowAckSize; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.InetSocketAddress; import java.net.Socket; -import java.net.SocketAddress; import java.net.SocketException; import java.util.Random; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.concurrent.atomic.AtomicInteger; - -import android.util.Log; - -import com.github.faucamp.simplertmp.RtmpPublisher; -import com.github.faucamp.simplertmp.amf.AmfMap; -import com.github.faucamp.simplertmp.amf.AmfNull; -import com.github.faucamp.simplertmp.amf.AmfNumber; -import com.github.faucamp.simplertmp.amf.AmfObject; -import com.github.faucamp.simplertmp.amf.AmfString; -import com.github.faucamp.simplertmp.packets.Abort; -import com.github.faucamp.simplertmp.packets.Data; -import com.github.faucamp.simplertmp.packets.Handshake; -import com.github.faucamp.simplertmp.packets.Command; -import com.github.faucamp.simplertmp.packets.Audio; -import com.github.faucamp.simplertmp.packets.SetPeerBandwidth; -import com.github.faucamp.simplertmp.packets.Video; -import com.github.faucamp.simplertmp.packets.UserControl; -import com.github.faucamp.simplertmp.packets.RtmpPacket; -import com.github.faucamp.simplertmp.packets.WindowAckSize; -import javax.net.ssl.SSLSocket; import net.ossrs.rtmp.ConnectCheckerRtmp; import net.ossrs.rtmp.CreateSSLSocket; @@ -70,7 +64,6 @@ public class RtmpConnection implements RtmpPublisher { private volatile boolean publishPermitted = false; private final Object connectingLock = new Object(); private final Object publishLock = new Object(); - private AtomicInteger videoFrameCacheNumber = new AtomicInteger(0); private int currentStreamId = 0; private int transactionIdCounter = 0; private int videoWidth; @@ -447,7 +440,6 @@ private void reset() { publishType = null; currentStreamId = 0; transactionIdCounter = 0; - videoFrameCacheNumber.set(0); socketExceptionCause = ""; socket = null; rtmpSessionInfo = null; @@ -498,7 +490,6 @@ public void publishVideoData(byte[] data, int size, int dts) { video.getHeader().setAbsoluteTimestamp(dts); video.getHeader().setMessageStreamId(currentStreamId); sendRtmpPacket(video); - videoFrameCacheNumber.decrementAndGet(); } private void sendRtmpPacket(RtmpPacket rtmpPacket) { @@ -730,10 +721,10 @@ public void run() { } } - @Override - public AtomicInteger getVideoFrameCacheNumber() { - return videoFrameCacheNumber; - } + //@Override + //public AtomicInteger getVideoFrameCacheNumber() { + // return videoFrameCacheNumber; + //} @Override public void setVideoResolution(int width, int height) { diff --git a/rtmp/src/main/java/net/ossrs/rtmp/SrsFlvMuxer.java b/rtmp/src/main/java/net/ossrs/rtmp/SrsFlvMuxer.java index 18731b932..25ea9b1d1 100644 --- a/rtmp/src/main/java/net/ossrs/rtmp/SrsFlvMuxer.java +++ b/rtmp/src/main/java/net/ossrs/rtmp/SrsFlvMuxer.java @@ -8,7 +8,6 @@ import java.util.ArrayList; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.atomic.AtomicInteger; /** * Created by winlin on 5/2/15. @@ -92,13 +91,6 @@ public void setJksData(InputStream inputStreamJks, String passPhraseJks) { publisher.setJksData(inputStreamJks, passPhraseJks); } - /** - * get cached video frame number in publisher - */ - public AtomicInteger getVideoFrameCacheNumber() { - return publisher == null ? null : publisher.getVideoFrameCacheNumber(); - } - /** * set video resolution for publisher * @@ -399,14 +391,6 @@ private class SrsRawH264Stream { private SrsFlvFrameBytes pps_hdr = new SrsFlvFrameBytes(); private SrsFlvFrameBytes pps_bb = new SrsFlvFrameBytes(); - public boolean isSps(SrsFlvFrameBytes frame) { - return frame.size >= 1 && (frame.data.get(0) & 0x1f) == SrsAvcNaluType.SPS; - } - - public boolean isPps(SrsFlvFrameBytes frame) { - return frame.size >= 1 && (frame.data.get(0) & 0x1f) == SrsAvcNaluType.PPS; - } - public SrsFlvFrameBytes muxNaluHeader(SrsFlvFrameBytes frame) { if (nalu_header.data == null) { nalu_header.data = ByteBuffer.allocate(4); @@ -428,7 +412,7 @@ public SrsFlvFrameBytes muxNaluHeader(SrsFlvFrameBytes frame) { return nalu_header; } - public void muxSequenceHeader(ByteBuffer sps, ByteBuffer pps, int dts, int pts, + public void muxSequenceHeader(ByteBuffer sps, ByteBuffer pps, ArrayList frames) { // 5bytes sps/pps header: // configurationVersion, AVCProfileIndication, profile_compatibility, @@ -640,7 +624,7 @@ public void writeAudioSample(final ByteBuffer bb, MediaCodec.BufferInfo bi) { // samplingFrequencyIndex; 4 bslbf byte samplingFrequencyIndex = 0x04; //44100 if (asample_rate == SrsCodecAudioSampleRate.R22050) { - samplingFrequencyIndex = 0x07; //2250 + samplingFrequencyIndex = 0x07; //22050 } else if (asample_rate == SrsCodecAudioSampleRate.R11025) { samplingFrequencyIndex = 0x0a; //11025 } @@ -706,7 +690,7 @@ private void writeAdtsHeader(byte[] frame, int offset) { // adts sync word 0xfff (12-bit) frame[offset] = (byte) 0xff; frame[offset + 1] = (byte) 0xf0; - // versioin 0 for MPEG-4, 1 for MPEG-2 (1-bit) + // version 0 for MPEG-4, 1 for MPEG-2 (1-bit) frame[offset + 1] |= 0 << 3; // layer 0 (2-bit) frame[offset + 1] |= 0 << 1; @@ -751,7 +735,7 @@ public void writeVideoSample(final ByteBuffer bb, MediaCodec.BufferInfo bi) { // 5bits, 7.3.1 NAL unit syntax, // H.264-AVC-ISO_IEC_14496-10.pdf, page 44. // 7: SPS, 8: PPS, 5: I Frame, 1: P Frame - int nal_unit_type = (int) (frame.data.get(0) & 0x1f); + int nal_unit_type = frame.data.get(0) & 0x1f; if (nal_unit_type == SrsAvcNaluType.SPS || nal_unit_type == SrsAvcNaluType.PPS) { Log.i(TAG, String.format("annexb demux %dB, pts=%d, frame=%dB, nalu=%d", bi.size, pts, frame.size, nal_unit_type)); @@ -767,28 +751,6 @@ public void writeVideoSample(final ByteBuffer bb, MediaCodec.BufferInfo bi) { continue; } - // for sps - //if (avc.isSps(frame)) { - // if (!frame.data.equals(h264_sps)) { - // byte[] sps = new byte[frame.size]; - // frame.data.get(sps); - // h264_sps_changed = true; - // h264_sps = ByteBuffer.wrap(sps); - // } - // continue; - //} - // - //// for pps - //if (avc.isPps(frame)) { - // if (!frame.data.equals(h264_pps)) { - // byte[] pps = new byte[frame.size]; - // frame.data.get(pps); - // h264_pps_changed = true; - // h264_pps = ByteBuffer.wrap(pps); - // } - // continue; - //} - // IPB frame. ipbs.add(avc.muxNaluHeader(frame)); ipbs.add(frame); @@ -807,9 +769,9 @@ public void setSpsPPs(ByteBuffer sps, ByteBuffer pps) { } private void writeH264SpsPps(int dts, int pts) { - // when sps or pps changed, update the sequence header, - // for the pps maybe not changed while sps changed. - // so, we must check when each video ts message frame parsed. + /* when sps or pps changed, update the sequence header, + for the pps maybe not changed while sps changed. + so, we must check when each video ts message frame parsed.*/ if (h264_sps_pps_sent && !h264_sps_changed && !h264_pps_changed) { return; } @@ -821,7 +783,7 @@ private void writeH264SpsPps(int dts, int pts) { // h264 raw to h264 packet. ArrayList frames = new ArrayList<>(); - avc.muxSequenceHeader(h264_sps, h264_pps, dts, pts, frames); + avc.muxSequenceHeader(h264_sps, h264_pps, frames); // h264 packet to flv packet. int frame_type = SrsCodecVideoAVCFrame.KeyFrame; @@ -880,10 +842,7 @@ private void writeRtmpPacket(int type, int dts, int frame_type, int avc_aac_type private void flvFrameCacheAdd(SrsFlvFrame frame) { try { mFlvTagCache.add(frame); - if (frame.is_video()) { - getVideoFrameCacheNumber().incrementAndGet(); - } - } catch (IllegalStateException e){ + } catch (IllegalStateException e) { Log.e(TAG, "frame discarded, cant add more frame: ", e); } } diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AccPacket.java b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AccPacket.java index b889e7347..e82491a21 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AccPacket.java +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/AccPacket.java @@ -19,6 +19,8 @@ public class AccPacket extends BasePacket { private final String TAG = "AccPacket"; + private long oldTs; + public AccPacket(RtspClient rtspClient, Protocol protocol) { super(rtspClient, protocol); } @@ -42,28 +44,35 @@ public void createAndSendPacket(ByteBuffer byteBuffer, BufferInfo bufferInfo) { int length = maxPacketSize - (RtpConstants.RTP_HEADER_LENGTH + 4) < bufferInfo.size - byteBuffer.position() ? maxPacketSize - (RtpConstants.RTP_HEADER_LENGTH + 4) : bufferInfo.size - byteBuffer.position(); - byteBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 4, length); - - ts = bufferInfo.presentationTimeUs * 1000; - socket.markNextPacket(); - socket.updateTimestamp(ts); - - // AU-headers-length field: contains the size in bits of a AU-header - // 13+3 = 16 bits -> 13bits for AU-size and 3bits for AU-Index / AU-Index-delta - // 13 bits will be enough because ADTS uses 13 bits for frame length - buffer[RtpConstants.RTP_HEADER_LENGTH] = 0; - buffer[RtpConstants.RTP_HEADER_LENGTH + 1] = 0x10; + if (length > 0) { + byteBuffer.get(buffer, RtpConstants.RTP_HEADER_LENGTH + 4, length); + oldTs = ts; + ts = bufferInfo.presentationTimeUs * 1000; + if (oldTs > ts) { + socket.commitBuffer(); + return; + } + socket.markNextPacket(); + socket.updateTimestamp(ts); - // AU-size - buffer[RtpConstants.RTP_HEADER_LENGTH + 2] = (byte) (length >> 5); - buffer[RtpConstants.RTP_HEADER_LENGTH + 3] = (byte) (length << 3); + // AU-headers-length field: contains the size in bits of a AU-header + // 13+3 = 16 bits -> 13bits for AU-size and 3bits for AU-Index / AU-Index-delta + // 13 bits will be enough because ADTS uses 13 bits for frame length + buffer[RtpConstants.RTP_HEADER_LENGTH] = 0; + buffer[RtpConstants.RTP_HEADER_LENGTH + 1] = 0x10; - // AU-Index - buffer[RtpConstants.RTP_HEADER_LENGTH + 3] &= 0xF8; - buffer[RtpConstants.RTP_HEADER_LENGTH + 3] |= 0x00; + // AU-size + buffer[RtpConstants.RTP_HEADER_LENGTH + 2] = (byte) (length >> 5); + buffer[RtpConstants.RTP_HEADER_LENGTH + 3] = (byte) (length << 3); - socket.commitBuffer(RtpConstants.RTP_HEADER_LENGTH + length + 4); + // AU-Index + buffer[RtpConstants.RTP_HEADER_LENGTH + 3] &= 0xF8; + buffer[RtpConstants.RTP_HEADER_LENGTH + 3] |= 0x00; + socket.commitBuffer(RtpConstants.RTP_HEADER_LENGTH + length + 4); + } else { + socket.commitBuffer(); + } } catch (IOException | InterruptedException | ArrayIndexOutOfBoundsException e) { e.printStackTrace(); } diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.java b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.java index b0daef389..3e7251763 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.java +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/packets/H264Packet.java @@ -1,7 +1,6 @@ package com.pedro.rtsp.rtp.packets; import android.media.MediaCodec; -import com.pedro.rtsp.rtp.packets.BasePacket; import com.pedro.rtsp.rtp.sockets.RtpSocketTcp; import com.pedro.rtsp.rtp.sockets.RtpSocketUdp; import com.pedro.rtsp.rtsp.Protocol; @@ -22,6 +21,7 @@ public class H264Packet extends BasePacket { //contain header from ByteBuffer (first 5 bytes) private byte[] header = new byte[5]; + private byte[] stapA; public H264Packet(RtspClient rtspClient, Protocol protocol) { super(rtspClient, protocol); @@ -44,6 +44,15 @@ public void createAndSendPacket(ByteBuffer byteBuffer, MediaCodec.BufferInfo buf byteBuffer.get(header, 0, 5); ts = bufferInfo.presentationTimeUs * 1000L; int naluLength = bufferInfo.size - byteBuffer.position() + 1; + int type = header[4] & 0x1F; + + if (type == 5) { + buffer = socket.requestBuffer(); + socket.markNextPacket(); + socket.updateTimestamp(ts); + System.arraycopy(stapA, 0, buffer, RtpConstants.RTP_HEADER_LENGTH, stapA.length); + socket.commitBuffer(stapA.length + RtpConstants.RTP_HEADER_LENGTH); + } // Small NAL unit => Single NAL unit if (naluLength <= maxPacketSize - RtpConstants.RTP_HEADER_LENGTH - 2) { @@ -98,4 +107,23 @@ public void createAndSendPacket(ByteBuffer byteBuffer, MediaCodec.BufferInfo buf e.printStackTrace(); } } + + public void setSPSandPPS(byte[] sps, byte[] pps) { + stapA = new byte[sps.length + pps.length + 5]; + + // STAP-A NAL header is 24 + stapA[0] = 24; + + // Write NALU 1 size into the array (NALU 1 is the SPS). + stapA[1] = (byte) (sps.length >> 8); + stapA[2] = (byte) (sps.length & 0xFF); + + // Write NALU 2 size into the array (NALU 2 is the PPS). + stapA[sps.length + 3] = (byte) (pps.length >> 8); + stapA[sps.length + 4] = (byte) (pps.length & 0xFF); + + // Write NALU 1 into the array, then write NALU 2 into the array. + System.arraycopy(sps, 0, stapA, 3, sps.length); + System.arraycopy(pps, 0, stapA, 5 + sps.length, pps.length); + } } \ No newline at end of file diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtp/sockets/BaseRtpSocket.java b/rtsp/src/main/java/com/pedro/rtsp/rtp/sockets/BaseRtpSocket.java index 5fefd9535..845989002 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtp/sockets/BaseRtpSocket.java +++ b/rtsp/src/main/java/com/pedro/rtsp/rtp/sockets/BaseRtpSocket.java @@ -37,7 +37,7 @@ public BaseRtpSocket() { /* | | ^ */ /* | -------- | */ /* | |--------------------- */ - /* | || -----------------------> Source Identifier(0) */ + /* | || -----------------------> Source Identifier(0) */ /* | || | */ mBuffers[i][0] = (byte) Integer.parseInt("10000000", 2); mBuffers[i][1] = (byte) RtpConstants.playLoadType; @@ -91,6 +91,15 @@ public void updateTimestamp(long timestamp) { setLong(mBuffers[mBufferIn], (timestamp / 100L) * (mClock / 1000L) / 10000L, 4, 8); } + public void commitBuffer() throws IOException { + if (mThread == null) { + mThread = new Thread(this); + mThread.start(); + } + if (++mBufferIn >= mBufferCount) mBufferIn = 0; + mBufferCommitted.release(); + } + public abstract void commitBuffer(int length) throws IOException; /** Sets the marker in the RTP packet. */ diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtsp/Body.java b/rtsp/src/main/java/com/pedro/rtsp/rtsp/Body.java index 262ba33f9..f4e9afc2d 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtsp/Body.java +++ b/rtsp/src/main/java/com/pedro/rtsp/rtsp/Body.java @@ -28,7 +28,7 @@ public class Body { -1, // 15 }; - public static String createAudioBody(int trackAudio, int sampleRate) { + public static String createAudioBody(int trackAudio, int sampleRate, boolean isStereo) { int sampleRateNum = -1; for(int i = 0; i < AUDIO_SAMPLING_RATES.length; i++){ if(AUDIO_SAMPLING_RATES[i] == sampleRate){ @@ -36,7 +36,8 @@ public static String createAudioBody(int trackAudio, int sampleRate) { break; } } - int config = (2 & 0x1F) << 11 | (sampleRateNum & 0x0F) << 7 | (1 & 0x0F) << 3; + int channel = (isStereo) ? 2 : 1; + int config = (2 & 0x1F) << 11 | (sampleRateNum & 0x0F) << 7 | (channel & 0x0F) << 3; return "m=audio " + (5000 + 2 * trackAudio) + " RTP/AVP" + RtpConstants.playLoadType + "\r\n" + "a=rtpmap:" + RtpConstants.playLoadType + " mpeg4-generic/" + sampleRate + "\r\n" + "a=fmtp:" + RtpConstants.playLoadType + " streamtype=5; profile-level-id=15; mode=AAC-hbr; config=" + diff --git a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.java b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.java index 5c6cd5e29..3d2568239 100644 --- a/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.java +++ b/rtsp/src/main/java/com/pedro/rtsp/rtsp/RtspClient.java @@ -1,8 +1,8 @@ package com.pedro.rtsp.rtsp; import android.media.MediaCodec; +import android.util.Base64; import android.util.Log; - import com.pedro.rtsp.rtp.packets.AccPacket; import com.pedro.rtsp.rtp.packets.H264Packet; import com.pedro.rtsp.utils.AuthUtil; @@ -32,7 +32,8 @@ public class RtspClient { private String host; private int port; private String path; - private int sampleRate; + private int sampleRate = 16000; + private boolean isStereo = true; private final int trackVideo = 1; private final int trackAudio = 0; @@ -49,7 +50,7 @@ public class RtspClient { private BufferedReader reader; private BufferedWriter writer; private Thread thread; - private String sps, pps; + private byte[] sps, pps; //for udp private int[] audioPorts = new int[] { 5000, 5001 }; private int[] videoPorts = new int[] { 5002, 5003 }; @@ -126,14 +127,19 @@ public String getPath() { return path; } - public void setSPSandPPS(String sps, String pps) { + public void setSPSandPPS(byte[] sps, byte[] pps) { this.sps = sps; this.pps = pps; } + public void setIsStereo(boolean isStereo){ + this.isStereo = isStereo; + } + public void connect() { if (!streaming) { h264Packet = new H264Packet(this, protocol); + h264Packet.setSPSandPPS(sps, pps); accPacket = new AccPacket(this, protocol); accPacket.setSampleRate(sampleRate); thread = new Thread(new Runnable() { @@ -283,6 +289,8 @@ private String sendAnnounce() { } private String createBody() { + String sSPS = Base64.encodeToString(sps, 0, sps.length, Base64.NO_WRAP); + String sPPS = Base64.encodeToString(pps, 0, pps.length, Base64.NO_WRAP); return "v=0\r\n" + // TODO: Add IPV6 support @@ -302,8 +310,8 @@ private String createBody() { // thread=0 0 means the session is permanent (we don'thread know when it will stop) "thread=0 0\r\n" + "a=recvonly\r\n" - + Body.createAudioBody(trackAudio, sampleRate) - + Body.createVideoBody(trackVideo, sps, pps); + + Body.createAudioBody(trackAudio, sampleRate, isStereo) + + Body.createVideoBody(trackVideo, sSPS, sPPS); } private String sendSetup(int track, Protocol protocol) { @@ -457,13 +465,13 @@ public int[] getVideoPorts() { } public void sendVideo(ByteBuffer h264Buffer, MediaCodec.BufferInfo info) { - if(isStreaming()) { + if (isStreaming()) { h264Packet.createAndSendPacket(h264Buffer, info); } } public void sendAudio(ByteBuffer accBuffer, MediaCodec.BufferInfo info) { - if(isStreaming()) { + if (isStreaming()) { accPacket.createAndSendPacket(accBuffer, info); } }