技术背景
我们在对接开发Android平台音视频模块的时候，遇到过这样的问题，厂商希望拉取到海康、大华等摄像机的RTSP流，然后解码后的YUV或RGB数据回给他们，他们做视频分析或处理后，再投递给轻量级RTSP服务模块或RTMP推送模块，实现处理后的数据，二次转发，本文以拉取RTSP流，解析后再注入轻量级RTSP服务为例，介绍下大概的技术实现。
技术实现
废话不多说，无图无真相，下图是测试的时候，Android终端拉取RTSP流，然后把YUV数据回调上来，又通过推送接口，注入到轻量级RTSP服务，然后Windows平台拉取轻量级RTSP的URL，整体下来，毫秒级延迟：
先说拉取RTSP流，需要注意的是，如果不要播放的话，可以SetSurface()的时候，第二个参数设置null，如果不需要audio的话，直接SetMute设置1即可，因为需要回调YUV上来，那么设置下I420回调，如果需要RGB的，只要开RGB的回调即可。
private boolean StartPlay()
{
if (!OpenPullHandle())
return false;
// 如果第二个参数设置为null，则播放纯音频
libPlayer.SmartPlayerSetSurface(playerHandle, sSurfaceView);
libPlayer.SmartPlayerSetRenderScaleMode(playerHandle, 1);
// libPlayer.SmartPlayerSetExternalRender(playerHandle, new
// RGBAExternalRender());
libPlayer.SmartPlayerSetExternalRender(playerHandle, new
I420ExternalRender());
libPlayer.SmartPlayerSetFastStartup(playerHandle, isFastStartup ? 1 : 0);
libPlayer.SmartPlayerSetAudioOutputType(playerHandle, 1);
if (isMute) {
libPlayer.SmartPlayerSetMute(playerHandle, isMute ? 1
: 0);
}
if (isHardwareDecoder)
{
int isSupportH264HwDecoder = libPlayer
.SetSmartPlayerVideoHWDecoder(playerHandle, 1);
int isSupportHevcHwDecoder = libPlayer.SetSmartPlayerVideoHevcHWDecoder(playerHandle, 1);
Log.i(TAG, "isSupportH264HwDecoder: " + isSupportH264HwDecoder + ", isSupportHevcHwDecoder: " + isSupportHevcHwDecoder);
}
libPlayer.SmartPlayerSetLowLatencyMode(playerHandle, isLowLatency ? 1
: 0);
libPlayer.SmartPlayerSetRotation(playerHandle, rotate_degrees);
int iPlaybackRet = libPlayer
.SmartPlayerStartPlay(playerHandle);
if (iPlaybackRet != 0) {
Log.e(TAG, "StartPlay failed!");
if ( !isPulling && !isRecording && !isPushing && !isRTSPPublisherRunning)
{
releasePlayerHandle();
}
return false;
}
isPlaying = true;
return true;
}
OpenPullHandle()对应的实现如下：
/*
* SmartRelayDemo.java
* Created: daniusdk.com
*/
private boolean OpenPullHandle()
{
//if (playerHandle != 0) {
// return true;
//}
if(isPulling || isPlaying || isRecording)
return true;
//playbackUrl = "rtsp://xxxx";
if (playbackUrl == null) {
Log.e(TAG, "playback URL is null...");
return false;
}
playerHandle = libPlayer.SmartPlayerOpen(myContext);
if (playerHandle == 0) {
Log.e(TAG, "playerHandle is nil..");
return false;
}
libPlayer.SetSmartPlayerEventCallbackV2(playerHandle,
new EventHandlePlayerV2());
libPlayer.SmartPlayerSetBuffer(playerHandle, playBuffer);
// set report download speed
libPlayer.SmartPlayerSetReportDownloadSpeed(playerHandle, 1, 5);
//设置RTSP超时时间
int rtsp_timeout = 12;
libPlayer.SmartPlayerSetRTSPTimeout(playerHandle, rtsp_timeout);
//设置RTSP TCP/UDP模式自动切换
int is_auto_switch_tcp_udp = 1;
libPlayer.SmartPlayerSetRTSPAutoSwitchTcpUdp(playerHandle, is_auto_switch_tcp_udp);
libPlayer.SmartPlayerSaveImageFlag(playerHandle, 1);
// It only used when playback RTSP stream..
//libPlayer.SmartPlayerSetRTSPTcpMode(playerHandle, 1);
libPlayer.SmartPlayerSetUrl(playerHandle, playbackUrl);
return true;
}
拉流端的Event回调状态如下，拉流端主要关注的是链接状态，还有实时下载速度：
class EventHandlePlayerV2 implements NTSmartEventCallbackV2 {
@Override
public void onNTSmartEventCallbackV2(long handle, int id, long param1,
long param2, String param3, String param4, Object param5) {
//Log.i(TAG, "EventHandleV2: handle=" + handle + " id:" + id);
String player_event = "";
switch (id) {
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_STARTED:
player_event = "开始..";
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTING:
player_event = "连接中..";
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTION_FAILED:
player_event = "连接失败..";
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CONNECTED:
player_event = "连接成功..";
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_DISCONNECTED:
player_event = "连接断开..";
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_STOP:
player_event = "停止播放..";
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_RESOLUTION_INFO:
player_event = "分辨率信息: width: " + param1 + ", height: " + param2;
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_NO_MEDIADATA_RECEIVED:
player_event = "收不到媒体数据，可能是url错误..";
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_SWITCH_URL:
player_event = "切换播放URL..";
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_CAPTURE_IMAGE:
player_event = "快照: " + param1 + " 路径：" + param3;
if (param1 == 0) {
player_event = player_event + ", 截取快照成功";
} else {
player_event = player_event + ", 截取快照失败";
}
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_RECORDER_START_NEW_FILE:
player_event = "[record]开始一个新的录像文件 : " + param3;
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_ONE_RECORDER_FILE_FINISHED:
player_event = "[record]已生成一个录像文件 : " + param3;
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_START_BUFFERING:
Log.i(TAG, "Start Buffering");
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_BUFFERING:
Log.i(TAG, "Buffering:" + param1 + "%");
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_STOP_BUFFERING:
Log.i(TAG, "Stop Buffering");
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_DOWNLOAD_SPEED:
player_event = "download_speed:" + param1 + "Byte/s" + ", "
+ (param1 * 8 / 1000) + "kbps" + ", " + (param1 / 1024)
+ "KB/s";
break;
case NTSmartEventID.EVENT_DANIULIVE_ERC_PLAYER_RTSP_STATUS_CODE:
Log.e(TAG, "RTSP error code received, please make sure username/password is correct, error code:" + param1);
player_event = "RTSP error code:" + param1;
break;
}
}
}
下一步，是启动RTSP服务：
//启动/停止RTSP服务
class ButtonRtspServiceListener implements OnClickListener {
public void onClick(View v) {
if (isRTSPServiceRunning) {
stopRtspService();
btnRtspService.setText("启动RTSP服务");
btnRtspPublisher.setEnabled(false);
isRTSPServiceRunning = false;
return;
}
if(!OpenPushHandle())
{
return;
}
Log.i(TAG, "onClick start rtsp service..");
rtsp_handle_ = libPublisher.OpenRtspServer(0);
if (rtsp_handle_ == 0) {
Log.e(TAG, "创建rtsp server实例失败! 请检查SDK有效性");
} else {
int port = 8554;
if (libPublisher.SetRtspServerPort(rtsp_handle_, port) != 0) {
libPublisher.CloseRtspServer(rtsp_handle_);
rtsp_handle_ = 0;
Log.e(TAG, "创建rtsp server端口失败! 请检查端口是否重复或者端口不在范围内!");
}
//String user_name = "admin";
//String password = "12345";
//libPublisher.SetRtspServerUserNamePassword(rtsp_handle_, user_name, password);
if (libPublisher.StartRtspServer(rtsp_handle_, 0) == 0) {
Log.i(TAG, "启动rtsp server 成功!");
} else {
libPublisher.CloseRtspServer(rtsp_handle_);
rtsp_handle_ = 0;
Log.e(TAG, "启动rtsp server失败! 请检查设置的端口是否被占用!");
}
btnRtspService.setText("停止RTSP服务");
btnRtspPublisher.setEnabled(true);
isRTSPServiceRunning = true;
}
}
}
如果需要停止服务，对应实现如下：
//停止RTSP服务
private void stopRtspService() {
if(!isRTSPServiceRunning)
return;
if (libPublisher != null && rtsp_handle_ != 0) {
libPublisher.StopRtspServer(rtsp_handle_);
libPublisher.CloseRtspServer(rtsp_handle_);
rtsp_handle_ = 0;
}
if(!isPushing)
{
releasePublisherHandle();
}
isRTSPServiceRunning = false;
}
发布、停止发布RTSP流：
private boolean StartRtspStream()
{
if (isRTSPPublisherRunning)
return false;
String rtsp_stream_name = "stream1";
libPublisher.SetRtspStreamName(publisherHandle, rtsp_stream_name);
libPublisher.ClearRtspStreamServer(publisherHandle);
libPublisher.AddRtspStreamServer(publisherHandle, rtsp_handle_, 0);
if (libPublisher.StartRtspStream(publisherHandle, 0) != 0)
{
Log.e(TAG, "调用发布rtsp流接口失败!");
if (!isPushing)
{
libPublisher.SmartPublisherClose(publisherHandle);
publisherHandle = 0;
}
return false;
}
isRTSPPublisherRunning = true;
return true;
}
//停止发布RTSP流
private void stopRtspPublisher()
{
if(!isRTSPPublisherRunning)
return;
isRTSPPublisherRunning = false;
if (null == libPublisher || 0 == publisherHandle)
return;
libPublisher.StopRtspStream(publisherHandle);
if (!isPushing && !isRTSPServiceRunning)
{
releasePublisherHandle();
}
}
因为处理后YUV或RGB数据需要重新编码，这时候需要推送端，设置下编码参数：
private boolean OpenPushHandle() {
if(publisherHandle != 0)
{
return true;
}
publisherHandle = libPublisher.SmartPublisherOpen(myContext, audio_opt, video_opt,
videoWidth, videoHeight);
if (publisherHandle == 0) {
Log.e(TAG, "sdk open failed!");
return false;
}
Log.i(TAG, "publisherHandle=" + publisherHandle);
int fps = 20;
int gop = fps * 1;
int videoEncodeType = 1; //1: h.264硬编码 2: H.265硬编码
if(videoEncodeType == 1) {
int h264HWKbps = setHardwareEncoderKbps(true, videoWidth, videoHeight);
h264HWKbps = h264HWKbps*fps/25;
Log.i(TAG, "h264HWKbps: " + h264HWKbps);
int isSupportH264HWEncoder = libPublisher
.SetSmartPublisherVideoHWEncoder(publisherHandle, h264HWKbps);
if (isSupportH264HWEncoder == 0) {
libPublisher.SetNativeMediaNDK(publisherHandle, 0);
libPublisher.SetVideoHWEncoderBitrateMode(publisherHandle, 1); // 0:CQ, 1:VBR, 2:CBR
libPublisher.SetVideoHWEncoderQuality(publisherHandle, 39);
libPublisher.SetAVCHWEncoderProfile(publisherHandle, 0x08); // 0x01: Baseline, 0x02: Main, 0x08: High
// libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x200); // Level 3.1
// libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x400); // Level 3.2
// libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x800); // Level 4
libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x1000); // Level 4.1 多数情况下,这个够用了
//libPublisher.SetAVCHWEncoderLevel(publisherHandle, 0x2000); // Level 4.2
// libPublisher.SetVideoHWEncoderMaxBitrate(publisherHandle, ((long)h264HWKbps)*1300);
Log.i(TAG, "Great, it supports h.264 hardware encoder!");
}
}
else if (videoEncodeType == 2) {
int hevcHWKbps = setHardwareEncoderKbps(false, videoWidth, videoHeight);
hevcHWKbps = hevcHWKbps*fps/25;
Log.i(TAG, "hevcHWKbps: " + hevcHWKbps);
int isSupportHevcHWEncoder = libPublisher
.SetSmartPublisherVideoHevcHWEncoder(publisherHandle, hevcHWKbps);
if (isSupportHevcHWEncoder == 0) {
libPublisher.SetNativeMediaNDK(publisherHandle, 0);
libPublisher.SetVideoHWEncoderBitrateMode(publisherHandle, 0); // 0:CQ, 1:VBR, 2:CBR
libPublisher.SetVideoHWEncoderQuality(publisherHandle, 39);
// libPublisher.SetVideoHWEncoderMaxBitrate(publisherHandle, ((long)hevcHWKbps)*1200);
Log.i(TAG, "Great, it supports hevc hardware encoder!");
}
}
libPublisher.SetSmartPublisherEventCallbackV2(publisherHandle, new EventHandlePublisherV2());
libPublisher.SmartPublisherSetGopInterval(publisherHandle, gop);
libPublisher.SmartPublisherSetFPS(publisherHandle, fps);
return true;
}
I420ExternalRender实现如下，这里可以拿到拉流的RTSP的YUV数据，然后处理后，可以调用推送端的PostLayerImageI420ByteBuffer()投递到轻量级RTSP服务或RTMP推送端编码发送出去。
class I420ExternalRender implements NTExternalRender {
// public static final int NT_FRAME_FORMAT_RGBA = 1;
// public static final int NT_FRAME_FORMAT_ABGR = 2;
// public static final int NT_FRAME_FORMAT_I420 = 3;
private int width_ = 0;
private int height_ = 0;
private int y_row_bytes_ = 0;
private int u_row_bytes_ = 0;
private int v_row_bytes_ = 0;
private ByteBuffer y_buffer_ = null;
private ByteBuffer u_buffer_ = null;
private ByteBuffer v_buffer_ = null;
@Override
public int getNTFrameFormat() {
Log.i(TAG, "I420ExternalRender::getNTFrameFormat return "
+ NT_FRAME_FORMAT_I420);
return NT_FRAME_FORMAT_I420;
}
@Override
public void onNTFrameSizeChanged(int width, int height) {
width_ = width;
height_ = height;
y_row_bytes_ = (width_ + 15) & (~15);
u_row_bytes_ = ((width_ + 1) / 2 + 15) & (~15);
v_row_bytes_ = ((width_ + 1) / 2 + 15) & (~15);
y_buffer_ = ByteBuffer.allocateDirect(y_row_bytes_ * height_);
u_buffer_ = ByteBuffer.allocateDirect(u_row_bytes_
* ((height_ + 1) / 2));
v_buffer_ = ByteBuffer.allocateDirect(v_row_bytes_
* ((height_ + 1) / 2));
Log.i(TAG, "I420ExternalRender::onNTFrameSizeChanged width_="
+ width_ + " height_=" + height_ + " y_row_bytes_="
+ y_row_bytes_ + " u_row_bytes_=" + u_row_bytes_
+ " v_row_bytes_=" + v_row_bytes_);
}
@Override
public ByteBuffer getNTPlaneByteBuffer(int index) {
if (index == 0) {
return y_buffer_;
} else if (index == 1) {
return u_buffer_;
} else if (index == 2) {
return v_buffer_;
} else {
Log.e(TAG, "I420ExternalRender::getNTPlaneByteBuffer index error:" + index);
return null;
}
}
@Override
public int getNTPlanePerRowBytes(int index) {
if (index == 0) {
return y_row_bytes_;
} else if (index == 1) {
return u_row_bytes_;
} else if (index == 2) {
return v_row_bytes_;
} else {
Log.e(TAG, "I420ExternalRender::getNTPlanePerRowBytes index error:" + index);
return 0;
}
}
public void onNTRenderFrame(int width, int height, long timestamp)
{
if ( y_buffer_ == null )
return;
if ( u_buffer_ == null )
return;
if ( v_buffer_ == null )
return;
y_buffer_.rewind();
u_buffer_.rewind();
v_buffer_.rewind();
if( isPushing || isRTSPPublisherRunning )
{
libPublisher.PostLayerImageI420ByteBuffer(publisherHandle, 0, 0, 0,
y_buffer_, 0, y_row_bytes_,
u_buffer_, 0, u_row_bytes_,
v_buffer_, 0, v_row_bytes_,
width_, height_, 0, 0,
960, 540, 0,0);
}
}
}
如果轻量级服务正常启动，会把rtsp的url回调上来：
class EventHandlePublisherV2 implements NTSmartEventCallbackV2 {
@Override
public void onNTSmartEventCallbackV2(long handle, int id, long param1, long param2, String param3, String param4, Object param5) {
Log.i(TAG, "EventHandlePublisherV2: handle=" + handle + " id:" + id);
String publisher_event = "";
switch (id) {
....
case NTSmartEventID.EVENT_DANIULIVE_ERC_PUBLISHER_RTSP_URL:
publisher_event = "RTSP服务URL: " + param3;
break;
}
}
}
技术总结
以上是大概的流程，从RTSP拉流到数据处理后，重新塞给轻量级RTSP服务，然后播放端再从轻量级RTSP服务端拉流，如果针对YUV或RGB算法处理延迟不大的话，整体延迟可轻松达到毫秒级，满足大多数场景的技术诉求。
