Summary of basic process of pushing h264 stream aac stream by android librtmp
1. x264 initialization configuration
Note: I stepped on some pits here, and removed pps and sps from nal in the process of rtmp pushing. Only one nal slice was allowed in a frame. Previously, the test initialization parameters of x264 failed to notice this problem, which led to my pushing stream not being cut into ts stream correctly in nginx server and playing abnormal.
In addition, we need to pay attention to the time line with faac stream when pushing, otherwise the phenomenon that audio and video are not synchronized or that there is no audio stream in the video is likely to occur.
typedef struct x264_args { x264_t *encoder; //Rongjin Copy Rongjin Copy Rongjin Copy int64_t i_pts; //Rongjin Copy Frame Rongjin Copy Rongjin Copy int outf; int width; int height; int init_flag; AVFrame * png_frame; #if H264_TO_MP4_SUPPORT Mp4Args mp4_args; #endif x264_picture_t pic_in; x264_picture_t pic_out;//X 264 Gongjin copying x 264_picture_t Gongjin copying frame Gongjin copying Gongjin copying front Gongjin copying Gongjin copying Gongjin copying Gongjin copying }X264Args;
Define the general structure of x264
int X264encode_Init_Args_ForRtmp(X264Args *args, char *name,int width, int height) { if (args == NULL ) { return -1; } Thread_Lock_By_Kind(THREAD_LOCK_X264_ENCODE); FilterArgs * png_args = NULL ; int ret = 0; int FPS = GOAL_FPS; x264_param_t param; args->i_pts = 0; args->width = width; args->height = height; dmprint_string("encode start"); x264_param_default_preset(¶m, "superfast", "zerolatency");//0 delay guarantees real-time performance param.i_level_idc=30;//Complexity of the algorithm param.i_threads = X264_SYNC_LOOKAHEAD_AUTO;//linux can be set to 0, mainly to prevent the opening of multi-threading led to the output of more NAL fragments, so as to ensure that only one NAL Yizhen // Otherwise, there are several threads that are encoded into several NALU s. The default is true. param.b_sliced_threads = false; param.rc.b_mb_tree = 0;//If this is not zero, it will result in encoding delay frames. In real-time encoding, it must be 0. // This parameter is set so that each I frame is accompanied by sps/pps. // param.b_repeat_headers = 1; / repeat SPS/PPS in front of the key frame // param.b_repeat_headers = 0; param.i_width = width; param.i_height = height; param.i_fps_num = FPS; param.i_fps_den = 1; param.i_csp = X264_CSP_I420;// param.i_keyint_min = FPS * 1; param.i_keyint_max = FPS * 2; param.rc.i_qp_min = 18; // param.rc.i_qp_max =31; //xiaoyu 70 param.analyse.i_me_range = 16; param.rc.i_qp_step = 2; // param.rc.i_rc_method = X264_RC_CQP; // param.rc.i_qp_constant = 23; x264_param_apply_profile(¶m, "baseline");//What is your manuscript like? args->encoder = x264_encoder_open(¶m);//Gallium cloth? x264_picture_alloc(&args->pic_in, X264_CSP_I420, width, height);//Favourite? #if LINUX_VERSION || ANDROID_VERSION free(args->pic_in.img.plane[0] ); #else free( *( ( ( void **) args->pic_in.img.plane[0] ) - 1 ) ); #endif //Do you love me? if (is_watermark_flag) { png_args = get_filter_args(width,height,AV_PIX_FMT_YUV420P); Filter_Alloc_Frame(&args->png_frame,png_args); Frame_Add_WaterMark(width,height,AV_PIX_FMT_YUV420P,2); free(png_args); } args->init_flag = 1; //end: if (ret < 0) { /*close(args->outf);//Smoke and rejoice */x264_encoder_close(args->encoder);//Do you twitch or twitch? if (is_watermark_flag) { is_watermark_flag = 0; Frame_Release_WaterMark(2); Filter_Free_Frame(&args->png_frame); args->png_frame = NULL; } // mp4_for_h264_release(&args->mp4_args); args->init_flag = 0; } Thread_unLock_By_Kind(THREAD_LOCK_X264_ENCODE); return ret; }
2. Encoding a frame
if (mode == 0) // mainly deal with the horizontal screen, vertical screen, mirror when the mobile phone shoots, and turn the image into a normal image {detailYuvPic (buffer, (uint8_t*) pbuf, nw, nh, args - > width, args - > height, orientation, isInversion, isFrontCamera);} else {detailYuvPic_for Android (buffer, (uint8_t*) pf, ngs, width, NH args - > args - > args - > args - > args - > args - > args - > args - > args - > args - > args - > args - Orientation, isInversion, isFrontCamera;} if (is_watermark_flag)// controls whether watermarking is added or not, you can remove this code if you don't need it.//mode = 0 I420 1 NV21 int X264encodeAddframe_Args_ForRtmp(X264Args *args, uint8_t* jb, int nw, int nh, int isFrontCamera, int orientation, int isInversion, char *out, int *nal_len, int *nal_count, int mode) { if (args == NULL || args->init_flag != 1) return -1; Thread_Lock_By_Kind(THREAD_LOCK_X264_ENCODE); int ret = 0; int len = h264_width*h264_height*3 >> 1; uint8_t* buffer = (uint8_t*) jb; x264_nal_t *nals = NULL; int nnal = 0; x264_nal_t *nal = NULL; char *pbuf = NULL; if (jb == NULL) { ret = -1; goto end; } pbuf = (char *)malloc(args->width * args->height * 3); if (pbuf == NULL) goto end;//mode = 0 I420 1 NV21
{ memcpy(args->png_frame->data[0],pbuf,len);Frame_Get_WaterMark(args->png_frame,args->png_frame, 2);buffer = args->png_frame->data[0]; }elsebuffer= (uint8_t *)pbuf;//args->pic_in.img.plane[0] = buffer;args->pic_in.img.plane[1] = buffer+(args->height * args->width);args->pic_in.img.plane[2] = buffer+(args->height * args->width * 5 >> 2);args->pic_in.img.i_plane = 3;//args->pic_in.img.i_stride[0] = args->width;args->pic_in.img.i_stride[1] = args->width >>1;args->pic_in.img.i_stride[2] = args->width >>1;//args->pic_in.img.i_csp = X264_CSP_I420;//pic_in.i_type X264_TYPE_AUTO; // / args - >pic_in.i_pts = args - >pic_in.i_pts =args-> i_pts+++; * NAL_count = 0; RET = x2644_encoder_encode (args->encoder, &nals, &nnals, & &nnal, & & &args->pic_in, & &args->pic pic pic_out); //if (ret > 0) {{len = 0; for (NAL = nals = nals; NAL < nals < nals < nals + + + + + + nnnal +// nananananananals + + + nanananananananaConvenient for rtmp to send video frame {memcpy(out) + len, nal->p_payload, nal->i_payload);len += nal->i_payload;*nal_len = nal->i_payload;*nal_count += 1;nal_len++;}dm_printf("count = %d", nnal);
// If you don't include pps, sps, count==1, if count is greater than 1, chances are you'll send an image that won't work out.
}else{len = 0;*nal_count = 0;}args->i_pts += 66;end:if (pbuf != NULL)free(pbuf);pbuf = NULL;Thread_unLock_By_Kind(THREAD_LOCK_X264_ENCODE);return ret;} 3. The end of x264
int X264encode_Finish_Args_ForRtmp(X264Args *args) { if (args == NULL || args->init_flag != 1) return -1; Thread_Lock_By_Kind(THREAD_LOCK_X264_ENCODE); int num = 0; int ret = 0; dmprint_string("encode finish"); num = x264_encoder_delayed_frames(args->encoder); dm_printf("num = %d\n",num);//A zero-latency code is used. This value is 0. If it is not 0, you may not initialize it correctly.
x264_encoder_close(args->encoder);//if (is_watermark_flag){is_watermark_flag = 0;Frame_Release_WaterMark(2);}if (args->png_frame != NULL){Filter_Free_Frame(&args->png_frame);args->png_frame = NULL;}memset(args, 0, sizeof(X264Args));Thread_unLock_By_Kind(THREAD_LOCK_X264_ENCODE);return ret;}
4. rtmp sends sps pps
static int send_video_sps_pps(RTMP* m_pRtmp, unsigned char *sps, int sps_len, unsigned char *pps, int pps_len) { RTMPPacket * packet; unsigned char * body; int i; int ret = 0; if (m_pRtmp == NULL) return -1; packet = (RTMPPacket *) malloc(RTMP_HEAD_SIZE + 1024); memset(packet, 0, RTMP_HEAD_SIZE); packet->m_body = (char *) packet + RTMP_HEAD_SIZE; body = (unsigned char *) packet->m_body; i = 0; body[i++] = 0x17; body[i++] = 0x00; body[i++] = 0x00; body[i++] = 0x00; body[i++] = 0x00; /*AVCDecoderConfigurationRecord*/ body[i++] = 0x01; body[i++] = sps[1]; body[i++] = sps[2]; body[i++] = sps[3]; body[i++] = 0xff; /*sps*/ body[i++] = 0xe1; body[i++] = (sps_len >> 8) & 0xff; body[i++] = sps_len & 0xff; memcpy(&body[i], sps, sps_len); i += sps_len; /*pps*/ body[i++] = 0x01; body[i++] = (pps_len >> 8) & 0xff; body[i++] = (pps_len) & 0xff; memcpy(&body[i], pps, pps_len); i += pps_len; packet->m_packetType = RTMP_PACKET_TYPE_VIDEO; packet->m_nBodySize = i; packet->m_nChannel = 0x04; packet->m_nTimeStamp = 0; packet->m_hasAbsTimestamp = 0; packet->m_headerType = RTMP_PACKET_SIZE_MEDIUM; packet->m_nInfoField2 = m_pRtmp->m_stream_id; if (RTMP_IsConnected(m_pRtmp)) { //Call Send Interface int success = RTMP_SendPacket(m_pRtmp, packet, TRUE); if (success != 1) { dm_printf("send_video_sps_pps fail"); ret = -1; } } else ret = -1; free(packet); return ret; }
5.rtmp sends normal NAL
6. Initialize rtmpstatic int send_rtmp_video(RTMP* m_pRtmp, unsigned char *data, int data_len, int timestamp) { int type; RTMPPacket * packet; unsigned char * body; unsigned char* buffer = data; unsigned int length = data_len; int ret = 0; if (m_pRtmp == NULL) return -1; /*Remove frame delimiters (there may be two, but SPS or PPS can only be 00 000 01)*/ if (buffer[2] == 0x00) { /*00 00 00 01*/ buffer += 4; length -= 4; } else if (buffer[2] == 0x01) { /*00 00 01*/ buffer += 3; length -= 3; } type = buffer[0] & 0x1f; packet = (RTMPPacket *) malloc(RTMP_HEAD_SIZE + length + 9); memset(packet, 0, RTMP_HEAD_SIZE); packet->m_body = (char *) packet + RTMP_HEAD_SIZE; packet->m_nBodySize = length + 9; /*send video packet*/ body = (unsigned char *) packet->m_body; memset(body, 0, length + 9); /*key frame*/ body[0] = 0x27; if (type == NAL_SLICE_IDR) //This is the key frame { body[0] = 0x17; } body[1] = 0x01; /*nal unit*/ body[2] = 0x00; body[3] = 0x00; body[4] = 0x00; body[5] = (length >> 24) & 0xff; body[6] = (length >> 16) & 0xff; body[7] = (length >> 8) & 0xff; body[8] = (length) & 0xff; /*copy data*/ memcpy(&body[9], buffer, length); packet->m_nTimeStamp = timestamp; packet->m_hasAbsTimestamp = 0; packet->m_packetType = RTMP_PACKET_TYPE_VIDEO; packet->m_nInfoField2 = m_pRtmp->m_stream_id; packet->m_nChannel = 0x04; packet->m_headerType = RTMP_PACKET_SIZE_MEDIUM; if (RTMP_IsConnected(m_pRtmp)) { // Call Send Interface ret = RTMP_SendPacket(m_pRtmp, packet, TRUE); if (ret != 1) { dm_printf("send_rtmp_video fail"); ret = -1; } } else { dm_printf("send_rtmp_video RTMP is not ready"); ret = -1; } free(packet); return ret; }
Defining Structures
typedef struct rtmp_infor { char *aac_buf; char *x264_buf; int nal_len[20]; int nal_count; int audio_first; int video_first; int init_flag; RTMP *prtmp ; X264Args x264args; AudioEncodeFaac aacargs; }RTMPInfor;
int RTMP_Send_Init(char *url, int width, int height, int sample_rate, int channel) { int ret = 0; if (rtmp_infor.init_flag == 1) { dm_printf("RTMP_Send_Init again init"); return -1; } if (url == NULL) { dm_printf("url is NULL"); return -1; } if (width <= 0 || height <= 0 || channel <= 0) { dm_printf("width or height or channel is error"); return -1; } pthread_mutex_lock(&thread_lock_rtmp_send); memset(&rtmp_infor, 0, sizeof(rtmp_infor)); ret = RTMP_Connect(&rtmp_infor.prtmp, url); if (ret < 0) { dm_printf("RTMP_Connect fail"); goto end; } ret = Audio_Encode_Init_Faac(&rtmp_infor.aacargs, sample_rate, channel); if (ret < 0) { dm_printf("Audio_Encode_Init_Faac fail"); goto end; } ret = X264encode_Init_Args_ForRtmp(&rtmp_infor.x264args, NULL, width, height); if (ret < 0) { dm_printf("Audio_Encode_Init_Faac fail"); goto end; } rtmp_infor.aac_buf = (char *)malloc(NB_SAMPLE << channel); rtmp_infor.x264_buf = (char *)malloc(width * height * 3 >> 1); if (rtmp_infor.aac_buf == NULL || rtmp_infor.x264_buf == NULL) { dm_printf("aac_buf or 264_buf alloc fail"); ret = -1; goto end; } rtmp_infor.init_flag = 1; end: if (ret < 0) { X264encode_Finish_Args_ForRtmp(&rtmp_infor.x264args); Audio_Encode_Release_Faac(&rtmp_infor.aacargs); RTMP_Close(&rtmp_infor.prtmp); if (rtmp_infor.aac_buf != NULL) free(rtmp_infor.aac_buf); if (rtmp_infor.x264_buf != NULL) free(rtmp_infor.x264_buf); memset(&rtmp_infor, 0, sizeof(rtmp_infor)); } pthread_mutex_unlock(&thread_lock_rtmp_send); return ret; }
7. rtmp sends a Zhen 264 stream
int RTMP_Send_Video(unsigned char *data, int nw, int nh, int isFrontCamera, int orientation, int isInversion, int mode, int timestamp) { int ret = 0; if (data == NULL) return -1; if (!rtmp_infor.init_flag) { dm_printf("RTMP_Send Not Init"); return -1; } pthread_mutex_lock(&thread_lock_rtmp_send); ret = X264encodeAddframe_Args_ForRtmp(&rtmp_infor.x264args,(uint8_t *)data, nw, nh, isFrontCamera, orientation, isInversion, rtmp_infor.x264_buf, rtmp_infor.nal_len, &rtmp_infor.nal_count, mode); if (ret < 0) goto end; rtmp_send_video(rtmp_infor.prtmp, (unsigned char *)rtmp_infor.x264_buf, rtmp_infor.nal_len, rtmp_infor.nal_count, timestamp, &rtmp_infor.video_first); end: pthread_mutex_unlock(&thread_lock_rtmp_send); return ret; }
8. rtmp releases resources
int RTMP_Send_Release() { if (!rtmp_infor.init_flag) return 0; pthread_mutex_lock(&thread_lock_rtmp_send); X264encode_Finish_Args_ForRtmp(&rtmp_infor.x264args); Audio_Encode_Release_Faac(&rtmp_infor.aacargs); RTMP_Close(&rtmp_infor.prtmp); if (rtmp_infor.aac_buf != NULL) free(rtmp_infor.aac_buf); if (rtmp_infor.x264_buf != NULL) free(rtmp_infor.x264_buf); memset(&rtmp_infor, 0, sizeof(rtmp_infor)); pthread_mutex_unlock(&thread_lock_rtmp_send); return 0; }
Summary: rtmp sends 264 streams, there are still some pits to tread on. I think the most important pit I encountered is that when I code, Yizhen compiles a lot of NAL units, which leads to the flow of pushed past can not be solved. I hope writing this blog will be helpful for beginners.