1. The test takes a video and finds that playing with ijk player does not feel the same as with system player. Playing with ijk player feels like playing Page Carton is a bit smoother than with system player.
Let's take a look at this problem. The reason for this problem is really simple. Because I set the frame loss value to 5, changing to 1 will make the two players feel similar in experience.(You can see the difference by losing 5 frames of the human eye!).
2. Play a 4k (30fps) video on the machine of Goldman 660, but it can't play properly. The decoded frame in one second is only 20 frames, and the actual play is only 4 frames.Causes picture to get stuck and audio and video to get out of sync.
It was later discovered that because of this video, many frames are decoded slowly on the highway machine, resulting in video being slower than audio. When hard decoding loses frames, it is judged that the video has been slower than audio, resulting in video losing frames all the time, thus appearing as Karton.
Frame Dropping Principle
First, you need to understand where frames need to be dropped and what frames need to be dropped.
Frame dropping can drop the frame before decoding or the frame after decoding.
If you want to lose the I frame, you need to lose the whole gop to p revent the flower screen.
The decoded frames can be discarded without regard to the frame type, since the frames are all decoded data, such as yuv, which directly determines whether the audio and video are discarded out of sync.
1. Frame Dropping Design in ffplay
Let's look at the design of frame loss in ffplay: ffplay loses decoded video frames
In the video_thread decoding thread, we can see that the get_video_frame function is mainly used to decode and get the decoded data avframe, and then to detect and determine frame dropouts.
static int get_video_frame(VideoState *is, AVFrame *frame)
{
int got_picture;
//Decode to get decoded data avframe
if ((got_picture = decoder_decode_frame(&is->viddec, frame, NULL)) < 0)
return -1;
if (got_picture) {
double dpts = NAN;
if (frame->pts != AV_NOPTS_VALUE)
dpts = av_q2d(is->video_st->time_base) * frame->pts;//The pts of the video are converted to ms, which is the current progress time
//Video aspect ratio
frame->sample_aspect_ratio = av_guess_sample_aspect_ratio(is->ic, is->video_st, frame);
//Number of frames dropped is greater than 0 and synchronization is not per video
if (framedrop>0 || (framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) {
if (frame->pts != AV_NOPTS_VALUE) {
//frame_last_filter_delay defaults to 0
//diff less than 0, video slower than audio, frame loss
//diff greater than 0, video faster than audio, no frame loss
double diff = dpts - get_master_clock(is);
// AV_NOSYNC_THRESHOLD: Synchronization threshold.If the error is too large, no correction is made and no frame is lost to synchronize
if (!isnan(diff) && fabs(diff) < AV_NOSYNC_THRESHOLD &&
diff - is->frame_last_filter_delay < 0 &&
is->viddec.pkt_serial == is->vidclk.serial &&
is->videoq.nb_packets) {
is->frame_drops_early++;
av_frame_unref(frame);//Frame Drop
got_picture = 0;
}
}
}
}
return got_picture;
}
2.Frame Dropping Design in ijk
In ijk, frame dropouts are also decoded video frames, which are divided into hard-decoded frame dropouts and soft-decoded frame dropouts.
2.1. Design of frame loss in hard decoding
In the ffpipenode_android_mediacodec_vdec.c file,
The func_run_sync function is mainly used to handle the implementation logic of the entire hard decode.
/**
*Hard Decode Processing Flow
**/
static int func_run_sync(IJKFF_Pipenode *node)
{
JNIEnv *env = NULL;
IJKFF_Pipenode_Opaque *opaque = node->opaque;
FFPlayer *ffp = opaque->ffp;
VideoState *is = ffp->is;
Decoder *d = &is->viddec;
PacketQueue *q = d->queue;
int ret = 0;
int dequeue_count = 0;
AVFrame *frame = NULL;
int got_frame = 0;
AVRational tb = is->video_st->time_base;
AVRational frame_rate = av_guess_frame_rate(is->ic, is->video_st, NULL);
double duration;
double pts;
if (!opaque->acodec) {
return ffp_video_thread(ffp);
}
if (JNI_OK != SDL_JNI_SetupThreadEnv(&env)) {
ALOGE("%s: SetupThreadEnv failed\n", __func__);
return -1;
}
frame = av_frame_alloc();
if (!frame)
goto fail;
//Create data queue thread enqueue_thread_func
opaque->enqueue_thread = SDL_CreateThreadEx(&opaque->_enqueue_thread, enqueue_thread_func, node, "amediacodec_input_thread");
if (!opaque->enqueue_thread) {
ALOGE("%s: SDL_CreateThreadEx failed\n", __func__);
ret = -1;
goto fail;
}
//Cyclic Pull Decoded Data
while (!q->abort_request) {
int64_t timeUs = opaque->acodec_first_dequeue_output_request ? 0 : AMC_OUTPUT_TIMEOUT_US;
got_frame = 0;
//Hard Unscramble Get frame
ret = drain_output_buffer(env, node, timeUs, &dequeue_count, frame, &got_frame);
if (opaque->acodec_first_dequeue_output_request) {
SDL_LockMutex(opaque->acodec_first_dequeue_output_mutex);
opaque->acodec_first_dequeue_output_request = false;
SDL_CondSignal(opaque->acodec_first_dequeue_output_cond);
SDL_UnlockMutex(opaque->acodec_first_dequeue_output_mutex);
}
//Data pull error
if (ret != 0) {
ret = -1;
if (got_frame && frame->opaque) //release buffer false notifies MediaCodec to discard this frame
{
SDL_VoutAndroid_releaseBufferProxyP(opaque->weak_vout, (SDL_AMediaCodecBufferProxy **)&frame->opaque, false);
}
goto fail;
}
if (got_frame) {
duration = (frame_rate.num && frame_rate.den ? av_q2d((AVRational){frame_rate.den, frame_rate.num}) : 0);
pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
//Set the frame loss greater than 0 or the number of frames lost is not equal to 0 and the main clock is not a video clock
if (ffp->framedrop > 0 || (ffp->framedrop && ffp_get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) {
ffp->stat.decode_frame_count++;//Decoded Frame Count
if (frame->pts != AV_NOPTS_VALUE) {
double dpts = pts;//Such as a frame of video pts
double diff = dpts - ffp_get_master_clock(is);//Difference between video and audio frames (if the main clock is audio)
//frame_last_filter_delay This time is 0, so diff is greater than 0, meaning that video is faster than audio, and no frame dropout is required
//If diff is less than 0, video is slower than audio and frames need to be dropped
//Audio is faster than video, and the gap is less than the maximum synchronization value, beyond which no synchronization is done
if (!isnan(diff) && fabs(diff) < AV_NOSYNC_THRESHOLD &&
diff - is->frame_last_filter_delay < 0 &&
is->viddec.pkt_serial == is->vidclk.serial &&
is->videoq.nb_packets) {//The packet sequence of the video frame in the decoder is equal to the sequence number in the video clock, and the video queue has video frames
is->frame_drops_early++;
is->continuous_frame_drops_early++;//Initial value is 0
if (is->continuous_frame_drops_early > ffp->framedrop) {//Initialize ontinuous_frame_drops_early to 0 if the continuous_frame_drops_early variable is greater than the number of frames dropped
is->continuous_frame_drops_early = 0;
} else {
ffp->stat.drop_frame_count++;//Frame Loss Plus
//Frame Drop Rate = Number of Frames Dropped/Decoded
ffp->stat.drop_frame_rate = (float)(ffp->stat.drop_frame_count) / (float)(ffp->stat.decode_frame_count);
if (frame->opaque) {//Notify mediacodec, release, do not show
SDL_VoutAndroid_releaseBufferProxyP(opaque->weak_vout, (SDL_AMediaCodecBufferProxy **)&frame->opaque, false);
}
av_frame_unref(frame);//Release frame
continue;
}
}
}
}
//Frame queued, placed in decoded video queue, processed in video_refresh
ret = ffp_queue_picture(ffp, frame, pts, duration, av_frame_get_pkt_pos(frame), is->viddec.pkt_serial);
if (ret) {//Queuing error, release buffer false tells MediaCodec to discard this frame and not show it
if (frame->opaque)
SDL_VoutAndroid_releaseBufferProxyP(opaque->weak_vout, (SDL_AMediaCodecBufferProxy **)&frame->opaque, false);
}
av_frame_unref(frame);
}
}
fail:
av_frame_free(&frame);
opaque->abort = true;
SDL_WaitThread(opaque->enqueue_thread, NULL);
SDL_AMediaCodecFake_abort(opaque->acodec);
if (opaque->n_buf_out) {
free(opaque->amc_buf_out);
opaque->n_buf_out = 0;
opaque->amc_buf_out = NULL;
opaque->off_buf_out = 0;
opaque->last_queued_pts = AV_NOPTS_VALUE;
}
if (opaque->acodec) {
SDL_VoutAndroid_invalidateAllBuffers(opaque->weak_vout);
SDL_LockMutex(opaque->acodec_mutex);
SDL_UnlockMutex(opaque->acodec_mutex);
}
SDL_AMediaCodec_stop(opaque->acodec);
SDL_AMediaCodec_decreaseReferenceP(&opaque->acodec);
ALOGI("MediaCodec: %s: exit: %d", __func__, ret);
return ret;
#if 0 // Hard Solve Error, Soft Solve
fallback_to_ffplay:
ALOGW("fallback to ffplay decoder\n");
return ffp_video_thread(opaque->ffp);
#endif
}
2.2 Soft Decode Frame Loss Design
static int get_video_frame(FFPlayer *ffp, AVFrame *frame)
{
VideoState *is = ffp->is;
int got_picture;
//Video Stream buffer Load Cache Statistics
ffp_video_statistic_l(ffp);
//Soft Dissolution Time-consuming Test
//int64_t starttime = av_gettime_relative();
//Decode to get decoded data avframe
if ((got_picture = decoder_decode_frame(ffp, &is->viddec, frame, NULL)) < 0)
return -1;
/*
if(frame->key_frame) {//Keyframe Soft Decomposition Time-consuming Test
int64_t endtime = av_gettime_relative();
int usetime = endtime - starttime;
ALOGE("zmlruan>>>>>>usetime:%d",usetime);
}*/
if (got_picture) {//Decode successfully, get decoded data
double dpts = NAN;
if (frame->pts != AV_NOPTS_VALUE)
dpts = av_q2d(is->video_st->time_base) * frame->pts;//The pts of the video are converted to ms, which is the current progress time
//Video aspect ratio
frame->sample_aspect_ratio = av_guess_sample_aspect_ratio(is->ic, is->video_st, frame);
//Number of frames dropped is greater than 0 and synchronization is not per video
if (ffp->framedrop>0 || (ffp->framedrop && get_master_sync_type(is) != AV_SYNC_VIDEO_MASTER)) {
ffp->stat.decode_frame_count++;//Decode Number
if (frame->pts != AV_NOPTS_VALUE) {//diff = Video timestamp minus the master clock timestamp (see audio timestamp here)
double diff = dpts - get_master_clock(is); // AV_NOSYNC_THRESHOLD: Synchronization threshold.If the error is too large, no correction will be made
if (!isnan(diff) && fabs(diff) < AV_NOSYNC_THRESHOLD &&
diff - is->frame_last_filter_delay < 0 &&
is->viddec.pkt_serial == is->vidclk.serial &&
is->videoq.nb_packets) {
is->frame_drops_early++;
is->continuous_frame_drops_early++;
if (is->continuous_frame_drops_early > ffp->framedrop) {
is->continuous_frame_drops_early = 0;
} else {
ffp->stat.drop_frame_count++;//Frame Loss Plus
//Frame Drop Rate = Number of Frames Dropped/Decoded
ffp->stat.drop_frame_rate = (float)(ffp->stat.drop_frame_count) / (float)(ffp->stat.decode_frame_count);
av_frame_unref(frame);//Frame Drop
got_picture = 0;//Modify the return parameter to indicate that no video frame was captured and lost
}
}
}
}
}
return got_picture;
}
In this way, we can see the implementation logic of frame dropping in ijk.