ffplay -- source code analysis: code architecture

Keywords: Javascript less Windows network

/* Minimum SDL audio buffer size, in samples. */
// Minimum audio buffer
/* Calculate actual buffer size keeping in mind not cause too frequent audio callbacks */
// To calculate the actual audio buffer size, you do not need to call back too often. The maximum number of audio calls set here is 30 times per second

/* Step size for volume control in dB */
// Audio control steps in db
#define SDL_VOLUME_STEP (0.75)

/* no AV sync correction is done if below the minimum AV sync threshold */
// Minimum synchronization threshold below which synchronization correction is not required
/* AV sync correction is done if above the maximum AV sync threshold */
// Maximum synchronization threshold, if greater than this value, synchronization correction is required
/* If a frame duration is longer than this, it will not be duplicated to compensate AV sync */
// Frame compensation synchronization threshold, if the frame duration is longer than this, it is not used to compensate synchronization
/* no AV correction is done if too big error */
// Synchronization threshold. If the error is too large, no correction will be made

/* maximum audio speed change to get correct sync */
// Maximum audio speed change (percent) for correct synchronization

/* external clock speed adjustment constants for realtime sources based on buffer fullness */
// Adjust the external clock according to the buffer filling time of the real-time bitstream
// minimum value
// Maximum value
// Stepping

/* we use about AUDIO_DIFF_AVG_NB A-V differences to make the average */
// Use difference to average
#define AUDIO_DIFF_AVG_NB   20

/* polls for possible required screen refresh at least this often, should be less than 1/fps */
// Refresh rate should be less than 1/fps
#define REFRESH_RATE 0.01

/* NOTE: the size must be big enough to compensate the hardware audio buffersize size */
/* TODO: We assume that a decoded and resampled frame fits into this buffer */
// Sampling size
#define SAMPLE_ARRAY_SIZE (8 * 65536)

#define CURSOR_HIDE_DELAY 1000000


// Sampling mark
static unsigned sws_flags = SWS_BICUBIC;

// Package list structure
typedef struct MyAVPacketList {
    AVPacket pkt;
    struct MyAVPacketList *next;
    int serial;
} MyAVPacketList;

// Packet queue to be decoded
typedef struct PacketQueue {
    MyAVPacketList *first_pkt, *last_pkt;
    int nb_packets;
    int size;
    int64_t duration;
    int abort_request;
    int serial;
    SDL_mutex *mutex;
    SDL_cond *cond;
} PacketQueue;


// Audio parameters
typedef struct AudioParams {
    int freq;                                   // frequency
    int channels;                               // Channels
    int64_t channel_layout;             // Channel design, mono, dual or stereo
    enum AVSampleFormat fmt;        // Sampling format
    int frame_size;                         //  Sampling size
    int bytes_per_sec;                      // How many bytes per second
} AudioParams;

// Clock
typedef struct Clock {
    double pts;                 // clock base*/
    double pts_drift;           // Update the clock difference / * clock base minus time at which we updated the clock*/
    double last_updated;        // Time of last update
    double speed;               // speed
    int serial;                     // clock is based on a packet with this serial*/
    int paused;                 // Stop sign
    int *queue_serial;          // pointer to the current packet queue serial, used for obsolete clock detection*/
} Clock;

/* Common struct for handling all types of decoded data and allocated render buffers. */
// Decode frame structure
typedef struct Frame {
    AVFrame *frame;     // Frame data
    AVSubtitle sub;         // Subtitle
    int serial;                 // sequence
    double pts;             // Display timestamp for the frame / * presentation timestamp for the frame*/
    double duration;        // Frame display time / * estimated duration of the frame*/
    int64_t pos;                // Position of the frame in the input file*/
    int width;                  // Frame width
    int height;                 // Frame height
    int format;             // format
    AVRational sar;         // Additional parameters
    int uploaded;           // upload
    int flip_v;                 // Reversal
} Frame;

// Decoded frame queue
typedef struct FrameQueue {
    Frame queue[FRAME_QUEUE_SIZE];  // Queue array
    int rindex;                                         // Read index
    int windex;                                     // Write index
    int size;                                               // Size
    int max_size;                                       // Maximum size
    int keep_last;                                      // Keep previous
    int rindex_shown;                               // Read and display
    SDL_mutex *mutex;                           // Mutually exclusive variables
    SDL_cond *cond;                             // Conditional variable
    PacketQueue *pktq;
} FrameQueue;

// Clock synchronization type
enum {
    AV_SYNC_AUDIO_MASTER,       // Audio as synchronization, audio as synchronization by default / * default choice*/
    AV_SYNC_VIDEO_MASTER,       // Video as sync
    AV_SYNC_EXTERNAL_CLOCK, // synchronize to an external clock*/

// Decoder structure
typedef struct Decoder {
    AVPacket pkt;                               // package
    AVPacket pkt_temp;                      // Tundish
    PacketQueue *queue;                 // Packet queue
    AVCodecContext *avctx;              // Decoding context
    int pkt_serial;                             // Packet sequence
    int finished;                                   // Is it over
    int packet_pending;                     // Is there a package waiting
    SDL_cond *empty_queue_cond;     // Empty queue condition variable
    int64_t start_pts;                          // Start timestamp
    AVRational start_pts_tb;                // Additional parameters to start
    int64_t next_pts;                           // Next frame timestamp
    AVRational next_pts_tb;                 // Additional parameters for next frame
    SDL_Thread *decoder_tid;                // Decoding thread
} Decoder;

// Video state structure
typedef struct VideoState {
    SDL_Thread *read_tid;                   // Read thread
    AVInputFormat *iformat;             // Input format
    int abort_request;                          // Request cancellation
    int force_refresh;                          // force refresh
    int paused;                                 // Stop it
    int last_paused;                                // Finally stop
    int queue_attachments_req;          // Queue attachment request
    int seek_req;                                   // Lookup request
    int seek_flags;                             // Lookup flag
    int64_t seek_pos;                           // Lookup location
    int64_t seek_rel;                           // 
    int read_pause_return;                  // Read stop return
    AVFormatContext *ic;                    // Decode format context
    int realtime;                                   // Real time bitstream or not

    Clock audclk;                               // Audio clock
    Clock vidclk;                                   // Video clock
    Clock extclk;                                   // External clock

    FrameQueue pictq;                       // Video queue
    FrameQueue subpq;                       // Caption queue
    FrameQueue sampq;                       // Audio queue

    Decoder auddec;                         // Audio decoder
    Decoder viddec;                         // Video Decoder 
    Decoder subdec;                         // subtitle decoder 

    int audio_stream;                           // Audio stream Id

    int av_sync_type;                           // Synchronization type

    double audio_clock;                     // Audio clock
    int audio_clock_serial;                 // Audio clock sequence
    double audio_diff_cum;                  // Used for AV difference average calculation*/
    double audio_diff_avg_coef;         //  
    double audio_diff_threshold;            // Audio differential threshold
    int audio_diff_avg_count;               // Average difference quantity
    AVStream *audio_st;                     // Audio bitstream
    PacketQueue audioq;                 // Audio packet queue
    int audio_hw_buf_size;                  // Hardware buffer size
    uint8_t *audio_buf;                     // Audio buffer
    uint8_t *audio_buf1;                        // Audio buffer 1
    unsigned int audio_buf_size;            // Audio buffer size / * in bytes*/
    unsigned int audio_buf1_size;       // Audio buffer size 1
    int audio_buf_index;                        // Audio buffer index / * in bytes*/
    int audio_write_buf_size;               // Audio write buffer size
    int audio_volume;                           // volume
    int muted;                                      // Is it mute?
    struct AudioParams audio_src;       // Audio parameters
#if CONFIG_AVFILTER                         
    struct AudioParams audio_filter_src; // Audio filter
    struct AudioParams audio_tgt;       // Audio parameters
    struct SwrContext *swr_ctx;         // Audio transcoding context
    int frame_drops_early;                  // 
    int frame_drops_late;                       // 

    enum ShowMode {                     // Display type
        SHOW_MODE_NONE = -1,        // No display
        SHOW_MODE_VIDEO = 0,            // Display video
        SHOW_MODE_WAVES,                // Show waves, audio
        SHOW_MODE_RDFT,                 // Adaptive filter
        SHOW_MODE_NB                        // 
    } show_mode;
    int16_t sample_array[SAMPLE_ARRAY_SIZE]; // Sampling array
    int sample_array_index;                 // Sampling index
    int last_i_start;                               // From the beginning
    RDFTContext *rdft;                      // Adaptive filter context
    int rdft_bits;                                  // Self use bit rate
    FFTSample *rdft_data;                   // Fast Fourier sampling
    int xpos;                                       // 
    double last_vis_time;                       // 
    SDL_Texture *vis_texture;               // Audio Texture
    SDL_Texture *sub_texture;               // Subtitle
    SDL_Texture *vid_texture;               // Video Texture

    int subtitle_stream;                        // Subtitle bitstream Id
    AVStream *subtitle_st;                  // Subtitle code stream
    PacketQueue subtitleq;                  // Subtitle package queue

    double frame_timer;                     // Frame timer
    double frame_last_returned_time;    // Last return time
    double frame_last_filter_delay;     // Last filter delay
    int video_stream;                           // Video stream Id
    AVStream *video_st;                     // video stream
    PacketQueue videoq;                 // Video packet queue
    double max_frame_duration;          // Maximum duration of a frame - about this, we consider the jump a timestamp continuity
    struct SwsContext *img_convert_ctx; // Video transcoding context
    struct SwsContext *sub_convert_ctx; // Subtitle transcoding context
    int eof;                                            // End sign

    char *filename;                             // file name
    int width, height, xleft, ytop;         // Width and height, actually coordinates
    int step;                                       // Stepping

    int vfilter_idx;                                // Filter index
    AVFilterContext *in_video_filter;   // First filter in the video chain
    AVFilterContext *out_video_filter;  // the last filter in the video chain
    AVFilterContext *in_audio_filter;   // First filter in the audio chain
    AVFilterContext *out_audio_filter;  // Last filter in the audio chain
    AVFilterGraph *agraph;                  // audio filter graph

    // Last video stream Id, last audio stream Id, last subtitle stream Id
    int last_video_stream, last_audio_stream, last_subtitle_stream;

    SDL_cond *continue_read_thread; // Continuous read thread
} VideoState;

/* options specified by the user */
static AVInputFormat *file_iformat; // File input format
static const char *input_filename;      // Enter file name
static const char *window_title;            // Title
static int default_width  = 640;            // Default width
static int default_height = 480;            // Default height
static int screen_width  = 0;               // Screen width
static int screen_height = 0;               // Screen height
static int audio_disable;                       // Do you want to disable playing sound
static int video_disable;                       // Disable video
static int subtitle_disable;                    // Do you want to disable subtitles
static const char* wanted_stream_spec[AVMEDIA_TYPE_NB] = {0};
static int seek_by_bytes = -1;              // 
static int display_disable;                 // Display prohibited
static int borderless;                          // 
static int startup_volume = 100;        // Initial volume
static int show_status = 1;                 // Display state
static int av_sync_type = AV_SYNC_AUDIO_MASTER; // Synchronization type
static int64_t start_time = AV_NOPTS_VALUE;         // start time
static int64_t duration = AV_NOPTS_VALUE;               // interval
static int fast = 0;                                // fast
static int genpts = 0;                          // 
static int lowres = 0;                          // Slow speed
static int decoder_reorder_pts = -1;    // Decoder rearranges timestamps
static int autoexit;                                // No auto exit
static int exit_on_keydown;             // Press to exit
static int exit_on_mousedown;           // Whether to press the mouse to exit
static int loop = 1;                                // loop
static int framedrop = -1;                  // Abandon frames
static int infinite_buffer = -1;                // Infinite buffer
static enum ShowMode show_mode = SHOW_MODE_NONE; // Display type
static const char *audio_codec_name;    // Audio decoder name
static const char *subtitle_codec_name; // Subtitle decoder name
static const char *video_codec_name;    // Video decoder name
double rdftspeed = 0.02;                        // Speed of adaptive filter
static int64_t cursor_last_shown;           // Last display cursor
static int cursor_hidden = 0;                   // Cursor hiding
static const char **vfilters_list = NULL;   // Video Effects
static int nb_vfilters = 0;                     // Number of video filters
static char *afilters = NULL;                   // Audio Filters
static int autorotate = 1;                      // Auto rotate or not

/* current context */
static int is_full_screen;                          // Full screen
static int64_t audio_callback_time;         // Audio callback time

static AVPacket flush_pkt;                      // Refreshed package


static SDL_Window *window;              // window
static SDL_Renderer *renderer;              // Renderer

Ffplay is a player implemented by ffmpeg code. It can play local files or network resources, similar to vlc player. After the ffmpeg environment is configured, open the console and input ffplay [resource name] in the directory where the resource is located to play the media resource. The source code of ffplay is only 3800 lines. Project C is the SDL library used for audio and video display and rendering. After downloading the ffmpeg source code, find ffplay.c, which is the whole code of the ffplay player. I use the source code of ffmpeg version 4.2.1. The main points are as follows: analyze the source code of ffplay. If you are interested, you can have a look. If there is something wrong, please point out. I will reply in time.

ffplay source code analysis (1): code architecture

         ffplay source code analysis (2): exploring custom queues

         ffplay source code analysis (3): read'thread

        Ffplay source code analysis (4): ffplay decoding thread analysis

         Ffplay source code analysis (5): ffplay video display thread analysis

         Ffplay source code analysis (6): ffplay audio output thread analysis

         Ffplay source code analysis (7): ffplay subtitle display thread analysis

        ffplay source code analysis (8): audio and video synchronization -- Foundation

         Source code analysis of ffplay (9): video synchronous audio

         ffplay source code analysis (10): audio synchronous video

         ffplay source code analysis (11): synchronization to external clock


In the broadcast of video files, it generally involves file reading, unpacking, decoding, audio-video output, audio-video synchronization and other technologies. The complete flow chart is as follows:


In this process, there are several threads:

(1) Read thread. Read files, unpack

(2) Audio decoding thread. Decode audio compressed data as PCM data.

(3) Video decoding thread. Decode the compressed video data as image data.

(4) Audio output thread. Based on SDL playback, this thread is actually an internal thread of SDL.

(5) Video output thread. Based on SDL playback, this thread is the main thread of the program.

Due to the existence of multiple threads, multi thread safe queues are used for data transmission between threads, including FrameQueue and PacketQueue. The process of independent output of audio and video will inevitably lead to the asynchronous phenomenon of audio and video, so there will be some control strategies to ensure the synchronous output of audio and video before output. After code analysis, I drew a sketch as follows:

You need the original picture, you can Download here.

ffplay main function code is as follows

  1. //Entry function, initialize SDL library, register SDL message event, start file parsing thread, enter message cycle
  2. int main(int argc, char **argv)
  3. {
  4.    int flags;
  5.    VideoState *is;
  6.    //Call windows API to set dll loading
  7.    init_dynload();
  8.    av_log_set_flags(AV_LOG_SKIP_REPEATED);
  9.    parse_loglevel(argc, argv, options);
  10.    /* register all codecs, demux and protocols */
  12.    avdevice_register_all();
  13. #endif
  14.    avformat_network_init();
  15.    init_opts();
  16.    //System signal setting
  17.    signal(SIGINT , sigterm_handler); /* Interrupt (ANSI).   */
  18.    signal(SIGTERM, sigterm_handler); /* Termination (ANSI). */
  19.    show_banner(argc, argv, options);
  20.    parse_options(NULL, argc, argv, options, opt_input_file);
  21.    //Detect file name, if it is empty, output error
  22.    if (!input_filename)
  23.   {
  24.        show_usage();
  25.        av_log(NULL, AV_LOG_FATAL, "An input file must be specified\n");
  26.        av_log(NULL, AV_LOG_FATAL, "Use -h to get full help or, even better, run 'man %s'\n", program_name);
  27.        exit(1);
  28.   }
  29.    if (display_disable)
  30.   {
  31.        video_disable = 1;
  32.   }
  34.    if (audio_disable)
  35.        flags &= ~SDL_INIT_AUDIO;
  36.    else {
  37.        /* Try to work around an occasional ALSA buffer underflow issue when the
  38.         * period size is NPOT due to ALSA resampling by forcing the buffer size. */
  39.        if (!SDL_getenv("SDL_AUDIO_ALSA_SET_BUFFER_SIZE"))
  40.            SDL_setenv("SDL_AUDIO_ALSA_SET_BUFFER_SIZE","1", 1);
  41.   }
  42.    if (display_disable)
  43.        flags &= ~SDL_INIT_VIDEO;
  44.    //Initialize SDL
  45.    if (SDL_Init (flags))
  46.   {
  47.        av_log(NULL, AV_LOG_FATAL, "Could not initialize SDL - %s\n", SDL_GetError());
  48.        av_log(NULL, AV_LOG_FATAL, "(Did you set the DISPLAY variable?)\n");
  49.        exit(1);
  50.   }
  51.    //SDL event settings
  54.    //Initialize AVPacket
  55.    av_init_packet(&flush_pkt);
  56.    flush_pkt.data = (uint8_t *)&flush_pkt;
  57.    if (!display_disable)
  58.   {
  59.        int flags = SDL_WINDOW_HIDDEN;
  60.        if (alwaysontop)
  61. #if SDL_VERSION_ATLEAST(2,0,5)
  62.            flags |= SDL_WINDOW_ALWAYS_ON_TOP;
  63. #else
  64.            av_log(NULL, AV_LOG_WARNING, "Your SDL version doesn't support SDL_WINDOW_ALWAYS_ON_TOP. Feature will be inactive.\n");
  65. #endif
  66.        if (borderless)
  67.            flags |= SDL_WINDOW_BORDERLESS;
  68.        else
  69.            flags |= SDL_WINDOW_RESIZABLE;
  70.        //create a window
  71.        window = SDL_CreateWindow(program_name, SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, default_width, default_height, flags);
  72.        SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear");
  73.        if (window)
  74.       {
  75.            //Creating a renderer from a window
  76.            renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC);
  77.            if (!renderer)
  78.           {
  79.                av_log(NULL, AV_LOG_WARNING, "Failed to initialize a hardware accelerated renderer: %s\n", SDL_GetError());
  80.                renderer = SDL_CreateRenderer(window, -1, 0);
  81.           }
  82.            if (renderer)
  83.           {
  84.                if (!SDL_GetRendererInfo(renderer, &renderer_info))
  85.                    av_log(NULL, AV_LOG_VERBOSE, "Initialized %s renderer.\n", renderer_info.name);
  86.           }
  87.       }
  88.        if (!window || !renderer || !renderer_info.num_texture_formats)
  89.       {
  90.            av_log(NULL, AV_LOG_FATAL, "Failed to create window or renderer: %s", SDL_GetError());
  91.            do_exit(NULL);
  92.       }
  93.   }
  94.    //Open flow
  95.    is = stream_open(input_filename, file_iformat);
  96.    if (!is)
  97.   {
  98.        av_log(NULL, AV_LOG_FATAL, "Failed to initialize VideoState!\n");
  99.        do_exit(NULL);
  100.   }
  101.    //Enter the event cycle, handle the events of keyboard and mouse
  102.    event_loop(is);
  103.    /* never returns */
  104.    return 0;
  105. }

At initialization, create SDL After windows and renderers are completed, access the stream_open Function. In this function, resources are unpacked, decoded, data frame to column operation, color space conversion, etc., which are carried out in the sub thread of the main thread event_loop Can handle video display, keyboard keys, mouse and other events. Then the mechanism of synchronization and mutual exclusion is used to realize the synchronous playback of audio and video.

main The function is preceded by ffmpeg, SDL Initialization and related information settings, for example, when we use ffplay When playing a file, if no name is entered in the console, an error will be reported as follows:

  1. //Detect file name, if it is empty, output error
  2. if (!input_filename)
  3. {
  4.    show_usage();
  5.    av_log(NULL, AV_LOG_FATAL, "An input file must be specified\n");
  6.    av_log(NULL, AV_LOG_FATAL, "Use -h to get full help or, even better, run 'man %s'\n", program_name);
  7.    exit(1);
  8. }

The error is as follows:


       ffplay The code is still very difficult. This is a complete player design, involving a lot of content, not only the code, but also some knowledge of audio and video. For example, how to design the data frame queue? How to ensure the synchronization of the reading and writing of the queue when multithreading? How to ensure the best performance? After all, the performance of audio and video codec and playback is very important. It is not possible for a file to be played, such as frame loss, carton, etc. all of these are in the code. Many blogs have been consulted, and there are many researches indeed ffmpeg Later, I will reprint them for later analysis and study. Friends who are interested can also have a look. If there is something wrong, please point out it and I will update it in time.

      ffplay The source code can be directly compiled from the project. You can see how I compiled it: VS2019 Compile ffplay Source code

                        <li class="tool-item tool-active is-like "><a href="javascript:;"><svg class="icon" aria-hidden="true">
                            <use xlink:href="#csdnc-thumbsup"></use>
                        </svg><span class="name">Give the thumbs-up</span>
                        <span class="count"></span>
                        <li class="tool-item tool-active is-collection "><a href="javascript:;" data-report-click="{&quot;mod&quot;:&quot;popu_824&quot;}"><svg class="icon" aria-hidden="true">
                            <use xlink:href="#icon-csdnc-Collection-G"></use>
                        </svg><span class="name">Collection</span></a></li>
                        <li class="tool-item tool-active is-share"><a href="javascript:;" data-report-click="{&quot;mod&quot;:&quot;1582594662_002&quot;}"><svg class="icon" aria-hidden="true">
                            <use xlink:href="#icon-csdnc-fenxiang"></use>
                        <!--Reward begins-->
                                                <!--End of reward-->
                                                <li class="tool-item tool-more">
                            <svg t="1575545411852" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5717" xmlns:xlink="http://www.w3.org/1999/xlink" width="200" height="200"><defs><style type="text/css"></style></defs><path d="M179.176 499.222m-113.245 0a113.245 113.245 0 1 0 226.49 0 113.245 113.245 0 1 0-226.49 0Z" p-id="5718"></path><path d="M509.684 499.222m-113.245 0a113.245 113.245 0 1 0 226.49 0 113.245 113.245 0 1 0-226.49 0Z" p-id="5719"></path><path d="M846.175 499.222m-113.245 0a113.245 113.245 0 1 0 226.49 0 113.245 113.245 0 1 0-226.49 0Z" p-id="5720"></path></svg>
                            <ul class="more-box">
                                <li class="item"><a class="article-report">Article report</a></li>
            <div class="person-messagebox">
                <div class="left-message"><a href="https://blog.csdn.net/yao_hou">
                    <img src="https://profile.csdnimg.cn/E/C/8/3_yao_hou" class="avatar_pic" username="yao_hou">
                                            <img src="https://g.csdnimg.cn/static/user-reg-year/1x/4.png" class="user-years">
                <div class="middle-message">
                                        <div class="title"><span class="tit"><a href="https://Blog.csdn.net/yao'ou "data report Click =" {& quot; mod & quot;: & quot; popu & quot;} "target =" ﹐ blank "> leader of Linghu</a></span>
                    <div class="text"><span>188 original articles published</span> · <span>Praise 90</span> · <span>180000 visitors+</span></div>
                                <div class="right-message">
                                            <a href="https://Im. CSDN. Net / im / main. HTML? Username = Yao [Hou "target =" _blank "class =" BTN BTN SM BTN red hole BT button personal letter "> private message
                                                            <a class="btn btn-sm  bt-button personal-watch" data-report-click="{&quot;mod&quot;:&quot;popu_379&quot;}">follow</a>
    Published 107 original articles, won praise 1, visited 4706
    Private letter follow

    Posted by swatisonee on Sun, 15 Mar 2020 04:47:17 -0700