force codec not call the ffmpeg api in av_file_open with ffmpeg5.1

[visitewen]

Describe the bug
in Latest version， mod_av force rtmp to H264 codec，seems not take effect,code below:

the codec variable only Assigned，but not call ffmpeg api to take effect.

Package version or git hash
freeswitch1.10.11 ffmpeg 5.1.3

Trace logs

[hthetiot]

I confirmed the issue started on v1.10.10

Captured lines in above issue description refer to

freeswitch/src/mod/applications/mod_av/avformat.c

Line 2328 in 9d69bd6

if ((handle->stream_name && (!strcasecmp(handle->stream_name, "rtmp") || !strcasecmp(handle->stream_name, "rtmps") || !strcasecmp(handle->stream_name, "youtube")))) {

The issue appear when using v1.10.11, v1.10.9 is not impacted and use h264 instead of flv properly, see capture of proper Trace logs when h264 is used.

[hthetiot]

Quick update of what I found to reproduce the issue.

It is related to ffmpeg4 vs ffmpeg5, 1.10.9 working version was on debian bullseye and 1.10.10+ started on bookworm, if I build freeswitch 1.10.12 on debian bullseye I get h264 instead of FLV because of libav packages on 4.3.8 versions when on libav packages 5.1.6 on bookworm fails.

Freeswitch 1.10.12 on debian bookworm based on ffmpeg v5.1.6

2025-07-09 00:29:02.195969 96.47% [INFO] avformat.c:3337 use video codec implementation Video: flv1 (flv), yuv420p(pc, bt709/unknown/unknown), 1280x720, q=2-31, 1320 kb/s
2025-07-09 00:29:02.195969 96.47% [NOTICE] avformat.c:958 video thread start
freeswitch@sip.sylaps.net> /quit
root@sip:/usr/src/freeswitch# dpkg -l | grep libav
ii  libavcodec-dev:arm64                  7:5.1.6-0+deb12u1              arm64        FFmpeg library with de/encoders for audio/video codecs - development files
ii  libavcodec59:arm64                    7:5.1.6-0+deb12u1              arm64        FFmpeg library with de/encoders for audio/video codecs - runtime files
ii  libavformat-dev:arm64                 7:5.1.6-0+deb12u1              arm64        FFmpeg library with (de)muxers for multimedia containers - development files
ii  libavformat59:arm64                   7:5.1.6-0+deb12u1              arm64        FFmpeg library with (de)muxers for multimedia containers - runtime files
ii  libavif15:arm64                       0.11.1-1+deb12u1               arm64        Library for handling .avif files
ii  libavutil-dev:arm64                   7:5.1.6-0+deb12u1              arm64        FFmpeg library with functions for simplifying programming - development files
ii  libavutil57:arm64                     7:5.1.6-0+deb12u1              arm64        FFmpeg library with functions for simplifying programming - runtime files

Freeswitch 1.10.12 on debian bookworm based on ffmpeg v4.3.8

2025-07-09 00:31:50.442432 95.27% [INFO] avformat.c:3337 use video codec implementation Video: h264 (libx264), yuv420p(pc, bt709/unknown/unknown), 1280x720, q=-1--1, 1320 kb/s
2025-07-09 00:31:50.442432 95.27% [NOTICE] avformat.c:958 video thread start
freeswitch@sip.sylaps.net> /quit
root@sip:/usr/src/freeswitch# dpkg -l | grep libav
ii  libavcodec-dev:arm64          7:4.3.8-0+deb11u3                  arm64        FFmpeg library with de/encoders for audio/video codecs - development files
ii  libavcodec58:arm64            7:4.3.8-0+deb11u3                  arm64        FFmpeg library with de/encoders for audio/video codecs - runtime files
ii  libavformat-dev:arm64         7:4.3.8-0+deb11u3                  arm64        FFmpeg library with (de)muxers for multimedia containers - development files
ii  libavformat58:arm64           7:4.3.8-0+deb11u3                  arm64        FFmpeg library with (de)muxers for multimedia containers - runtime files
ii  libavutil-dev:arm64           7:4.3.8-0+deb11u3                  arm64        FFmpeg library with functions for simplifying programming - development files
ii  libavutil56:arm64             7:4.3.8-0+deb11u3                  arm64        FFmpeg library with functions for simplifying programming - runtime files

Current solution is to use ffmpeg v4.

[hthetiot]

and the fix was the following for me avformat.c for ffmpeg v5 aka LIBAVFORMAT_V support

diff --git i/src/mod/applications/mod_av/avformat.c w/src/mod/applications/mod_av/avformat.c
index c1e0052530..62df6489a2 100644
--- i/src/mod/applications/mod_av/avformat.c
+++ w/src/mod/applications/mod_av/avformat.c
@@ -155,6 +155,11 @@ struct av_file_context {
 #endif
        enum AVColorSpace colorspace;

+#if (LIBAVFORMAT_VERSION_MAJOR >= LIBAVFORMAT_V)
+       enum AVCodecID video_codec_id;
+       enum AVCodecID audio_codec_id;
+#endif
+
        int has_audio;
        int has_video;
        
@@ -605,8 +610,6 @@ static switch_status_t add_stream(av_file_context_t *context, MediaStream *mst,
                        mst->height = mm->vh;
                }

-               c->codec_id = codec_id;
-
                /* Resolution must be a multiple of two. */
                c->width    = mst->width;
                c->height   = mst->height;
@@ -705,6 +708,10 @@ static switch_status_t add_stream(av_file_context_t *context, MediaStream *mst,
                c->color_range = AVCOL_RANGE_JPEG;
                switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "colorspace = %d\n", c->colorspace);

+               // switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "AV_CODEC_ID_H264 = %d\n", AV_CODEC_ID_H264);
+               c->codec_id = codec_id;
+               switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "codec_id = %d\n", c->codec_id);
+
                break;
        default:
                break;
@@ -2384,6 +2391,12 @@ static switch_status_t av_file_open(switch_file_handle_t *handle, const char *pa
                switch_log_printf(SWITCH_CHANNEL_LOG, SWITCH_LOG_DEBUG, "use video codec: [%d] %s (%s)\n", video_codec, desc->name, desc->long_name);
 #endif
        }
+       
+       // Store codec IDs AFTER all forcing logic has run
+#if (LIBAVFORMAT_VERSION_MAJOR >= LIBAVFORMAT_V)
+       context->video_codec_id = video_codec;
+       context->audio_codec_id = audio_codec;
+#endif

 #if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V)
        if (fmt->audio_codec != AV_CODEC_ID_NONE) {
@@ -3324,7 +3337,12 @@ static switch_status_t av_file_write_video(switch_file_handle_t *handle, switch_
                context->video_st.width = frame->img->d_w;
                context->video_st.height = frame->img->d_h;
                context->video_st.next_pts = switch_time_now() / 1000;
+
+#if (LIBAVFORMAT_VERSION_MAJOR < LIBAVFORMAT_V)
                if (add_stream(context, &context->video_st, context->fc, &context->video_codec, context->fc->oformat->video_codec, &handle->mm) == SWITCH_STATUS_SUCCESS &&
+#else
+               if (add_stream(context, &context->video_st, context->fc, &context->video_codec, context->video_codec_id, &handle->mm) == SWITCH_STATUS_SUCCESS &&
+#endif
                        open_video(context->fc, context->video_codec, &context->video_st) == SWITCH_STATUS_SUCCESS) {

                        char codec_str[256] = "";

after the fix is applied 'conference $id recording start $rtmpUrl' result in proper h264 not flv1

profile Constrained Baseline, level 4.1, 4:2:0, 8-bit
264 - core 164 r3095 baee400 - H.264/MPEG-4 AVC codec - Copyleft 2003-2022 - http://www.videolan.org/x264.html - options: cabac=0 ref=1 deblock=0:0:0 analyse=0x1:0x111 me=hex subme=2 psy=1 psy_rd=1.00:0.00 mixed_ref=0 me_range=16 chroma_me=1 trellis=0 8x8dct=0 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=0 threads=18 lookahead_threads=5 sliced_threads=0 nr=0 decimate=1 interlaced=0 bluray_compat=0 constrained_intra=0 bframes=0 weightp=0 keyint=40 keyint_min=21 scenecut=40 intra_refresh=0 rc_lookahead=10 rc=cbr mbtree=1 bitrate=1320 ratetol=1.0 qcomp=0.00 qpmin=0 qpmax=69 qpstep=4 vbv_maxrate=1320 vbv_bufsize=1320 nal_hrd=none filler=0 ip_ratio=1.40 aq=1:1.00
2025-07-09 02:04:36.788481 99.20% [INFO] avformat.c:3353 use video codec implementation Video: h264 (libx264), yuv420p(pc, bt709/unknown/unknown), 1280x720, q=-1--1, 1320 kb/s
2025-07-09 02:04:36.788481 99.20% [NOTICE] avformat.c:963 video thread start