refactor plugin to use OpenGL fbo rendering

CMakeLists.txt

@@ -14,19 +14,27 @@ find_package(projectM4 4.1.0 REQUIRED Playlist)
 find_package(GStreamer REQUIRED COMPONENTS gstreamer-audio gstreamer-gl gstreamer-pbutils gstreamer-video)
 find_package(GLIB2 REQUIRED)
 
-add_library(gstprojectm SHARED
-    src/caps.h
-    src/caps.c
+add_library(gstprojectm MODULE
+    src/bufferdisposal.h
+    src/bufferdisposal.c
     src/debug.h
     src/debug.c
-    src/config.h
-    src/enums.h
-    src/plugin.h
-    src/plugin.c
-    src/projectm.h
-    src/projectm.c
     src/gstglbaseaudiovisualizer.h
     src/gstglbaseaudiovisualizer.c
+    src/gstpmaudiovisualizer.h
+    src/gstpmaudiovisualizer.c
+    src/gstprojectm.h
+    src/gstprojectm.c
+    src/gstprojectmbase.h
+    src/gstprojectmbase.c
+    src/gstprojectmcaps.h
+    src/gstprojectmcaps.c
+    src/gstprojectmconfig.h
+    src/pushbuffer.h
+    src/pushbuffer.c
+    src/register.c
+    src/renderbuffer.h
+    src/renderbuffer.c
 )
 
 target_include_directories(gstprojectm

README.md

@@ -57,25 +57,50 @@ The documentation has been organized into distinct files, each dedicated to a sp
 - **[OSX](docs/OSX.md)**
 - **[Windows](docs/WINDOWS.md)**
 
-Once the plugin has been installed, you can use it something like this:
+Once the plugin has been installed, you can use it something like this to render to an OpenGL window:
 
 ```shell
-gst-launch pipewiresrc ! queue ! audioconvert ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 ! video/x-raw,width=2048,height=1440,framerate=60/1 ! videoconvert ! xvimagesink sync=false
+gst-launch pipewiresrc ! queue ! audioconvert ! "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! projectm preset=/usr/local/share/projectM/presets preset-duration=10 mesh-size=48,32 is-live=true ! 'video/x-raw(memory:GLMemory),width=2048,height=1440,framerate=60/1' ! glimagesink sync=false
 ```
 
-Or to convert an audio file to video:
+To render from a live source in real-time to a gl window, an identity element can be used to provide a proper timestamp source for the pipeline. This example also includes a texture directory: 
+```shell
+gst-launch souphttpsrc location=http://your-radio-stream is-live=true ! queue ! decodebin ! audioconvert ! "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! identity single-segment=true sync=true ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 mesh-size=48,32 is-live=true texture-dir=/usr/local/share/projectM/presets-milkdrop-texture-pack ! video/x-raw(memory:GLMemory),width=1920,height=1080,framerate=60/1 ! glimagesink sync=false
+```
+
+Or to convert an audio file to video using offline rendering:
 
 ```shell
+gst-launch-1.0 -e \
 filesrc location=input.mp3 ! decodebin name=dec \
     decodebin ! tee name=t \
       t. ! queue ! audioconvert ! audioresample ! \
             capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! avenc_aac bitrate=256000 ! queue ! mux. \
-      t. ! queue ! audioconvert ! projectm preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 ! \
-            identity sync=false ! videoconvert ! videorate ! video/x-raw,framerate=60/1,width=3840,height=2160 ! \
+      t. ! queue ! audioconvert ! capsfilter caps="audio/x-raw, format=S16LE, channels=2, rate=44100" ! \
+           projectm preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 is-live=false ! \
+            identity sync=false ! videoconvert ! videorate ! video/x-raw\(memory:GLMemory\),framerate=60/1,width=3840,height=2160 ! \
+            gldownload \
             x264enc bitrate=35000 key-int-max=300 speed-preset=veryslow ! video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \
   mp4mux name=mux ! filesink location=render.mp4;
 ```
 
+Or converting an audio file with the nVidia optimized encoder, directly from GL memory:
+```shell
+gst-launch-1.0 -e \
+  filesrc location=input.mp3 ! \
+    decodebin ! tee name=t \
+      t. ! queue ! audioconvert ! audioresample ! \
+            capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! \
+            avenc_aac bitrate=320000 ! queue ! mux. \
+      t. ! queue ! audioconvert ! capsfilter caps="audio/x-raw, format=S16LE, channels=2, rate=44100" ! projectm \
+            preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 is-live=false ! \
+            identity sync=false ! videoconvert ! videorate ! \
+            video/x-raw\(memory:GLMemory\),framerate=60/1,width=1920,height=1080 ! \
+            nvh264enc ! h264parse ! \
+            video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \
+    mp4mux name=mux ! filesink location=render.mp4;
+```
+
 Available options
 
 ```shell
@@ -193,21 +218,23 @@ If you have your own ProjectM preset files:
 Once the plugin has been installed, you can use it something like this:
 
 ```shell
-gst-launch pipewiresrc ! queue ! audioconvert ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 ! video/x-raw,width=2048,height=1440,framerate=60/1 ! videoconvert ! xvimagesink sync=false
+gst-launch pipewiresrc ! queue ! audioconvert ! "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 mesh-size=48,32 ! 'video/x-raw(memory:GLMemory),width=2048,height=1440,framerate=60/1' ! glimagesink sync=false
 ```
 
 Or to convert an audio file to video:
 
 ```shell
 gst-launch-1.0 -e \
-  filesrc location=input.mp3  ! \
+  filesrc location=input.mp3 ! decodebin name=dec \
     decodebin ! tee name=t \
       t. ! queue ! audioconvert ! audioresample ! \
-            capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! avenc_aac bitrate=320000 ! queue ! mux. \
-      t. ! queue ! audioconvert ! projectm preset=/usr/local/share/projectM/presets texture-dir=/usr/local/share/projectM/textures preset-duration=6 mesh-size=1024,576 ! \
-            identity sync=false ! videoconvert ! videorate ! video/x-raw,framerate=60/1,width=3840,height=2160 ! \
-            x264enc bitrate=50000 key-int-max=200 speed-preset=veryslow ! video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \
-    mp4mux name=mux ! filesink location=output.mp4
+            capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! avenc_aac bitrate=256000 ! queue ! mux. \
+      t. ! queue ! audioconvert ! capsfilter caps="audio/x-raw, format=S16LE, channels=2, rate=44100" ! \
+           projectm preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 is-live=false ! \
+            identity sync=false ! videoconvert ! videorate ! video/x-raw\(memory:GLMemory\),framerate=60/1,width=3840,height=2160 ! \
+            gldownload \
+            x264enc bitrate=35000 key-int-max=300 speed-preset=veryslow ! video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \
+  mp4mux name=mux ! filesink location=render.mp4;
 ```
 
 You may need to adjust some elements which may or may not be present in your GStreamer installation, such as x264enc, avenc_aac, etc.
@@ -220,6 +247,55 @@ gst-inspect projectm
 
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
 
+## Technical Details
+
+### OpenGL Rendering and Buffer Handling
+
+- projectM output is rendered to OpenGL textures via **Frame Buffer Object (FBO)**.
+- **Textures are pooled** and reused across frames. 
+- Each rendered texture becomes a GStreamer video buffer pushed downstream. **All video buffers stay in GPU memory**.
+
+---
+
+### Timing and Synchronization
+
+The plugin synchronizes rendering to the GStreamer pipeline clock using **audio presentation timestamp (PTS) as the leading reference**.
+
+Pipeline caps control the desired video framerate for rendering. The render loop is **push-based** to conform with 
+GStreamer's pipeline timing concept, and to enable faster-than-real-time rendering.
+A **fixed number of audio samples is consumed per video frame**. 
+
+**Example:** `735 samples per frame at 44.1 kHz = ~60 FPS.`
+
+**Note:** Live pipelines are auto-detected by the plugin if Gstreamer supports it (not supported on Windows). 
+For Windows or other cases where auto-detection is not appropriate, the `is-live` property can be configured.
+The default mode is offline rendering, `is-live=false`.
+
+**Live pipelines only:** Frames may be dropped or rendering FPS adjusted if frame rendering can't keep up with 
+pipeline caps FPS.
+
+Video frame PTS offset is derived from the **first audio buffer PTS** or **segment event** plus accumulated samples to align with audio timing.
+
+
+| Timing Source              | Origin             | Applies to clock | Purpose                                                                                                                                                                                                                                                           |
+|----------------------------|--------------------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Audio Timestamps           | Audio Input        | Always           | Determine video timing and sync.                                                                                                                                                                                                                                  |
+| Sample Rate / Pipeline FPS | Audio Input / Caps | Always           | Defines how many audio samples are used per frame and target FPS.                                                                                                                                                                                                 |
+| Segment Info               | Segment Event      | Always           | Tracks running time and playback position. Used for PTS offsets.                                                                                                                                                                                                  |
+| QoS Feedback               | QoS Event          | Live             | Skips outdated frames to correct sync with downstream sink/pipeline clock.                                                                                                                                                                                        |
+| Render Frame Drop          | Render Loop        | Live             | Drop frames that cannot be rendered in time to keep sync with pipeline clock.                                                                                                                                                                                     |
+| GL Frame Render Duration   | Render Loop        | Live             | Exponential Moving Average of the frame render duration. Adjusts plugin target FPS in case exceeds the real-time budget most of the time.                                                                                                                         |
+| Latency Event              | Render Loop        | Live             | Inform upstream of latency changes in case of adaptive FPS changes (via EMA).                                                                                                                                                                                     |
+| Buffer push clock jitter   | Render Loop        | Live             | Exponential Moving Average of the source pad push jitter caused by the scheduler. Clocks in gstreamer are not guaranteed to be precise with timed waits, as this cannot be guaranteed by the operating system. Adds jitter EMA as a correction to the buffer PTS. |
+
+
+---
+
+
+<p align="right">(<a href="#readme-top">back to top</a>)</p>
+
+---
+
 <!-- CONTRIBUTING -->
 
 ## Contributing
@@ -261,6 +337,8 @@ Blaquewithaq (Discord: SoFloppy#1289) - [@anomievision](https://twitter.com/anom
 
 Mischa (Discord: mish) - [@revmischa](https://github.com/revmischa)
 
+Michael [@mbaetgen-wup](https://github.com/mbaetgen-wup) - michael -at- widerup.com
+
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
 
 <!----------------------------------------------------------------------->

build.ps1

@@ -54,7 +54,7 @@ function Start-ConfigureBuild {
         -DVCPKG_TARGET_TRIPLET=x64-windows `
         -DCMAKE_MSVC_RUNTIME_LIBRARY="MultiThreaded$<$<CONFIG:Debug>:Debug>DLL" `
         -DCMAKE_VERBOSE_MAKEFILE=YES `
-        -DCMAKE_PREFIX_PATH="${Env:PROJECTM_ROOT}/lib/cmake/projectM4"
+        -DCMAKE_PREFIX_PATH="${Env:PROJECTM_ROOT}"
 }
 
 # Copy required DLLs to dist directory
@@ -137,7 +137,7 @@ function Invoke-PromptInstall {
             # Print example command
             Write-Host
             Write-Host "Done! Here's an example command:"
-            Write-Host 'gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! xvimagesink sync=false'
+            Write-Host 'gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw(memory:GLMemory),width=512,height=512,framerate=60/1" ! glimagesink sync=false'
         }
         else {
             Write-Host

build.sh

@@ -4,10 +4,8 @@ set -e
 # Set variables based on OS
 if [[ "$OSTYPE" == "linux-gnu"* ]]; then
     LIB_EXT="so"
-    VIDEO_SINK="xvimagesink"
 elif [[ "$OSTYPE" == "darwin"* ]]; then
     LIB_EXT="dylib"
-    VIDEO_SINK="osxvideosink"
 else
     echo "Unsupported OS!"
     exit 1
@@ -99,7 +97,7 @@ prompt_install() {
             # Print example command
             echo
             echo "Done! Here's an example command:"
-            echo "gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! $VIDEO_SINK sync=false"
+            echo "gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! \"video/x-raw(memory:GLMemory),width=512,height=512,framerate=60/1\" ! videoconvert ! glimagesink sync=false"
         else
             echo
             echo "Done!"

convert.sh

@@ -154,13 +154,16 @@ gst-launch-1.0 -e \
       t. ! queue ! audioconvert ! audioresample ! \
             capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! \
             avenc_aac bitrate=320000 ! queue ! mux. \
-      t. ! queue ! audioconvert ! projectm \
+      t. ! queue ! audioconvert ! capsfilter caps="audio/x-raw, format=S16LE, channels=2, rate=44100" ! \
+          projectm \
             preset=$PRESET_PATH \
             texture-dir=$TEXTURE_DIR \
             preset-duration=$PRESET_DURATION \
+            is-live=false \
             mesh-size=${MESH_X},${MESH_Y} ! \
             identity sync=false ! videoconvert ! videorate ! \
-            video/x-raw,framerate=$FRAMERATE/1,width=$VIDEO_WIDTH,height=$VIDEO_HEIGHT ! \
+            video/x-raw\(memory:GLMemory\),framerate=$FRAMERATE/1,width=$VIDEO_WIDTH,height=$VIDEO_HEIGHT ! \
+            gldownload ! \
             x264enc bitrate=$(($BITRATE * 1000)) key-int-max=200 speed-preset=$SPEED_PRESET ! \
             video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \
     mp4mux name=mux ! filesink location=$OUTPUT_FILE &

docs/LINUX.md

@@ -92,7 +92,7 @@ source ~/.bash_profile
 To utilize the plugin with the example, please install GStreamer
 
 ```bash
-gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! xvimagesink sync=false
+gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw(memory:GLMemory),width=512,height=512,framerate=60/1" ! glimagesink sync=false
 ```
 
 ### Testing

docs/OSX.md

@@ -102,7 +102,7 @@ source ~/.bash_profile
 To utilize the plugin with the example, please install GStreamer
 
 ```bash
-gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! xvimagesink sync=false
+gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw(memory:GLMemory),width=512,height=512,framerate=60/1" ! glimagesink sync=false
 ```
 
 ### Testing

docs/OVERVIEW.md

@@ -15,7 +15,7 @@
   - [x] OSX
   - [ ] Windows (see issues)
 - [x] Accepting an audio/x-raw stream (coded to add more formats later, if needed)
-- [x] Generating a video/x-raw stream (coded to add more formats later, if needed)
+- [x] Generating a video/x-raw(memory:GLMemory) stream (coded to add more formats later, if needed)
 - [x] Utilizing the new C API in libprojectM 4.0
 - [x] Implemented properties with defaults (aka settings)
 

docs/WINDOWS.md

@@ -66,7 +66,7 @@ Copy-Item -Path "dist\gstprojectm.dll" -Destination "$Env:USERPROFILE\.gstreamer
 To utilize the plugin with the example, please install GStreamer
 
 ```powershell
-gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw,width=512,height=512,framerate=60/1" ! videoconvert ! xvimagesink sync=false
+gst-launch-1.0 audiotestsrc ! queue ! audioconvert ! projectm ! "video/x-raw(memory:GLMemory),width=512,height=512,framerate=60/1" ! glimagesink sync=false
 ```
 
 ### Testing