(Java) Add support for resolution alignment during encoding

kanat · kanat · commit 20f3203f396e · 2024-03-25T21:18:11.000-07:00
diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn
@@ -529,6 +529,10 @@ if (is_android) {
       "api/org/webrtc/SoftwareVideoEncoderFactory.java",
       "api/org/webrtc/SimulcastVideoEncoder.java",
       "api/org/webrtc/SimulcastVideoEncoderFactory.java",
+      "api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java",
+      "api/org/webrtc/DefaultAlignedVideoEncoderFactory.java",
+      "api/org/webrtc/HardwareVideoEncoderWrapper.java",
+      "api/org/webrtc/ResolutionAdjustment.java",
     ]
 
     deps = [
diff --git a/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java b/sdk/android/api/org/webrtc/DefaultAlignedVideoEncoderFactory.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2014-2023 Stream.io Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.webrtc;
+
+import java.util.LinkedHashSet;
+
+/**
+ * The main difference with the standard [DefaultAlignedVideoEncoderFactory] is that this fixes
+ * issues with resolutions that are not aligned (e.g. VP8 requires 16x16 alignment). You can
+ * set the alignment by setting [resolutionAdjustment]. Internally the resolution during streaming
+ * will be cropped to comply with the adjustment. Fallback behaviour is the same as with the
+ * standard [DefaultVideoEncoderFactory] and it will use the SW encoder if HW fails
+ * or is not available.
+ *
+ * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf3042072
+ * e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/SimulcastVideoEnco
+ * derFactoryWrapper.kt#L18
+ */
+public class DefaultAlignedVideoEncoderFactory implements VideoEncoderFactory {
+    private final VideoEncoderFactory hardwareVideoEncoderFactory;
+    private final VideoEncoderFactory softwareVideoEncoderFactory;
+
+    public DefaultAlignedVideoEncoderFactory(
+            EglBase.Context eglContext,
+            boolean enableIntelVp8Encoder,
+            boolean enableH264HighProfile,
+            ResolutionAdjustment resolutionAdjustment
+    ) {
+        VideoEncoderFactory defaultFactory =
+                new HardwareVideoEncoderFactory(eglContext, enableIntelVp8Encoder, enableH264HighProfile);
+        hardwareVideoEncoderFactory = (resolutionAdjustment == ResolutionAdjustment.NONE) ?
+                defaultFactory :
+                new HardwareVideoEncoderWrapperFactory(defaultFactory, resolutionAdjustment.getValue());
+        softwareVideoEncoderFactory = new SoftwareVideoEncoderFactory();
+    }
+
+    @Override
+    public VideoEncoder createEncoder(VideoCodecInfo info) {
+        VideoEncoder softwareEncoder = softwareVideoEncoderFactory.createEncoder(info);
+        VideoEncoder hardwareEncoder = hardwareVideoEncoderFactory.createEncoder(info);
+        if (hardwareEncoder != null && softwareEncoder != null) {
+            return new VideoEncoderFallback(softwareEncoder, hardwareEncoder);
+        }
+        return hardwareEncoder != null ? hardwareEncoder : softwareEncoder;
+    }
+
+    @Override
+    public VideoCodecInfo[] getSupportedCodecs() {
+        LinkedHashSet<VideoCodecInfo> supportedCodecInfos = new LinkedHashSet<>();
+        supportedCodecInfos.addAll(Arrays.asList(softwareVideoEncoderFactory.getSupportedCodecs()));
+        supportedCodecInfos.addAll(Arrays.asList(hardwareVideoEncoderFactory.getSupportedCodecs()));
+        return supportedCodecInfos.toArray(new VideoCodecInfo[0]);
+    }
+}
+
diff --git a/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java b/sdk/android/api/org/webrtc/HardwareVideoEncoderWrapper.java
@@ -0,0 +1,255 @@
+/*
+ * Copyright (c) 2014-2024 Stream.io Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.webrtc;
+
+/**
+ * Original source: https://github.com/shiguredo/sora-android-sdk/blob/3cc88e806ab2f2327bf304207
+ * 2e98d6da9df4408/sora-android-sdk/src/main/kotlin/jp/shiguredo/sora/sdk/codec/HardwareVideoEnco
+ * derWrapperFactory.kt
+ */
+class HardwareVideoEncoderWrapper implements VideoEncoder {
+
+    private static final String TAG = "HardwareVideoEncoderWrapper";
+
+    private final VideoEncoder internalEncoder;
+    private final int alignment;
+
+    public HardwareVideoEncoderWrapper(VideoEncoder internalEncoder, int alignment) {
+        this.internalEncoder = internalEncoder;
+        this.alignment = alignment;
+    }
+
+    private static class CropSizeCalculator {
+
+        private static final String TAG = "CropSizeCalculator";
+
+        private final int alignment;
+        private final int originalWidth;
+        private final int originalHeight;
+        private final int cropX;
+        private final int cropY;
+
+        public CropSizeCalculator(int alignment, int originalWidth, int originalHeight) {
+            this.alignment = alignment;
+            this.originalWidth = originalWidth;
+            this.originalHeight = originalHeight;
+            this.cropX = originalWidth % alignment;
+            this.cropY = originalHeight % alignment;
+            if (originalWidth != 0 && originalHeight != 0) {
+                Logging.v(TAG + " init(): alignment=" + alignment +
+                        " size=" + originalWidth + "x" + originalHeight + " => " + croppedWidth + "x" + croppedHeight);
+            }
+        }
+
+        public int getCroppedWidth() {
+            return originalWidth - cropX;
+        }
+
+        public int getCroppedHeight() {
+            return originalHeight - cropY;
+        }
+
+        public boolean isCropRequired() {
+            return cropX != 0 || cropY != 0;
+        }
+
+        public boolean hasFrameSizeChanged(int nextWidth, int nextHeight) {
+            if (originalWidth == nextWidth && originalHeight == nextHeight) {
+                return false;
+            } else {
+                Logging.v(TAG + " frame size has changed: " +
+                        originalWidth + "x" + originalHeight + " => " + nextWidth + "x" + nextHeight);
+                return true;
+            }
+            return originalWidth != nextWidth || originalHeight != nextHeight;
+        }
+    }
+
+    private CropSizeCalculator calculator = new CropSizeCalculator(1, 0, 0);
+
+    private VideoCodecStatus retryWithoutCropping(int width, int height, Runnable retryFunc) {
+        Logging.v(TAG, "retrying without resolution adjustment");
+        calculator = new CropSizeCalculator(1, width, height);
+        retryFunc.run();
+        return VideoCodecStatus.OK;
+    }
+
+    @Override
+    public VideoCodecStatus initEncode(VideoEncoder.Settings originalSettings, VideoEncoder.Callback callback) {
+        calculator = new CropSizeCalculator(alignment, originalSettings.width, originalSettings.height);
+        if (!calculator.isCropRequired()) {
+            return internalEncoder.initEncode(originalSettings, callback);
+        } else {
+            VideoEncoder.Settings croppedSettings = new VideoEncoder.Settings(
+                    originalSettings.numberOfCores,
+                    calculator.getCroppedWidth(),
+                    calculator.getCroppedHeight(),
+                    originalSettings.startBitrate,
+                    originalSettings.maxFramerate,
+                    originalSettings.numberOfSimulcastStreams,
+                    originalSettings.automaticResizeOn,
+                    originalSettings.capabilities
+            );
+            try {
+                VideoCodecStatus result = internalEncoder.initEncode(croppedSettings, callback);
+                if (result == VideoCodecStatus.FALLBACK_SOFTWARE) {
+                    Logging.e(TAG, "internalEncoder.initEncode() returned FALLBACK_SOFTWARE: " +
+                            "croppedSettings " + croppedSettings);
+                    return retryWithoutCropping(
+                            originalSettings.width,
+                            originalSettings.height,
+                            () -> internalEncoder.initEncode(originalSettings, callback)
+                    );
+                } else {
+                    return result;
+                }
+            } catch (Exception e) {
+                Logging.e(TAG, "internalEncoder.initEncode() failed", e);
+                return retryWithoutCropping(
+                        originalSettings.width,
+                        originalSettings.height,
+                        () -> internalEncoder.initEncode(originalSettings, callback)
+                );
+            }
+        }
+    }
+
+    @Override
+    public VideoCodecStatus release() {
+        return internalEncoder.release();
+    }
+
+    @Override
+    public VideoCodecStatus encode(VideoFrame frame, VideoEncoder.EncodeInfo encodeInfo) {
+        if (calculator.hasFrameSizeChanged(frame.getBuffer().getWidth(), frame.getBuffer().getHeight())) {
+            calculator = new CropSizeCalculator(alignment, frame.getBuffer().getWidth(), frame.getBuffer().getHeight());
+        }
+        if (!calculator.isCropRequired()) {
+            return internalEncoder.encode(frame, encodeInfo);
+        } else {
+            int croppedWidth = calculator.getCroppedWidth();
+            int croppedHeight = calculator.getCroppedHeight();
+            VideoFrame.Buffer croppedBuffer = frame.getBuffer().cropAndScale(
+                    calculator.cropX / 2,
+                    calculator.cropY / 2,
+                    croppedWidth,
+                    croppedHeight,
+                    croppedWidth,
+                    croppedHeight
+            );
+            VideoFrame croppedFrame = new VideoFrame(croppedBuffer, frame.getRotation(), frame.getTimestampNs());
+            try {
+                VideoCodecStatus result = internalEncoder.encode(croppedFrame, encodeInfo);
+                if (result == VideoCodecStatus.FALLBACK_SOFTWARE) {
+                    Logging.e(TAG, "internalEncoder.encode() returned FALLBACK_SOFTWARE");
+                    return retryWithoutCropping(
+                            frame.getBuffer().getWidth(),
+                            frame.getBuffer().getHeight(),
+                            () -> internalEncoder.encode(frame, encodeInfo)
+                    );
+                } else {
+                    return result;
+                }
+            } catch (Exception e) {
+                Logging.e(TAG, "internalEncoder.encode() failed", e);
+                return retryWithoutCropping(
+                        frame.getBuffer().getWidth(),
+                        frame.getBuffer().getHeight(),
+                        () -> internalEncoder.encode(frame, encodeInfo)
+                );
+            } finally {
+                croppedBuffer.release();
+            }
+        }
+    }
+
+    @Override
+    public VideoCodecStatus setRateAllocation(VideoEncoder.BitrateAllocation allocation, int frameRate) {
+        return internalEncoder.setRateAllocation(allocation, frameRate);
+    }
+
+    @Override
+    public VideoEncoder.ScalingSettings getScalingSettings() {
+        return internalEncoder.getScalingSettings();
+    }
+
+    @Override
+    public String getImplementationName() {
+        return internalEncoder.getImplementationName();
+    }
+
+    @Override
+    public long createNativeVideoEncoder() {
+        return internalEncoder.createNativeVideoEncoder();
+    }
+
+    @Override
+    public boolean isHardwareEncoder() {
+        return internalEncoder.isHardwareEncoder();
+    }
+
+    @Override
+    public VideoCodecStatus setRates(VideoEncoder.RateControlParameters rcParameters) {
+        return internalEncoder.setRates(rcParameters);
+    }
+
+    @Override
+    public VideoEncoder.ResolutionBitrateLimits[] getResolutionBitrateLimits() {
+        return internalEncoder.getResolutionBitrateLimits();
+    }
+
+    @Override
+    public VideoEncoder.EncoderInfo getEncoderInfo() {
+        return internalEncoder.getEncoderInfo();
+    }
+}
+
+class HardwareVideoEncoderWrapperFactory implements VideoEncoderFactory {
+
+    private static final String TAG = "HardwareVideoEncoderWrapperFactory";
+
+    private final HardwareVideoEncoderFactory factory;
+    private final int resolutionPixelAlignment;
+
+    public HardwareVideoEncoderWrapperFactory(HardwareVideoEncoderFactory factory, int resolutionPixelAlignment) {
+        this.factory = factory;
+        this.resolutionPixelAlignment = resolutionPixelAlignment;
+        if (resolutionPixelAlignment == 0) {
+            throw new IllegalArgumentException("resolutionPixelAlignment should not be 0");
+        }
+    }
+
+    @Override
+    public VideoEncoder createEncoder(VideoCodecInfo videoCodecInfo) {
+        try {
+            VideoEncoder encoder = factory.createEncoder(videoCodecInfo);
+            if (encoder == null) {
+                return null;
+            }
+            return new HardwareVideoEncoderWrapper(encoder, resolutionPixelAlignment);
+        } catch (Exception e) {
+            Logging.e(TAG, "createEncoder failed", e);
+            return null;
+        }
+    }
+
+    @Override
+    public VideoCodecInfo[] getSupportedCodecs() {
+        return factory.getSupportedCodecs();
+    }
+}
+
+
diff --git a/sdk/android/api/org/webrtc/ResolutionAdjustment.java b/sdk/android/api/org/webrtc/ResolutionAdjustment.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2014-2023 Stream.io Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.webrtc;
+
+/**
+ * Resolution alignment values. Generally the MULTIPLE_OF_16 is recommended
+ * for both VP8 and H264
+ */
+public enum ResolutionAdjustment {
+    NONE(1),
+    MULTIPLE_OF_2(2),
+    MULTIPLE_OF_4(4),
+    MULTIPLE_OF_8(8),
+    MULTIPLE_OF_16(16);
+
+    private final int value;
+
+    private ResolutionAdjustment(int value) {
+        this.value = value;
+    }
+
+    public int getValue() {
+        return value;
+    }
+}
+
diff --git a/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java b/sdk/android/api/org/webrtc/SimulcastAlignedVideoEncoderFactory.java
