From e51d241eac9fecc372b3e8a9417e36dc16fba05c Mon Sep 17 00:00:00 2001 From: Youenn Fablet Date: Tue, 13 Sep 2022 18:45:22 -0700 Subject: [PATCH 01/11] Introduce VideoFrame metadata --- index.src.html | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/index.src.html b/index.src.html index 85dd7ac8..a6055f42 100644 --- a/index.src.html +++ b/index.src.html @@ -99,6 +99,9 @@ spec: webrtc-svc; urlPrefix: https://w3c.github.io/webrtc-svc/ type: dfn; text: scalability mode identifier; url:#scalabilitymodes* + +spec: ECMASCRIPT; urlPrefix: https://tc39.es/ecma262/ + type: dfn; text: the current Realm; url: #current-realm
@@ -3182,6 +3185,8 @@
   readonly attribute long long? timestamp;          // microseconds
   readonly attribute VideoColorSpace colorSpace;
 
+  VideoFrameMetadata getMetadata();
+
   unsigned long allocationSize(
       optional VideoFrameCopyToOptions options = {});
   Promise> copyTo(
@@ -3204,6 +3209,8 @@
   // Default matches image unless visibleRect is provided.
   [EnforceRange] unsigned long displayWidth;
   [EnforceRange] unsigned long displayHeight;
+
+  VideoFrameMetadata metadata;
 };
 
 dictionary VideoFrameBufferInit {
@@ -3225,6 +3232,9 @@
 
   VideoColorSpaceInit colorSpace;
 };
+
+dictionary VideoFrameMetadata {
+};
 
 
 ### Internal Slots ###{#videoframe-internal-slots}
@@ -3279,6 +3289,14 @@
 : [[color space]]
 :: The {{VideoColorSpace}} associated with this frame.
 
+: \[[metadata]]
+:: The {{VideoFrameMetadata}} associated with this frame.
+
+{{VideoFrameMetadata}} is designed to be extended by other specifications. +Specifications that do extend {{VideoFrameMetadata}} need to make sure that +added properties are serializable. +
+ ### Constructors ###{#videoframe-constructors} @@ -3426,6 +3444,8 @@ 9. Assign the result of running the [=VideoFrame/Pick Color Space=] algorithm, with |colorSpace| and {{VideoFrame/[[format]]}}, to {{VideoFrame/[[color space]]}}. + 10. Assign the result of calling [=Copy VideoFrame metadata=] + with |init|'s {{VideoFrameInit/metadata}} to {{VideoFrame/[[metadata]]}}. 17. Return |frame|. ### Attributes ###{#videoframe-attributes} @@ -3624,6 +3644,12 @@ When invoked, run the [=Close VideoFrame=] algorithm with [=this=]. +: getMetadata() +:: Gets the {{VideoFrameMetadata}} associated with this frame. + + When invoked, return the result of calling [=Copy VideoFrame metadata=] + with [=this=] {{VideoFrame/[[metadata]]}}. + ### Algorithms ###{#videoframe-algorithms} : Create a VideoFrame (with |output|, |timestamp|, |duration|, |displayAspectWidth|, |displayAspectHeight|, and |colorSpace|) :: 1. Let |frame| be a new {{VideoFrame}}, constructed as follows: @@ -3747,6 +3773,8 @@ |otherFrame|.{{VideoFrame/timestamp}} to |frame|.{{VideoFrame/timestamp}}. 11. Assign |format| to |frame|.{{VideoFrame/[[format]]}}. + 12. Assign the result of calling [=Copy VideoFrame metadata=] + with |init|'s {{VideoFrameInit/metadata}} to {{VideoFrame/metadata}}. : Initialize Frame With Resource and Size (with |init|, |frame|, |resource|, |width| and |height|) @@ -3825,6 +3853,7 @@ and {{VideoFrame/[[display height]]}}. 5. Assign `null` to |frame|'s {{VideoFrame/[[duration]]}} and {{VideoFrame/[[timestamp]]}}. + 6. Assign a new {{VideoFrameMetadata}} to |frame|'s {{VideoFrame/[[metadata]]}}. : Parse VideoFrameCopyToOptions (with |options|) :: 1. Let |defaultRect| be the result of performing the getter steps for @@ -3998,6 +4027,13 @@ [=combined buffer layout/allocationSize=]. 9. Return |combinedLayout|. +: Copy VideoFrame metadata (with |metadata|) +:: 1. Let |metadataCopySerialized| be [$StructuredSerialize$](|metadata|). + 2. Let |metadataCopy| be [$StructuredDeserialize$](|metadataCopySerialized|, [=the current Realm=]). + 3. return |metadataCopy|. + +The goal of this algorithm is to ensure that metadata owned by a {{VideoFrame}} is immutable. + ### Transfer and Serialization ###{#videoframe-transfer-serialization} : The {{VideoFrame}} [=transfer steps=] (with |value| and |dataHolder|) are: From c05057b8e49203625423180dcef6b2c9d3cd6a58 Mon Sep 17 00:00:00 2001 From: Youenn Fablet Date: Thu, 15 Sep 2022 11:09:42 -0700 Subject: [PATCH 02/11] Copy metadata from HTMLVideoElement if init metadata is absent. Fix some additional nits. --- index.src.html | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/index.src.html b/index.src.html index a6055f42..1c62e12f 100644 --- a/index.src.html +++ b/index.src.html @@ -3293,7 +3293,7 @@ :: The {{VideoFrameMetadata}} associated with this frame.
{{VideoFrameMetadata}} is designed to be extended by other specifications. -Specifications that do extend {{VideoFrameMetadata}} need to make sure that +Specifications that do extend {{VideoFrameMetadata}} must make sure that added properties are serializable.
@@ -3335,7 +3335,9 @@ {{InvalidStateError}} {{DOMException}}. 2. Let |currentPlaybackFrame| be the {{VideoFrame}} at the [=current playback position=]. - 3. Run the [=VideoFrame/Initialize Frame From Other Frame=] algorithm + 3. If {{VideoFrameInit/metadata}} does not [=map/exists=] in |init|, + assign |currentPlaybackFrame|.{{VideoFrame/[[metadata]]}} to it. + 4. Run the [=VideoFrame/Initialize Frame From Other Frame=] algorithm with |init|, |frame|, and |currentPlaybackFrame|. - {{HTMLCanvasElement}} @@ -3445,7 +3447,7 @@ algorithm, with |colorSpace| and {{VideoFrame/[[format]]}}, to {{VideoFrame/[[color space]]}}. 10. Assign the result of calling [=Copy VideoFrame metadata=] - with |init|'s {{VideoFrameInit/metadata}} to {{VideoFrame/[[metadata]]}}. + with |init|'s {{VideoFrameInit/metadata}} to |frame|.{{VideoFrame/[[metadata]]}}. 17. Return |frame|. ### Attributes ###{#videoframe-attributes} @@ -3774,7 +3776,7 @@ |frame|.{{VideoFrame/timestamp}}. 11. Assign |format| to |frame|.{{VideoFrame/[[format]]}}. 12. Assign the result of calling [=Copy VideoFrame metadata=] - with |init|'s {{VideoFrameInit/metadata}} to {{VideoFrame/metadata}}. + with |init|'s {{VideoFrameInit/metadata}} to |frame|.{{VideoFrame/[[metadata]]}}. : Initialize Frame With Resource and Size (with |init|, |frame|, |resource|, |width| and |height|) From 4a30345f33b94f7d99e04936ea7070b672caf0b3 Mon Sep 17 00:00:00 2001 From: Youenn Fablet Date: Thu, 15 Sep 2022 11:14:13 -0700 Subject: [PATCH 03/11] Additional nit --- index.src.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.src.html b/index.src.html index 1c62e12f..d62275f5 100644 --- a/index.src.html +++ b/index.src.html @@ -3855,7 +3855,7 @@ and {{VideoFrame/[[display height]]}}. 5. Assign `null` to |frame|'s {{VideoFrame/[[duration]]}} and {{VideoFrame/[[timestamp]]}}. - 6. Assign a new {{VideoFrameMetadata}} to |frame|'s {{VideoFrame/[[metadata]]}}. + 6. Assign a new {{VideoFrameMetadata}} to |frame|.{{VideoFrame/[[metadata]]}}. : Parse VideoFrameCopyToOptions (with |options|) :: 1. Let |defaultRect| be the result of performing the getter steps for From 6dd11bcbb1e6a47bb0f14d1931274177c1951723 Mon Sep 17 00:00:00 2001 From: Youenn Fablet Date: Mon, 19 Sep 2022 10:15:06 -0700 Subject: [PATCH 04/11] Fix build error and rename getMetadata to metadata --- index.src.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/index.src.html b/index.src.html index d62275f5..f1e986bf 100644 --- a/index.src.html +++ b/index.src.html @@ -3185,7 +3185,7 @@ readonly attribute long long? timestamp; // microseconds readonly attribute VideoColorSpace colorSpace; - VideoFrameMetadata getMetadata(); + VideoFrameMetadata metadata(); unsigned long allocationSize( optional VideoFrameCopyToOptions options = {}); @@ -3291,6 +3291,7 @@ : \[[metadata]] :: The {{VideoFrameMetadata}} associated with this frame. +
{{VideoFrameMetadata}} is designed to be extended by other specifications. Specifications that do extend {{VideoFrameMetadata}} must make sure that @@ -3646,7 +3647,7 @@ When invoked, run the [=Close VideoFrame=] algorithm with [=this=]. -: getMetadata() +: metadata() :: Gets the {{VideoFrameMetadata}} associated with this frame. When invoked, return the result of calling [=Copy VideoFrame metadata=] From 673a0faa765caef32ca108c7bc32cba04f0d2e7c Mon Sep 17 00:00:00 2001 From: Youenn Fablet Date: Mon, 19 Sep 2022 11:35:47 -0700 Subject: [PATCH 05/11] Throw when calling metadata() in case VideoFrame is closed --- index.src.html | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/index.src.html b/index.src.html index f1e986bf..ef03eb40 100644 --- a/index.src.html +++ b/index.src.html @@ -3650,8 +3650,11 @@ : metadata() :: Gets the {{VideoFrameMetadata}} associated with this frame. - When invoked, return the result of calling [=Copy VideoFrame metadata=] - with [=this=] {{VideoFrame/[[metadata]]}}. + When invoked, run these steps: + 1. If {{platform object/[[Detached]]}} is true, + throw an {{InvalidStateError}} {{DOMException}}. + 2. Return the result of calling [=Copy VideoFrame metadata=] + with {{VideoFrame/[[metadata]]}}. ### Algorithms ###{#videoframe-algorithms} : Create a VideoFrame (with |output|, |timestamp|, |duration|, |displayAspectWidth|, |displayAspectHeight|, and |colorSpace|) From c8fb718058278b2412702b47f6fa8365f6701bcc Mon Sep 17 00:00:00 2001 From: Youenn Fablet Date: Thu, 22 Sep 2022 22:35:58 -0700 Subject: [PATCH 06/11] Introduce metadata registry --- .github/workflows/auto-publish.yml | 5 +++ .gitignore | 1 + index.src.html | 12 ++--- video_frame_metadata_registry.src.html | 61 ++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 video_frame_metadata_registry.src.html diff --git a/.github/workflows/auto-publish.yml b/.github/workflows/auto-publish.yml index eede3057..a951671d 100644 --- a/.github/workflows/auto-publish.yml +++ b/.github/workflows/auto-publish.yml @@ -98,6 +98,11 @@ jobs: echidna_token: ECHIDNA_TOKEN_HEVC_REGISTRATION build_override: | status: NOTE-WD + - source: video_frame_metadata_registry.src.html + destination: video_frame_metadata_registry..html + echidna_token: ECHIDNA_TOKEN_HEVC_REGISTRATION + build_override: | + status: NOTE-WD steps: # See doc at https://github.com/actions/checkout#checkout-v2 - name: Checkout repository diff --git a/.gitignore b/.gitignore index 95a1e5f1..1755a8a0 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ out/ pcm_codec_registration.html ulaw_codec_registration.html ulaw_codec_registration.html +video_frame_metadata_registry.html vorbis_codec_registration.html vp8_codec_registration.html vp9_codec_registration.html diff --git a/index.src.html b/index.src.html index ef03eb40..20bad273 100644 --- a/index.src.html +++ b/index.src.html @@ -100,6 +100,10 @@ spec: webrtc-svc; urlPrefix: https://w3c.github.io/webrtc-svc/ type: dfn; text: scalability mode identifier; url:#scalabilitymodes* +spec: webcodecs-videoframemetadata-registry; urlPrefix: https://w3c.github.io/webcodecs/videoframemetadata_registry.html + type: dictionary + text: VideoFrameMetadata; url: dictdef-videoframemetadata + spec: ECMASCRIPT; urlPrefix: https://tc39.es/ecma262/ type: dfn; text: the current Realm; url: #current-realm
@@ -3232,9 +3236,6 @@ VideoColorSpaceInit colorSpace; }; - -dictionary VideoFrameMetadata { -}; ### Internal Slots ###{#videoframe-internal-slots} @@ -3293,9 +3294,8 @@ :: The {{VideoFrameMetadata}} associated with this frame.
-{{VideoFrameMetadata}} is designed to be extended by other specifications. -Specifications that do extend {{VideoFrameMetadata}} must make sure that -added properties are serializable. +{{VideoFrameMetadata}} is defined in [[webcodecs-videoframemetadata-registry]]. +By design, all {{VideoFrameMetadata}} properties must be serializable.
### Constructors ###{#videoframe-constructors} diff --git a/video_frame_metadata_registry.src.html b/video_frame_metadata_registry.src.html new file mode 100644 index 00000000..9a841805 --- /dev/null +++ b/video_frame_metadata_registry.src.html @@ -0,0 +1,61 @@ +
+Title: WebCodecs VideoFrame Metadata Registry
+Repository: w3c/webcodecs
+Status: NOTE-ED
+Shortname: webcodecs-videoframemetadata-registry
+Level: none
+Group: mediawg
+ED: https://w3c.github.io/webcodecs/videoframemetadata_registry.html
+TR: https://www.w3.org/TR/webcodecs-videoframemetadata-registry/
+Editor:
+
+Boilerplate: omit conformance
+
+Abstract: This registry is intended to enumerate the metadata fields that can be attached
+    to {{VideoFrame}} objects via the {{VideoFrameMetadata}} dictionary..
+
+Markup Shorthands:css no, markdown yes, dfn yes
+!Participate: Git Repository.
+!Participate: File an issue.
+!Version History: https://github.com/w3c/webcodecs/commits
+
+ +
+spec: WEBCODECS; urlPrefix: https://w3c.github.io/webcodecs/#
+    type: dictionary
+        text: VideoFrame; url: dictdef-videoframe
+
+ + +Registration Entry Requirements {#registration-entry-requirements} +================================================================== + +A registration entry, called metadata, has the following requirements: +1. Each metadata is defined as a separate {{VideoFrameMetadata}} dictionary member. +2. Each metadata MUST be serializable. +3. Each metadata should have clearly defined semantics. + In particular, its interactions with the media processing pipeline + (encoders, decoders, renderers...) should be well defined. + +VideoFrameMetadata definition +============================================ + +dictionary VideoFrameMetadata { +}; + + +VideoFrameMetadata members +============================================ + + +Privacy Considerations {#privacy-considerations} +========================================================================== + +Please refer to the section [[WEBCODECS#privacy-considerations|Privacy +Considerations]] in [[WEBCODECS]]. + +Security Considerations {#security-considerations} +========================================================================== + +Please refer to the section [[WEBCODECS#security-considerations|Security +Considerations]] in [[WEBCODECS]]. From 97fee9422fab2191b4083834ee59812b86d32427 Mon Sep 17 00:00:00 2001 From: Youenn Fablet Date: Sat, 1 Oct 2022 23:40:44 +0200 Subject: [PATCH 07/11] Address some of Chris's feedback --- .github/workflows/auto-publish.yml | 4 +-- index.src.html | 14 +++----- video_frame_metadata_registry.src.html | 45 +++++++++++++++----------- 3 files changed, 34 insertions(+), 29 deletions(-) diff --git a/.github/workflows/auto-publish.yml b/.github/workflows/auto-publish.yml index a951671d..2d9adfca 100644 --- a/.github/workflows/auto-publish.yml +++ b/.github/workflows/auto-publish.yml @@ -99,8 +99,8 @@ jobs: build_override: | status: NOTE-WD - source: video_frame_metadata_registry.src.html - destination: video_frame_metadata_registry..html - echidna_token: ECHIDNA_TOKEN_HEVC_REGISTRATION + destination: video_frame_metadata_registry.html + echidna_token: ECHIDNA_TOKEN_VIDEOFRAMEMETADATA_REGISTRY build_override: | status: NOTE-WD steps: diff --git a/index.src.html b/index.src.html index 20bad273..ef2661b3 100644 --- a/index.src.html +++ b/index.src.html @@ -100,9 +100,8 @@ spec: webrtc-svc; urlPrefix: https://w3c.github.io/webrtc-svc/ type: dfn; text: scalability mode identifier; url:#scalabilitymodes* -spec: webcodecs-videoframemetadata-registry; urlPrefix: https://w3c.github.io/webcodecs/videoframemetadata_registry.html - type: dictionary - text: VideoFrameMetadata; url: dictdef-videoframemetadata +spec: webcodecs-video-frame-metadata-registry; urlPrefix: https://w3c.github.io/webcodecs/video-frame-metadata-registry.html + type: dictionary; text: VideoFrameMetadata; url: dictdef-videoframemetadata spec: ECMASCRIPT; urlPrefix: https://tc39.es/ecma262/ type: dfn; text: the current Realm; url: #current-realm @@ -3292,11 +3291,8 @@ : \[[metadata]] :: The {{VideoFrameMetadata}} associated with this frame. - -
-{{VideoFrameMetadata}} is defined in [[webcodecs-videoframemetadata-registry]]. -By design, all {{VideoFrameMetadata}} properties must be serializable. -
+ {{VideoFrameMetadata}} is defined in [[webcodecs-video-frame-metadata-registry]]. + By design, all {{VideoFrameMetadata}} properties are serializable. ### Constructors ###{#videoframe-constructors} @@ -3336,7 +3332,7 @@ {{InvalidStateError}} {{DOMException}}. 2. Let |currentPlaybackFrame| be the {{VideoFrame}} at the [=current playback position=]. - 3. If {{VideoFrameInit/metadata}} does not [=map/exists=] in |init|, + 3. If {{VideoFrameInit/metadata}} does not [=map/exist=] in |init|, assign |currentPlaybackFrame|.{{VideoFrame/[[metadata]]}} to it. 4. Run the [=VideoFrame/Initialize Frame From Other Frame=] algorithm with |init|, |frame|, and |currentPlaybackFrame|. diff --git a/video_frame_metadata_registry.src.html b/video_frame_metadata_registry.src.html index 9a841805..3f8da08a 100644 --- a/video_frame_metadata_registry.src.html +++ b/video_frame_metadata_registry.src.html @@ -1,18 +1,18 @@