Introduce VideoFrame metadata

Author: youennf

index.src.html

@@ -99,6 +99,10 @@
 
 spec: webrtc-svc; urlPrefix: https://w3c.github.io/webrtc-svc/
     type: dfn; text: scalability mode identifier; url:#scalabilitymodes*
+
+spec: ECMASCRIPT; urlPrefix: https://tc39.es/ecma262/
+    type: dfn; text: the current Realm; url: #current-realm
+    type: dfn; text: SetIntegrityLevel; url: #sec-setintegritylevel
 </pre>
 
 <pre class='biblio'>
@@ -3182,6 +3186,8 @@
   readonly attribute long long? timestamp;          // microseconds
   readonly attribute VideoColorSpace colorSpace;
 
+  VideoFrameMetadata getMetadata();
+
   unsigned long allocationSize(
       optional VideoFrameCopyToOptions options = {});
   Promise<sequence<PlaneLayout>> copyTo(
@@ -3204,6 +3210,8 @@
   // Default matches image unless visibleRect is provided.
   [EnforceRange] unsigned long displayWidth;
   [EnforceRange] unsigned long displayHeight;
+
+  VideoFrameMetadata metadata;
 };
 
 dictionary VideoFrameBufferInit {
@@ -3225,6 +3233,10 @@
 
   VideoColorSpaceInit colorSpace;
 };
+
+dictionary VideoFrameMetadata {
+  any user;
+};
 </xmp>
 
 ### Internal Slots ###{#videoframe-internal-slots}
@@ -3279,6 +3291,10 @@
 : <dfn attribute for=VideoFrame>[[color space]]</dfn>
 :: The {{VideoColorSpace}} associated with this frame.
 
+: <dfn attribute for=VideoFrame>\[[metadata]]</dfn>
+:: The {{VideoFrameMetadata}} associated with this frame.
+
+
 ### Constructors ###{#videoframe-constructors}
 
 <dfn constructor for=VideoFrame title="VideoFrame(image, init)">
@@ -3291,8 +3307,11 @@
     with the [=webappapis/entry settings object=]'s
     [=origin/origin=], then throw a {{SecurityError}}
     {{DOMException}}.
-3. Let |frame| be a new {{VideoFrame}}.
-5. Switch on |image|:
+3. Let |metadata| be the result of calling [=Copy VideoFrame metadata=]
+    with |init|'s {{VideoFrameInit/metadata}}.
+4. Let |frame| be a new {{VideoFrame}}.
+5. Assign |metadata| to |frame|'s {{VideoFrame/metadata}}.
+6. Switch on |image|:
     - {{HTMLImageElement}}
     - {{SVGImageElement}}
         1. If {{VideoFrameInit/timestamp}} does not [=map/exist=] in
@@ -3339,7 +3358,7 @@
         1. Run the [=VideoFrame/Initialize Frame From Other Frame=] algorithm
             with |init|, |frame|, and |image|.
 
-6. Return |frame|.
+7. Return |frame|.
 
 
 <dfn constructor for=VideoFrame title="VideoFrame(data, init)">
@@ -3365,9 +3384,11 @@
     Layout and Allocation Size=] algorithm with |parsedRect|,
     |init|.{{VideoFrameBufferInit/format}}, and |optLayout|.
 9. If |combinedLayout| is an exception, throw |combinedLayout|.
-10. If `data.byteLength` is less than |combinedLayout|’s
+10. Let |metadata| be the result of calling [=Copy VideoFrame metadata]
+    with |init|'s {{VideoFrameInit/metadata}}.
+11. If `data.byteLength` is less than |combinedLayout|’s
     [=combined buffer layout/allocationSize=], throw a {{TypeError}}.
-11. Let |resource| be a new [=media resource=] containing a copy of |data|. Use
+12. Let |resource| be a new [=media resource=] containing a copy of |data|. Use
         {{VideoFrameBufferInit/visibleRect}} and {{VideoFrameBufferInit/layout}}
         to determine where in |data| the pixels for each plane reside.
 
@@ -3380,18 +3401,18 @@
         reposition the visible rectangle within |resource|. The final position
         will be reflected by {{VideoFrame/visibleRect}}.
 
-12. Let |resourceCodedWidth| be the coded width of |resource|.
-13. Let |resourceCodedHeight| be the coded height of |resource|.
-14. Let |resourceVisibleLeft| be the left offset for the visible rectangle of
+13. Let |resourceCodedWidth| be the coded width of |resource|.
+14. Let |resourceCodedHeight| be the coded height of |resource|.
+15. Let |resourceVisibleLeft| be the left offset for the visible rectangle of
     |resource|.
-15. Let |resourceVisibleTop| be the top offset for the visible rectangle of
+16. Let |resourceVisibleTop| be the top offset for the visible rectangle of
     |resource|.
 
     ISSUE: The spec <em class="rfc2119">SHOULD</em> provide definitions (and
       possibly diagrams) for coded size, visible rectangle, and display size.
       See [#166](https://github.com/w3c/webcodecs/issues/166).
 
-16. Let |frame| be a new {{VideoFrame}} object initialized as follows:
+17. Let |frame| be a new {{VideoFrame}} object initialized as follows:
     1. Assign |resourceCodedWidth|, |resourceCodedHeight|,
         |resourceVisibleLeft|, and |resourceVisibleTop| to
         {{VideoFrame/[[coded width]]}}, {{VideoFrame/[[coded height]]}},
@@ -3426,7 +3447,8 @@
     9. Assign the result of running the [=VideoFrame/Pick Color Space=]
         algorithm, with |colorSpace| and {{VideoFrame/[[format]]}}, to
         {{VideoFrame/[[color space]]}}.
-17. Return |frame|.
+    10. Assign |metadata| to {{VideoFrame/[[metadata]]}}.
+18. Return |frame|.
 
 ### Attributes ###{#videoframe-attributes}
 : <dfn attribute for=VideoFrame>format</dfn>
@@ -3624,6 +3646,11 @@
 
     When invoked, run the [=Close VideoFrame=] algorithm with [=this=].
 
+: <dfn method for=VideoFrame>getMetadata()</dfn>
+:: Gets the {{VideoFrameMetadata}} associated with this frame.
+
+    When invoked, return [=this=] {{VideoFrame/[[metadata]]}}.
+
 ### Algorithms ###{#videoframe-algorithms}
 : <dfn>Create a VideoFrame</dfn> (with |output|, |timestamp|, |duration|, |displayAspectWidth|, |displayAspectHeight|, and |colorSpace|)
 :: 1. Let |frame| be a new {{VideoFrame}}, constructed as follows:
@@ -3825,6 +3852,8 @@
         and {{VideoFrame/[[display height]]}}.
     5. Assign `null` to |frame|'s {{VideoFrame/[[duration]]}} and
         {{VideoFrame/[[timestamp]]}}.
+    6. Assign `null` to |frame|'s {{VideoFrame/[[metadata]]}}.
+
 
 : <dfn for=VideoFrame>Parse VideoFrameCopyToOptions</dfn> (with |options|)
 ::  1. Let |defaultRect| be the result of performing the getter steps for
@@ -3998,19 +4027,29 @@
             [=combined buffer layout/allocationSize=].
     9. Return |combinedLayout|.
 
+
+: <dfn for=VideoFrame>Copy VideoFrame metadata</dfn> (with |metadata|)
+:: 1. Let |metadataCopySerialized| be [$StructuredSerialize$](|metadata|).
+    2. Let |metadataCopy| be [$StructuredDeserialize$](|metadataCopySerialized|, [=the current Realm=]).
+    3. Let |userMetadataCopy| be |metadataCopy|'s {{VideoFrameMetadata/user}}.
+    4. Call [=SetIntegrityLevel=] with |userMetadataCopy| and frozen.
+    5. return |metadataCopy|.
+
 ### Transfer and Serialization ###{#videoframe-transfer-serialization}
 
 : The {{VideoFrame}} [=transfer steps=] (with |value| and |dataHolder|) are:
 :: 1. If |value|'s {{platform object/[[Detached]]}} is `true`, throw a
         {{DataCloneError}} {{DOMException}}.
-    2. For all {{VideoFrame}} internal slots in |value|, assign the value of
+    2. Let |metadata| be |value|'s {{VideoFrame/metadata}}.
+    3. Assign [$StructuredSerialize$](|metadata|) to |dataHolder|.[[metadata]].
+    4. For all other {{VideoFrame}} internal slots in |value|, assign the value of
         each internal slot to a field in |dataHolder| with the same name as the
         internal slot.
-    3. Run the [=Close VideoFrame=] algorithm with |value|.
+    5. Run the [=Close VideoFrame=] algorithm with |value|.
 
 : The {{VideoFrame}} [=transfer-receiving steps=] (with |dataHolder| and
     |value|) are:
-:: 1. For all named fields in |dataHolder|, assign the value of each named
+:: 1. For all other named fields in |dataHolder|, assign the value of each named
         field to the {{VideoFrame}} internal slot in |value| with the same name
         as the named field.
 
@@ -4023,14 +4062,18 @@
             |value|'s {{VideoFrame/[[resource reference]]}}.
     4. Let |newReference| be a new reference to |resource|.
     5. Assign |newReference| to |serialized|.[[resource reference]].
-    6. For all remaining {{VideoFrame}} internal slots (excluding
-        {{VideoFrame/[[resource reference]]}}) in |value|, assign the value of
+    6. Let |metadata| be |value|'s {{VideoFrame/metadata}}.
+    7. Assign [$StructuredSerialize$](|metadata|) to |serialized|.[[metadata]].
+    8. For all remaining {{VideoFrame}} internal slots (excluding
+        {{VideoFrame/[[resource reference]]}} and {{VideoFrame/[[metadata]]}}) in |value|, assign the value of
         each internal slot to a field in |serialized| with the same name as the
         internal slot.
 
 : The {{VideoFrame}} [=deserialization steps=] (with |serialized| and |value|)
     are:
-:: 1. For all named fields in |serialized|, assign the value of each named
+:: 1. Let |metadata| be [$StructuredDeserialize$](|serialized|.[[metadata]], [=the current Realm=]).
+    2. Assign |metadata| to |value|'s {{VideoFrame/metadata}}.
+    3. For all other named fields in |serialized|, assign the value of each named
         field to the {{VideoFrame}} internal slot in |value| with the same name
         as the named field.