|
| 1 | +# Copyright (c) Meta Platforms, Inc. and affiliates. |
| 2 | +# All rights reserved. |
| 3 | +# |
| 4 | +# This source code is licensed under the BSD-style license found in the |
| 5 | +# LICENSE file in the root directory of this source tree. |
| 6 | + |
| 7 | +# This functionality requires SQLAlchemy 2.0 or later. |
| 8 | + |
| 9 | +import math |
| 10 | +import struct |
| 11 | +from typing import Optional, Tuple |
| 12 | + |
| 13 | +import numpy as np |
| 14 | + |
| 15 | +from pytorch3d.implicitron.dataset.types import ( |
| 16 | + DepthAnnotation, |
| 17 | + ImageAnnotation, |
| 18 | + MaskAnnotation, |
| 19 | + PointCloudAnnotation, |
| 20 | + VideoAnnotation, |
| 21 | + ViewpointAnnotation, |
| 22 | +) |
| 23 | + |
| 24 | +from sqlalchemy import LargeBinary |
| 25 | +from sqlalchemy.orm import ( |
| 26 | + composite, |
| 27 | + DeclarativeBase, |
| 28 | + Mapped, |
| 29 | + mapped_column, |
| 30 | + MappedAsDataclass, |
| 31 | +) |
| 32 | +from sqlalchemy.types import TypeDecorator |
| 33 | + |
| 34 | + |
| 35 | +# these produce policies to serialize structured types to blobs |
| 36 | +def ArrayTypeFactory(shape): |
| 37 | + class NumpyArrayType(TypeDecorator): |
| 38 | + impl = LargeBinary |
| 39 | + |
| 40 | + def process_bind_param(self, value, dialect): |
| 41 | + if value is not None: |
| 42 | + if value.shape != shape: |
| 43 | + raise ValueError(f"Passed an array of wrong shape: {value.shape}") |
| 44 | + return value.astype(np.float32).tobytes() |
| 45 | + return None |
| 46 | + |
| 47 | + def process_result_value(self, value, dialect): |
| 48 | + if value is not None: |
| 49 | + return np.frombuffer(value, dtype=np.float32).reshape(shape) |
| 50 | + return None |
| 51 | + |
| 52 | + return NumpyArrayType |
| 53 | + |
| 54 | + |
| 55 | +def TupleTypeFactory(dtype=float, shape: Tuple[int, ...] = (2,)): |
| 56 | + format_symbol = { |
| 57 | + float: "f", # float32 |
| 58 | + int: "i", # int32 |
| 59 | + }[dtype] |
| 60 | + |
| 61 | + class TupleType(TypeDecorator): |
| 62 | + impl = LargeBinary |
| 63 | + _format = format_symbol * math.prod(shape) |
| 64 | + |
| 65 | + def process_bind_param(self, value, _): |
| 66 | + if value is None: |
| 67 | + return None |
| 68 | + |
| 69 | + if len(shape) > 1: |
| 70 | + value = np.array(value, dtype=dtype).reshape(-1) |
| 71 | + |
| 72 | + return struct.pack(TupleType._format, *value) |
| 73 | + |
| 74 | + def process_result_value(self, value, _): |
| 75 | + if value is None: |
| 76 | + return None |
| 77 | + |
| 78 | + loaded = struct.unpack(TupleType._format, value) |
| 79 | + if len(shape) > 1: |
| 80 | + loaded = _rec_totuple( |
| 81 | + np.array(loaded, dtype=dtype).reshape(shape).tolist() |
| 82 | + ) |
| 83 | + |
| 84 | + return loaded |
| 85 | + |
| 86 | + return TupleType |
| 87 | + |
| 88 | + |
| 89 | +def _rec_totuple(t): |
| 90 | + if isinstance(t, list): |
| 91 | + return tuple(_rec_totuple(x) for x in t) |
| 92 | + |
| 93 | + return t |
| 94 | + |
| 95 | + |
| 96 | +class Base(MappedAsDataclass, DeclarativeBase): |
| 97 | + """subclasses will be converted to dataclasses""" |
| 98 | + |
| 99 | + |
| 100 | +class SqlFrameAnnotation(Base): |
| 101 | + __tablename__ = "frame_annots" |
| 102 | + |
| 103 | + sequence_name: Mapped[str] = mapped_column(primary_key=True) |
| 104 | + frame_number: Mapped[int] = mapped_column(primary_key=True) |
| 105 | + frame_timestamp: Mapped[float] = mapped_column(index=True) |
| 106 | + |
| 107 | + image: Mapped[ImageAnnotation] = composite( |
| 108 | + mapped_column("_image_path"), |
| 109 | + mapped_column("_image_size", TupleTypeFactory(int)), |
| 110 | + ) |
| 111 | + |
| 112 | + depth: Mapped[DepthAnnotation] = composite( |
| 113 | + mapped_column("_depth_path", nullable=True), |
| 114 | + mapped_column("_depth_scale_adjustment", nullable=True), |
| 115 | + mapped_column("_depth_mask_path", nullable=True), |
| 116 | + ) |
| 117 | + |
| 118 | + mask: Mapped[MaskAnnotation] = composite( |
| 119 | + mapped_column("_mask_path", nullable=True), |
| 120 | + mapped_column("_mask_mass", index=True, nullable=True), |
| 121 | + mapped_column( |
| 122 | + "_mask_bounding_box_xywh", |
| 123 | + TupleTypeFactory(float, shape=(4,)), |
| 124 | + nullable=True, |
| 125 | + ), |
| 126 | + ) |
| 127 | + |
| 128 | + viewpoint: Mapped[ViewpointAnnotation] = composite( |
| 129 | + mapped_column( |
| 130 | + "_viewpoint_R", TupleTypeFactory(float, shape=(3, 3)), nullable=True |
| 131 | + ), |
| 132 | + mapped_column( |
| 133 | + "_viewpoint_T", TupleTypeFactory(float, shape=(3,)), nullable=True |
| 134 | + ), |
| 135 | + mapped_column( |
| 136 | + "_viewpoint_focal_length", TupleTypeFactory(float), nullable=True |
| 137 | + ), |
| 138 | + mapped_column( |
| 139 | + "_viewpoint_principal_point", TupleTypeFactory(float), nullable=True |
| 140 | + ), |
| 141 | + mapped_column("_viewpoint_intrinsics_format", nullable=True), |
| 142 | + ) |
| 143 | + |
| 144 | + |
| 145 | +class SqlSequenceAnnotation(Base): |
| 146 | + __tablename__ = "sequence_annots" |
| 147 | + |
| 148 | + sequence_name: Mapped[str] = mapped_column(primary_key=True) |
| 149 | + category: Mapped[str] = mapped_column(index=True) |
| 150 | + |
| 151 | + video: Mapped[VideoAnnotation] = composite( |
| 152 | + mapped_column("_video_path", nullable=True), |
| 153 | + mapped_column("_video_length", nullable=True), |
| 154 | + ) |
| 155 | + point_cloud: Mapped[PointCloudAnnotation] = composite( |
| 156 | + mapped_column("_point_cloud_path", nullable=True), |
| 157 | + mapped_column("_point_cloud_quality_score", nullable=True), |
| 158 | + mapped_column("_point_cloud_n_points", nullable=True), |
| 159 | + ) |
| 160 | + # the bigger the better |
| 161 | + viewpoint_quality_score: Mapped[Optional[float]] = mapped_column(default=None) |
0 commit comments