enhance: Sync PyMilvus milvus-proto commit to v2.6.2 (#2995)

- enhance: Make milvus-lite an optional dependency (#2976)

this PR is a possible implementation addressing this
#2975

Signed-off-by: yangxuan <[email protected]>
Co-authored-by: Jagath Jai Kumar <[email protected]>
Co-authored-by: Jagath Jai Kumar <[email protected]>

Author: XuanYang-cn

pymilvus/grpc_gen/common_pb2.py

@@ -375,6 +375,15 @@ class LoadPriority(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
 class FileResourceType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     __slots__ = ()
     ANALYZER_DICTIONARY: _ClassVar[FileResourceType]
+
+class WALName(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    Unknown: _ClassVar[WALName]
+    RocksMQ: _ClassVar[WALName]
+    Pulsar: _ClassVar[WALName]
+    Kafka: _ClassVar[WALName]
+    WoodPecker: _ClassVar[WALName]
+    Test: _ClassVar[WALName]
 Success: ErrorCode
 UnexpectedError: ErrorCode
 ConnectFailed: ErrorCode
@@ -696,6 +705,12 @@ LoadStateLoaded: LoadState
 HIGH: LoadPriority
 LOW: LoadPriority
 ANALYZER_DICTIONARY: FileResourceType
+Unknown: WALName
+RocksMQ: WALName
+Pulsar: WALName
+Kafka: WALName
+WoodPecker: WALName
+Test: WALName
 PRIVILEGE_EXT_OBJ_FIELD_NUMBER: _ClassVar[int]
 privilege_ext_obj: _descriptor.FieldDescriptor
 
@@ -892,3 +907,74 @@ class NodeInfo(_message.Message):
     address: str
     hostname: str
     def __init__(self, node_id: _Optional[int] = ..., address: _Optional[str] = ..., hostname: _Optional[str] = ...) -> None: ...
+
+class ReplicateConfiguration(_message.Message):
+    __slots__ = ("clusters", "cross_cluster_topology")
+    CLUSTERS_FIELD_NUMBER: _ClassVar[int]
+    CROSS_CLUSTER_TOPOLOGY_FIELD_NUMBER: _ClassVar[int]
+    clusters: _containers.RepeatedCompositeFieldContainer[MilvusCluster]
+    cross_cluster_topology: _containers.RepeatedCompositeFieldContainer[CrossClusterTopology]
+    def __init__(self, clusters: _Optional[_Iterable[_Union[MilvusCluster, _Mapping]]] = ..., cross_cluster_topology: _Optional[_Iterable[_Union[CrossClusterTopology, _Mapping]]] = ...) -> None: ...
+
+class ConnectionParam(_message.Message):
+    __slots__ = ("uri", "token")
+    URI_FIELD_NUMBER: _ClassVar[int]
+    TOKEN_FIELD_NUMBER: _ClassVar[int]
+    uri: str
+    token: str
+    def __init__(self, uri: _Optional[str] = ..., token: _Optional[str] = ...) -> None: ...
+
+class MilvusCluster(_message.Message):
+    __slots__ = ("cluster_id", "connection_param", "pchannels")
+    CLUSTER_ID_FIELD_NUMBER: _ClassVar[int]
+    CONNECTION_PARAM_FIELD_NUMBER: _ClassVar[int]
+    PCHANNELS_FIELD_NUMBER: _ClassVar[int]
+    cluster_id: str
+    connection_param: ConnectionParam
+    pchannels: _containers.RepeatedScalarFieldContainer[str]
+    def __init__(self, cluster_id: _Optional[str] = ..., connection_param: _Optional[_Union[ConnectionParam, _Mapping]] = ..., pchannels: _Optional[_Iterable[str]] = ...) -> None: ...
+
+class CrossClusterTopology(_message.Message):
+    __slots__ = ("source_cluster_id", "target_cluster_id")
+    SOURCE_CLUSTER_ID_FIELD_NUMBER: _ClassVar[int]
+    TARGET_CLUSTER_ID_FIELD_NUMBER: _ClassVar[int]
+    source_cluster_id: str
+    target_cluster_id: str
+    def __init__(self, source_cluster_id: _Optional[str] = ..., target_cluster_id: _Optional[str] = ...) -> None: ...
+
+class MessageID(_message.Message):
+    __slots__ = ("id", "WAL_name")
+    ID_FIELD_NUMBER: _ClassVar[int]
+    WAL_NAME_FIELD_NUMBER: _ClassVar[int]
+    id: str
+    WAL_name: WALName
+    def __init__(self, id: _Optional[str] = ..., WAL_name: _Optional[_Union[WALName, str]] = ...) -> None: ...
+
+class ImmutableMessage(_message.Message):
+    __slots__ = ("id", "payload", "properties")
+    class PropertiesEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
+    ID_FIELD_NUMBER: _ClassVar[int]
+    PAYLOAD_FIELD_NUMBER: _ClassVar[int]
+    PROPERTIES_FIELD_NUMBER: _ClassVar[int]
+    id: MessageID
+    payload: bytes
+    properties: _containers.ScalarMap[str, str]
+    def __init__(self, id: _Optional[_Union[MessageID, _Mapping]] = ..., payload: _Optional[bytes] = ..., properties: _Optional[_Mapping[str, str]] = ...) -> None: ...
+
+class ReplicateCheckpoint(_message.Message):
+    __slots__ = ("cluster_id", "pchannel", "message_id", "time_tick")
+    CLUSTER_ID_FIELD_NUMBER: _ClassVar[int]
+    PCHANNEL_FIELD_NUMBER: _ClassVar[int]
+    MESSAGE_ID_FIELD_NUMBER: _ClassVar[int]
+    TIME_TICK_FIELD_NUMBER: _ClassVar[int]
+    cluster_id: str
+    pchannel: str
+    message_id: MessageID
+    time_tick: int
+    def __init__(self, cluster_id: _Optional[str] = ..., pchannel: _Optional[str] = ..., message_id: _Optional[_Union[MessageID, _Mapping]] = ..., time_tick: _Optional[int] = ...) -> None: ...

pymilvus/grpc_gen/common_pb2.pyi

@@ -752,7 +752,7 @@ class DropIndexRequest(_message.Message):
     def __init__(self, base: _Optional[_Union[_common_pb2.MsgBase, _Mapping]] = ..., db_name: _Optional[str] = ..., collection_name: _Optional[str] = ..., field_name: _Optional[str] = ..., index_name: _Optional[str] = ...) -> None: ...
 
 class InsertRequest(_message.Message):
-    __slots__ = ("base", "db_name", "collection_name", "partition_name", "fields_data", "hash_keys", "num_rows", "schema_timestamp")
+    __slots__ = ("base", "db_name", "collection_name", "partition_name", "fields_data", "hash_keys", "num_rows", "schema_timestamp", "namespace")
     BASE_FIELD_NUMBER: _ClassVar[int]
     DB_NAME_FIELD_NUMBER: _ClassVar[int]
     COLLECTION_NAME_FIELD_NUMBER: _ClassVar[int]
@@ -761,6 +761,7 @@ class InsertRequest(_message.Message):
     HASH_KEYS_FIELD_NUMBER: _ClassVar[int]
     NUM_ROWS_FIELD_NUMBER: _ClassVar[int]
     SCHEMA_TIMESTAMP_FIELD_NUMBER: _ClassVar[int]
+    NAMESPACE_FIELD_NUMBER: _ClassVar[int]
     base: _common_pb2.MsgBase
     db_name: str
     collection_name: str
@@ -769,7 +770,8 @@ class InsertRequest(_message.Message):
     hash_keys: _containers.RepeatedScalarFieldContainer[int]
     num_rows: int
     schema_timestamp: int
-    def __init__(self, base: _Optional[_Union[_common_pb2.MsgBase, _Mapping]] = ..., db_name: _Optional[str] = ..., collection_name: _Optional[str] = ..., partition_name: _Optional[str] = ..., fields_data: _Optional[_Iterable[_Union[_schema_pb2.FieldData, _Mapping]]] = ..., hash_keys: _Optional[_Iterable[int]] = ..., num_rows: _Optional[int] = ..., schema_timestamp: _Optional[int] = ...) -> None: ...
+    namespace: str
+    def __init__(self, base: _Optional[_Union[_common_pb2.MsgBase, _Mapping]] = ..., db_name: _Optional[str] = ..., collection_name: _Optional[str] = ..., partition_name: _Optional[str] = ..., fields_data: _Optional[_Iterable[_Union[_schema_pb2.FieldData, _Mapping]]] = ..., hash_keys: _Optional[_Iterable[int]] = ..., num_rows: _Optional[int] = ..., schema_timestamp: _Optional[int] = ..., namespace: _Optional[str] = ...) -> None: ...
 
 class AddCollectionFieldRequest(_message.Message):
     __slots__ = ("base", "db_name", "collection_name", "collectionID", "schema")
@@ -786,7 +788,7 @@ class AddCollectionFieldRequest(_message.Message):
     def __init__(self, base: _Optional[_Union[_common_pb2.MsgBase, _Mapping]] = ..., db_name: _Optional[str] = ..., collection_name: _Optional[str] = ..., collectionID: _Optional[int] = ..., schema: _Optional[bytes] = ...) -> None: ...
 
 class UpsertRequest(_message.Message):
-    __slots__ = ("base", "db_name", "collection_name", "partition_name", "fields_data", "hash_keys", "num_rows", "schema_timestamp", "partial_update")
+    __slots__ = ("base", "db_name", "collection_name", "partition_name", "fields_data", "hash_keys", "num_rows", "schema_timestamp", "partial_update", "namespace")
     BASE_FIELD_NUMBER: _ClassVar[int]
     DB_NAME_FIELD_NUMBER: _ClassVar[int]
     COLLECTION_NAME_FIELD_NUMBER: _ClassVar[int]
@@ -796,6 +798,7 @@ class UpsertRequest(_message.Message):
     NUM_ROWS_FIELD_NUMBER: _ClassVar[int]
     SCHEMA_TIMESTAMP_FIELD_NUMBER: _ClassVar[int]
     PARTIAL_UPDATE_FIELD_NUMBER: _ClassVar[int]
+    NAMESPACE_FIELD_NUMBER: _ClassVar[int]
     base: _common_pb2.MsgBase
     db_name: str
     collection_name: str
@@ -805,7 +808,8 @@ class UpsertRequest(_message.Message):
     num_rows: int
     schema_timestamp: int
     partial_update: bool
-    def __init__(self, base: _Optional[_Union[_common_pb2.MsgBase, _Mapping]] = ..., db_name: _Optional[str] = ..., collection_name: _Optional[str] = ..., partition_name: _Optional[str] = ..., fields_data: _Optional[_Iterable[_Union[_schema_pb2.FieldData, _Mapping]]] = ..., hash_keys: _Optional[_Iterable[int]] = ..., num_rows: _Optional[int] = ..., schema_timestamp: _Optional[int] = ..., partial_update: bool = ...) -> None: ...
+    namespace: str
+    def __init__(self, base: _Optional[_Union[_common_pb2.MsgBase, _Mapping]] = ..., db_name: _Optional[str] = ..., collection_name: _Optional[str] = ..., partition_name: _Optional[str] = ..., fields_data: _Optional[_Iterable[_Union[_schema_pb2.FieldData, _Mapping]]] = ..., hash_keys: _Optional[_Iterable[int]] = ..., num_rows: _Optional[int] = ..., schema_timestamp: _Optional[int] = ..., partial_update: bool = ..., namespace: _Optional[str] = ...) -> None: ...
 
 class MutationResult(_message.Message):
     __slots__ = ("status", "IDs", "succ_index", "err_index", "acknowledged", "insert_cnt", "delete_cnt", "upsert_cnt", "timestamp")
@@ -2542,3 +2546,47 @@ class ListRowPoliciesResponse(_message.Message):
     db_name: str
     collection_name: str
     def __init__(self, status: _Optional[_Union[_common_pb2.Status, _Mapping]] = ..., policies: _Optional[_Iterable[_Union[RowPolicy, _Mapping]]] = ..., db_name: _Optional[str] = ..., collection_name: _Optional[str] = ...) -> None: ...
+
+class UpdateReplicateConfigurationRequest(_message.Message):
+    __slots__ = ("replicate_configuration",)
+    REPLICATE_CONFIGURATION_FIELD_NUMBER: _ClassVar[int]
+    replicate_configuration: _common_pb2.ReplicateConfiguration
+    def __init__(self, replicate_configuration: _Optional[_Union[_common_pb2.ReplicateConfiguration, _Mapping]] = ...) -> None: ...
+
+class GetReplicateInfoRequest(_message.Message):
+    __slots__ = ("source_cluster_id",)
+    SOURCE_CLUSTER_ID_FIELD_NUMBER: _ClassVar[int]
+    source_cluster_id: str
+    def __init__(self, source_cluster_id: _Optional[str] = ...) -> None: ...
+
+class GetReplicateInfoResponse(_message.Message):
+    __slots__ = ("checkpoints",)
+    CHECKPOINTS_FIELD_NUMBER: _ClassVar[int]
+    checkpoints: _containers.RepeatedCompositeFieldContainer[_common_pb2.ReplicateCheckpoint]
+    def __init__(self, checkpoints: _Optional[_Iterable[_Union[_common_pb2.ReplicateCheckpoint, _Mapping]]] = ...) -> None: ...
+
+class ReplicateMessage(_message.Message):
+    __slots__ = ("source_cluster_id", "message")
+    SOURCE_CLUSTER_ID_FIELD_NUMBER: _ClassVar[int]
+    MESSAGE_FIELD_NUMBER: _ClassVar[int]
+    source_cluster_id: str
+    message: _common_pb2.ImmutableMessage
+    def __init__(self, source_cluster_id: _Optional[str] = ..., message: _Optional[_Union[_common_pb2.ImmutableMessage, _Mapping]] = ...) -> None: ...
+
+class ReplicateRequest(_message.Message):
+    __slots__ = ("replicate_message",)
+    REPLICATE_MESSAGE_FIELD_NUMBER: _ClassVar[int]
+    replicate_message: ReplicateMessage
+    def __init__(self, replicate_message: _Optional[_Union[ReplicateMessage, _Mapping]] = ...) -> None: ...
+
+class ReplicateConfirmedMessageInfo(_message.Message):
+    __slots__ = ("confirmed_time_tick",)
+    CONFIRMED_TIME_TICK_FIELD_NUMBER: _ClassVar[int]
+    confirmed_time_tick: int
+    def __init__(self, confirmed_time_tick: _Optional[int] = ...) -> None: ...
+
+class ReplicateResponse(_message.Message):
+    __slots__ = ("replicate_confirmed_message_info",)
+    REPLICATE_CONFIRMED_MESSAGE_INFO_FIELD_NUMBER: _ClassVar[int]
+    replicate_confirmed_message_info: ReplicateConfirmedMessageInfo
+    def __init__(self, replicate_confirmed_message_info: _Optional[_Union[ReplicateConfirmedMessageInfo, _Mapping]] = ...) -> None: ...

pymilvus/grpc_gen/milvus-proto

@@ -571,6 +571,21 @@ def __init__(self, channel):
                 request_serializer=milvus__pb2.ListRowPoliciesRequest.SerializeToString,
                 response_deserializer=milvus__pb2.ListRowPoliciesResponse.FromString,
                 _registered_method=True)
+        self.UpdateReplicateConfiguration = channel.unary_unary(
+                '/milvus.proto.milvus.MilvusService/UpdateReplicateConfiguration',
+                request_serializer=milvus__pb2.UpdateReplicateConfigurationRequest.SerializeToString,
+                response_deserializer=common__pb2.Status.FromString,
+                _registered_method=True)
+        self.GetReplicateInfo = channel.unary_unary(
+                '/milvus.proto.milvus.MilvusService/GetReplicateInfo',
+                request_serializer=milvus__pb2.GetReplicateInfoRequest.SerializeToString,
+                response_deserializer=milvus__pb2.GetReplicateInfoResponse.FromString,
+                _registered_method=True)
+        self.CreateReplicateStream = channel.stream_stream(
+                '/milvus.proto.milvus.MilvusService/CreateReplicateStream',
+                request_serializer=milvus__pb2.ReplicateRequest.SerializeToString,
+                response_deserializer=milvus__pb2.ReplicateResponse.FromString,
+                _registered_method=True)
 
 
 class MilvusServiceServicer(object):
@@ -1118,7 +1133,8 @@ def DescribeDatabase(self, request, context):
         raise NotImplementedError('Method not implemented!')
 
     def ReplicateMessage(self, request, context):
-        """Missing associated documentation comment in .proto file."""
+        """Deprecated CDC API
+        """
         context.set_code(grpc.StatusCode.UNIMPLEMENTED)
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
@@ -1226,6 +1242,45 @@ def ListRowPolicies(self, request, context):
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
+    def UpdateReplicateConfiguration(self, request, context):
+        """CDC v2 APIs
+        UpdateReplicateConfiguration applies a full replacement of the current
+        replication configuration across Milvus clusters.
+
+        Semantics:
+        - The provided ReplicateConfiguration completely replaces any existing
+        configuration persisted in the metadata store.
+        - Passing an empty ReplicateConfiguration is treated as a "clear"
+        operation, effectively removing all replication configuration.
+        - The RPC is expected to be idempotent: submitting the same configuration
+        multiple times must not cause side effects.
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def GetReplicateInfo(self, request, context):
+        """
+        GetReplicateInfo retrieves replication-related metadata from a target Milvus cluster.
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
+    def CreateReplicateStream(self, request_iterator, context):
+        """
+        CreateReplicateStream establishes a replication stream on the target Milvus cluster.
+
+        Semantics:
+        - Sets up a continuous data stream that receives replicated messages
+        (DDL, insert, delete, etc.) from the source cluster via CDC.
+        - Once established, the target cluster persists incoming messages into
+        its WAL (Write-Ahead Log) ensuring durability and consistency.
+        """
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details('Method not implemented!')
+        raise NotImplementedError('Method not implemented!')
+
 
 def add_MilvusServiceServicer_to_server(servicer, server):
     rpc_method_handlers = {
@@ -1764,6 +1819,21 @@ def add_MilvusServiceServicer_to_server(servicer, server):
                     request_deserializer=milvus__pb2.ListRowPoliciesRequest.FromString,
                     response_serializer=milvus__pb2.ListRowPoliciesResponse.SerializeToString,
             ),
+            'UpdateReplicateConfiguration': grpc.unary_unary_rpc_method_handler(
+                    servicer.UpdateReplicateConfiguration,
+                    request_deserializer=milvus__pb2.UpdateReplicateConfigurationRequest.FromString,
+                    response_serializer=common__pb2.Status.SerializeToString,
+            ),
+            'GetReplicateInfo': grpc.unary_unary_rpc_method_handler(
+                    servicer.GetReplicateInfo,
+                    request_deserializer=milvus__pb2.GetReplicateInfoRequest.FromString,
+                    response_serializer=milvus__pb2.GetReplicateInfoResponse.SerializeToString,
+            ),
+            'CreateReplicateStream': grpc.stream_stream_rpc_method_handler(
+                    servicer.CreateReplicateStream,
+                    request_deserializer=milvus__pb2.ReplicateRequest.FromString,
+                    response_serializer=milvus__pb2.ReplicateResponse.SerializeToString,
+            ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
             'milvus.proto.milvus.MilvusService', rpc_method_handlers)
@@ -4664,6 +4734,87 @@ def ListRowPolicies(request,
             metadata,
             _registered_method=True)
 
+    @staticmethod
+    def UpdateReplicateConfiguration(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            '/milvus.proto.milvus.MilvusService/UpdateReplicateConfiguration',
+            milvus__pb2.UpdateReplicateConfigurationRequest.SerializeToString,
+            common__pb2.Status.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)
+
+    @staticmethod
+    def GetReplicateInfo(request,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            '/milvus.proto.milvus.MilvusService/GetReplicateInfo',
+            milvus__pb2.GetReplicateInfoRequest.SerializeToString,
+            milvus__pb2.GetReplicateInfoResponse.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)
+
+    @staticmethod
+    def CreateReplicateStream(request_iterator,
+            target,
+            options=(),
+            channel_credentials=None,
+            call_credentials=None,
+            insecure=False,
+            compression=None,
+            wait_for_ready=None,
+            timeout=None,
+            metadata=None):
+        return grpc.experimental.stream_stream(
+            request_iterator,
+            target,
+            '/milvus.proto.milvus.MilvusService/CreateReplicateStream',
+            milvus__pb2.ReplicateRequest.SerializeToString,
+            milvus__pb2.ReplicateResponse.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True)
+
 
 class ProxyServiceStub(object):
     """Missing associated documentation comment in .proto file."""