From 72d9d5d847d18edf0694c085a6d15b849c728252 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sat, 13 Aug 2022 20:19:21 -0700 Subject: [PATCH 01/12] YARN-9708. Yarn Router Support DelegationToken. --- .../hadoop-yarn-server-common/pom.xml | 3 + .../FederationDelegationTokenStateStore.java | 34 ++++ .../store/FederationStateStore.java | 3 +- .../impl/MemoryFederationStateStore.java | 86 +++++++++ .../store/impl/SQLFederationStateStore.java | 24 +++ .../impl/ZookeeperFederationStateStore.java | 24 +++ .../GetRouterDelegationTokenRequest.java | 50 ++++++ .../GetRouterDelegationTokenResponse.java | 62 +++++++ .../records/RemoveStoredMasterKeyRequest.java | 42 +++++ .../RemoveStoredMasterKeyResponse.java | 44 +++++ .../store/records/RouterMasterKey.java | 127 ++++++++++++++ .../records/RouterRMDTSecretManagerState.java | 35 ++++ .../records/RouterStoreNewTokenRequest.java | 27 +++ .../records/RouterStoreNewTokenResponse.java | 27 +++ .../store/records/RouterStoreToken.java | 81 +++++++++ .../records/StoreNewMasterKeyRequest.java | 43 +++++ .../records/StoreNewMasterKeyResponse.java | 45 +++++ .../pb/RemoveNewMasterKeyRequestPBImpl.java | 130 ++++++++++++++ .../impl/pb/RouterMasterKeyPBImpl.java | 116 +++++++++++++ .../impl/pb/RouterStoreTokenPBImpl.java | 102 +++++++++++ .../pb/StoreNewMasterKeyRequestPBImpl.java | 130 ++++++++++++++ .../utils/FederationStateStoreFacade.java | 11 ++ .../proto/yarn_server_federation_protos.proto | 35 +++- .../FederationStateStoreService.java | 24 +++ .../RouterDelegationTokenSecretManager.java | 164 ++++++++++++++++++ .../server/router/security/package-info.java | 20 +++ 26 files changed, 1487 insertions(+), 2 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveNewMasterKeyRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml index 3bf70bf7560e5..cecfd1e3ed67d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml @@ -196,6 +196,9 @@ ${basedir}/../../hadoop-yarn-api/src/main/proto + + ${basedir}/../../hadoop-yarn-common/src/main/proto + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java new file mode 100644 index 0000000000000..34e6a0f70109d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.yarn.server.federation.store; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.federation.store.records.*; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public interface FederationDelegationTokenStateStore { + + StoreNewMasterKeyResponse storeNewMasterKey(StoreNewMasterKeyRequest request) throws Exception; + + RemoveStoredMasterKeyResponse removeStoredMasterKey(RemoveStoredMasterKeyRequest request) + throws Exception; + + RouterStoreNewTokenResponse storeNewToken(RouterStoreNewTokenRequest request) throws Exception; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java index 67461e6c30b86..3ca8ccc2bfb69 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationStateStore.java @@ -31,7 +31,8 @@ */ public interface FederationStateStore extends FederationApplicationHomeSubClusterStore, FederationMembershipStateStore, - FederationPolicyStore, FederationReservationHomeSubClusterStore { + FederationPolicyStore, FederationReservationHomeSubClusterStore, + FederationDelegationTokenStateStore { /** * Initialize the FederationStore. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java index 24ed54d63668b..c95b9a2f42734 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java @@ -17,15 +17,20 @@ package org.apache.hadoop.yarn.server.federation.store.impl; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Calendar; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import java.util.TimeZone; import java.util.concurrent.ConcurrentHashMap; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ReservationId; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -66,7 +71,16 @@ import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMDTSecretManagerState; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; import org.apache.hadoop.yarn.server.federation.store.utils.FederationApplicationHomeSubClusterStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationReservationHomeSubClusterStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator; @@ -86,6 +100,7 @@ public class MemoryFederationStateStore implements FederationStateStore { private Map applications; private Map reservations; private Map policies; + private RouterRMDTSecretManagerState routerRMSecretManagerState; private final MonotonicClock clock = new MonotonicClock(); @@ -98,6 +113,7 @@ public void init(Configuration conf) { applications = new ConcurrentHashMap(); reservations = new ConcurrentHashMap(); policies = new ConcurrentHashMap(); + routerRMSecretManagerState = new RouterRMDTSecretManagerState(); } @Override @@ -365,4 +381,74 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( return GetReservationsHomeSubClusterResponse.newInstance(result); } + + @Override + public StoreNewMasterKeyResponse storeNewMasterKey( + StoreNewMasterKeyRequest request) throws Exception { + + // Restore the DelegationKey from the request + RouterMasterKey masterKey = request.getRouterMasterKey(); + ByteBuffer keyByteBuf = masterKey.getKeyBytes(); + byte[] keyBytes = new byte[keyByteBuf.remaining()]; + keyByteBuf.get(keyBytes); + DelegationKey delegationKey = + new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes); + + Set rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); + if (rmDTMasterKeyState.contains(delegationKey)) { + LOG.info("Error storing info for RMDTMasterKey with keyID: {}.", delegationKey.getKeyId()); + throw new IOException("RMDTMasterKey with keyID: " + + delegationKey.getKeyId() + " is already stored"); + } + + routerRMSecretManagerState.getMasterKeyState().add(delegationKey); + LOG.info("Store Router-RMDT master key with key id: {}. Currently rmDTMasterKeyState size: {}", + delegationKey.getKeyId(), rmDTMasterKeyState.size()); + + return StoreNewMasterKeyResponse.newInstance(masterKey); + } + + @Override + public RemoveStoredMasterKeyResponse removeStoredMasterKey( + RemoveStoredMasterKeyRequest request) throws Exception { + + // Restore the DelegationKey from the request + RouterMasterKey masterKey = request.getRouterMasterKey(); + ByteBuffer keyByteBuf = masterKey.getKeyBytes(); + byte[] keyBytes = new byte[keyByteBuf.remaining()]; + keyByteBuf.get(keyBytes); + DelegationKey delegationKey = + new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes); + + LOG.info("Remove Router-RMDT master key with key id: {}.", delegationKey.getKeyId()); + Set rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); + rmDTMasterKeyState.remove(delegationKey); + + return RemoveStoredMasterKeyResponse.newInstance(masterKey); + } + + @Override + public RouterStoreNewTokenResponse storeNewToken( + RouterStoreNewTokenRequest request) throws Exception { + RouterStoreToken storeToken = request.getRouterStoreToken(); + RMDelegationTokenIdentifier tokenIdentifier = storeToken.getTokenIdentifier(); + Long renewDate = storeToken.getRenewDate(); + storeOrUpdateRouterRMDT(tokenIdentifier, renewDate, false); + return RouterStoreNewTokenResponse.newInstance(storeToken); + } + + private void storeOrUpdateRouterRMDT(RMDelegationTokenIdentifier rmDTIdentifier, + Long renewDate, boolean isUpdate) throws Exception { + Map rmDTState = routerRMSecretManagerState.getTokenState(); + if (rmDTState.containsKey(rmDTIdentifier)) { + LOG.info("Error storing info for RMDelegationToken: {}.", rmDTIdentifier); + throw new IOException("RMDelegationToken: " + rmDTIdentifier + "is already stored."); + } + rmDTState.put(rmDTIdentifier, renewDate); + if(!isUpdate) { + routerRMSecretManagerState.setDtSequenceNumber(rmDTIdentifier.getSequenceNumber()); + } + LOG.info("Store RM-RMDT with sequence number {}.", rmDTIdentifier.getSequenceNumber()); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java index dfcfb06bb46b8..97148dc0bc46c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java @@ -74,6 +74,12 @@ import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; import org.apache.hadoop.yarn.server.federation.store.utils.FederationApplicationHomeSubClusterStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator; @@ -1027,4 +1033,22 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( GetReservationsHomeSubClusterRequest request) throws YarnException { throw new NotImplementedException("Code is not implemented"); } + + @Override + public StoreNewMasterKeyResponse storeNewMasterKey( + StoreNewMasterKeyRequest request) throws Exception{ + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RemoveStoredMasterKeyResponse removeStoredMasterKey( + RemoveStoredMasterKeyRequest request) throws Exception { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterStoreNewTokenResponse storeNewToken( + RouterStoreNewTokenRequest request) throws Exception { + throw new NotImplementedException("Code is not implemented"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java index 686ba077c7da5..b8d134f3b79e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java @@ -79,6 +79,12 @@ import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils; +import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; import org.apache.hadoop.yarn.server.records.Version; import org.apache.zookeeper.data.ACL; import org.slf4j.Logger; @@ -662,4 +668,22 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( GetReservationsHomeSubClusterRequest request) throws YarnException { throw new NotImplementedException("Code is not implemented"); } + + @Override + public StoreNewMasterKeyResponse storeNewMasterKey( + StoreNewMasterKeyRequest request) throws Exception{ + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RemoveStoredMasterKeyResponse removeStoredMasterKey( + RemoveStoredMasterKeyRequest request) throws Exception { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterStoreNewTokenResponse storeNewToken( + RouterStoreNewTokenRequest request) throws Exception { + throw new NotImplementedException("Code is not implemented"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenRequest.java new file mode 100644 index 0000000000000..faba59c322442 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenRequest.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.yarn.util.Records; + +/** + * The request issued by the client to get a delegation token from + * the {@code ResourceManager}. + * for more information. + */ +@Public +@Stable +public abstract class GetRouterDelegationTokenRequest { + + @Public + @Stable + public static GetRouterDelegationTokenRequest newInstance(String renewer) { + GetRouterDelegationTokenRequest request = + Records.newRecord(GetRouterDelegationTokenRequest.class); + request.setRenewer(renewer); + return request; + } + + @Public + @Stable + public abstract String getRenewer(); + + @Public + @Stable + public abstract void setRenewer(String renewer); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenResponse.java new file mode 100644 index 0000000000000..b9f304cc6909f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenResponse.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Stable; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; +import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; +import org.apache.hadoop.yarn.api.records.Token; +import org.apache.hadoop.yarn.util.Records; + + +/** + * Response to a {@link GetDelegationTokenRequest} request + * from the client. The response contains the token that + * can be used by the containers to talk to ClientRMService. + * + */ +@Public +@Stable +public abstract class GetRouterDelegationTokenResponse { + + @Private + @Unstable + public static GetRouterDelegationTokenResponse newInstance(Token rmDTToken) { + GetRouterDelegationTokenResponse response = + Records.newRecord(GetRouterDelegationTokenResponse.class); + response.setRMDelegationToken(rmDTToken); + return response; + } + + /** + * The Delegation tokens have a identifier which maps to + * {@link AbstractDelegationTokenIdentifier}. + * @return the delegation tokens + */ + @Public + @Stable + public abstract Token getRMDelegationToken(); + + @Private + @Unstable + public abstract void setRMDelegationToken(Token rmDTToken); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyRequest.java new file mode 100644 index 0000000000000..c44945e467dca --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyRequest.java @@ -0,0 +1,42 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RemoveStoredMasterKeyRequest { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RemoveStoredMasterKeyRequest newInstance(RouterMasterKey routerMasterKey) { + RemoveStoredMasterKeyRequest request = Records.newRecord(RemoveStoredMasterKeyRequest.class); + request.setRouterMasterKey(routerMasterKey); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterMasterKey getRouterMasterKey(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterMasterKey(RouterMasterKey routerMasterKey); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyResponse.java new file mode 100644 index 0000000000000..6ff44d3948b1f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyResponse.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RemoveStoredMasterKeyResponse { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RemoveStoredMasterKeyResponse newInstance(RouterMasterKey masterKey) { + RemoveStoredMasterKeyResponse response = Records.newRecord(RemoveStoredMasterKeyResponse.class); + response.setRouterMasterKey(masterKey); + return response; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterMasterKey getRouterMasterKey(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterMasterKey(RouterMasterKey routerMasterKey); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java new file mode 100644 index 0000000000000..b7c7f0a3ddf52 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +import java.nio.ByteBuffer; +import java.util.Arrays; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterMasterKey { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterMasterKey newInstance(Integer keyId, ByteBuffer keyBytes, Long expiryDate) { + RouterMasterKey policy = Records.newRecord(RouterMasterKey.class); + policy.setKeyId(keyId); + policy.setKeyBytes(keyBytes); + policy.setExpiryDate(expiryDate); + return policy; + } + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterMasterKey newInstance(RouterMasterKey masterKey) { + RouterMasterKey routerMasterKey = Records.newRecord(RouterMasterKey.class); + routerMasterKey.setKeyId(masterKey.getKeyId()); + routerMasterKey.setKeyBytes(masterKey.getKeyBytes()); + routerMasterKey.setExpiryDate(masterKey.getExpiryDate()); + return routerMasterKey; + } + + /** + * Get the keyId of the MasterKey. + * + * @return MasterKeyId. + */ + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract Integer getKeyId(); + + /** + * Set the keyId of the MasterKey. + * + * @param keyId MasterKeyId. + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setKeyId(Integer keyId); + + /** + * Get the keyBytes of the DelegationKey. + * + * @return KeyBytes of the DelegationKey. + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract ByteBuffer getKeyBytes(); + + /** + * Set the keyBytes of the DelegationKey. + * + * @param keyBytes KeyBytes of the DelegationKey. + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setKeyBytes(ByteBuffer keyBytes); + + /** + * Get the ExpiryDate of the DelegationKey. + * + * @return ExpiryDate of the DelegationKey. + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract Long getExpiryDate(); + + /** + * Set the expiryDate of the DelegationKey. + * + * @param expiryDate expiryDate of the DelegationKey. + */ + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setExpiryDate(Long expiryDate); + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (int) (getExpiryDate() ^ (getExpiryDate() >>> 32)); + result = prime * result + Arrays.hashCode(getKeyBytes().array()); + result = prime * result + getKeyId(); + return result; + } + + @Override + public boolean equals(Object right) { + if (this == right) { + return true; + } else if (right == null || getClass() != right.getClass()) { + return false; + } else { + RouterMasterKey r = (RouterMasterKey) right; + return getKeyId() == r.getKeyId() && getExpiryDate() == r.getExpiryDate() && + Arrays.equals(getKeyBytes().array(), r.getKeyBytes().array()); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java new file mode 100644 index 0000000000000..ecf39c16a77c8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java @@ -0,0 +1,35 @@ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class RouterRMDTSecretManagerState { + + // DTIdentifier -> renewDate + private Map delegationTokenState = new HashMap(); + + private Set masterKeyState = new HashSet(); + + private int dtSequenceNumber = 0; + + public Map getTokenState() { + return delegationTokenState; + } + + public Set getMasterKeyState() { + return masterKeyState; + } + + public int getDTSequenceNumber() { + return dtSequenceNumber; + } + + public void setDtSequenceNumber(int dtSequenceNumber) { + this.dtSequenceNumber = dtSequenceNumber; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenRequest.java new file mode 100644 index 0000000000000..0cbe02a156e90 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenRequest.java @@ -0,0 +1,27 @@ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterStoreNewTokenRequest { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterStoreNewTokenRequest newInstance(RouterStoreToken routerStoreToken) { + RouterStoreNewTokenRequest request = Records.newRecord(RouterStoreNewTokenRequest.class); + request.setRouterStoreToken(routerStoreToken); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterStoreToken getRouterStoreToken(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenResponse.java new file mode 100644 index 0000000000000..357773ae79e6d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenResponse.java @@ -0,0 +1,27 @@ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterStoreNewTokenResponse { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterStoreNewTokenResponse newInstance(RouterStoreToken routerStoreToken) { + RouterStoreNewTokenResponse request = Records.newRecord(RouterStoreNewTokenResponse.class); + request.setRouterStoreToken(routerStoreToken); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterStoreToken getRouterStoreToken(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java new file mode 100644 index 0000000000000..e77301bd92e74 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; +import org.apache.hadoop.yarn.util.Records; + +import java.io.ByteArrayInputStream; +import java.io.DataInput; +import java.io.DataInputStream; +import java.io.IOException; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterStoreToken { + private RouterStoreTokenProto.Builder builder = RouterStoreTokenProto.newBuilder(); + + public RouterStoreToken() {} + + public RouterStoreToken(YARNDelegationTokenIdentifier identifier, Long renewdate) { + builder.setTokenIdentifier(identifier.getProto()); + builder.setRenewDate(renewdate); + } + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterStoreToken newInstance(YARNDelegationTokenIdentifier identifier, + Long renewdate) { + RouterStoreToken storeToken = Records.newRecord(RouterStoreToken.class); + storeToken.setIdentifier(identifier); + storeToken.setRenewDate(renewdate); + return storeToken; + } + + public void readFields(DataInput in) throws IOException { + builder.mergeFrom((DataInputStream) in); + } + + public byte[] toByteArray() throws IOException { + return builder.build().toByteArray(); + } + + public RMDelegationTokenIdentifier getTokenIdentifier() throws IOException { + ByteArrayInputStream in = + new ByteArrayInputStream(builder.getTokenIdentifier().toByteArray()); + RMDelegationTokenIdentifier identifer = new RMDelegationTokenIdentifier(); + identifer.readFields(new DataInputStream(in)); + return identifer; + } + + public Long getRenewDate() { + return builder.getRenewDate(); + } + + public void setIdentifier(YARNDelegationTokenIdentifier identifier) { + builder.setTokenIdentifier(identifier.getProto()); + } + + public void setRenewDate(Long renewDate) { + builder.setRenewDate(renewDate); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyRequest.java new file mode 100644 index 0000000000000..d3dd7ba3f3d52 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyRequest.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class StoreNewMasterKeyRequest { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static StoreNewMasterKeyRequest newInstance(RouterMasterKey routerMasterKey) { + StoreNewMasterKeyRequest request = Records.newRecord(StoreNewMasterKeyRequest.class); + request.setRouterMasterKey(routerMasterKey); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterMasterKey getRouterMasterKey(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterMasterKey(RouterMasterKey routerMasterKey); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyResponse.java new file mode 100644 index 0000000000000..3c491bfc7342b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyResponse.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class StoreNewMasterKeyResponse { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static StoreNewMasterKeyResponse newInstance(RouterMasterKey masterKey) { + StoreNewMasterKeyResponse response = Records.newRecord(StoreNewMasterKeyResponse.class); + response.setRouterMasterKey(masterKey); + return response; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterMasterKey getRouterMasterKey(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterMasterKey(RouterMasterKey routerMasterKey); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveNewMasterKeyRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveNewMasterKeyRequestPBImpl.java new file mode 100644 index 0000000000000..9555a4874db9a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveNewMasterKeyRequestPBImpl.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyRequestProtoOrBuilder; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; +import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class RemoveNewMasterKeyRequestPBImpl extends StoreNewMasterKeyRequest { + + private StoreNewMasterKeyRequestProto proto = StoreNewMasterKeyRequestProto.getDefaultInstance(); + + private StoreNewMasterKeyRequestProto.Builder builder = null; + private boolean viaProto = false; + + private RouterMasterKey routerMasterKey = null; + + public RemoveNewMasterKeyRequestPBImpl() { + builder = StoreNewMasterKeyRequestProto.newBuilder(); + } + + public RemoveNewMasterKeyRequestPBImpl(StoreNewMasterKeyRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public StoreNewMasterKeyRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = StoreNewMasterKeyRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + RouterMasterKeyPBImpl masterKeyRequest = + (RouterMasterKeyPBImpl) this.routerMasterKey; + RouterMasterKeyProto routerMasterKeyProto = builder.getRouterMasterKey(); + if (this.routerMasterKey != null && !masterKeyRequest.getProto().equals(routerMasterKeyProto)) { + builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); + } + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public RouterMasterKey getRouterMasterKey() { + StoreNewMasterKeyRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerMasterKey != null) { + return this.routerMasterKey; + } + if (!p.hasRouterMasterKey()) { + return null; + } + this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey()); + return this.routerMasterKey; + } + + @Override + public void setRouterMasterKey(RouterMasterKey masterKey) { + maybeInitBuilder(); + if (masterKey == null) { + builder.clearRouterMasterKey(); + } + this.routerMasterKey = masterKey; + } + + private RouterMasterKey convertFromProtoFormat(RouterMasterKeyProto masterKeyProto) { + return new RouterMasterKeyPBImpl(masterKeyProto); + } + + private RouterMasterKeyProto convertToProtoFormat(RouterMasterKey masterKey) { + return ((RouterMasterKeyPBImpl) masterKey).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyPBImpl.java new file mode 100644 index 0000000000000..1388c51aef457 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyPBImpl.java @@ -0,0 +1,116 @@ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.thirdparty.protobuf.ByteString; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProtoOrBuilder; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; + +import java.nio.ByteBuffer; + +public class RouterMasterKeyPBImpl extends RouterMasterKey { + + private RouterMasterKeyProto proto = RouterMasterKeyProto.getDefaultInstance(); + private RouterMasterKeyProto.Builder builder = null; + private boolean viaProto = false; + + public RouterMasterKeyPBImpl() { + builder = RouterMasterKeyProto.newBuilder(); + } + + public RouterMasterKeyPBImpl(RouterMasterKeyProto masterKeyProto) { + this.proto = masterKeyProto; + viaProto = true; + } + + public RouterMasterKeyProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RouterMasterKeyProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public Integer getKeyId() { + RouterMasterKeyProtoOrBuilder p = viaProto ? proto : builder; + return p.getKeyId(); + } + + @Override + public void setKeyId(Integer keyId) { + maybeInitBuilder(); + if (keyId == null) { + builder.clearKeyId(); + return; + } + builder.setKeyId(keyId); + } + + @Override + public ByteBuffer getKeyBytes() { + RouterMasterKeyProtoOrBuilder p = viaProto ? proto : builder; + return convertFromProtoFormat(p.getKeyBytes()); + } + + @Override + public void setKeyBytes(ByteBuffer keyBytes) { + maybeInitBuilder(); + if (keyBytes == null) { + builder.clearKeyBytes(); + return; + } + builder.setKeyBytes(convertToProtoFormat(keyBytes)); + } + + @Override + public Long getExpiryDate() { + RouterMasterKeyProtoOrBuilder p = viaProto ? proto : builder; + return p.getExpiryDate(); + } + + @Override + public void setExpiryDate(Long expiryDate) { + maybeInitBuilder(); + if (expiryDate == null) { + builder.clearExpiryDate(); + return; + } + builder.setExpiryDate(expiryDate); + } + + protected final ByteBuffer convertFromProtoFormat(ByteString byteString) { + return ProtoUtils.convertFromProtoFormat(byteString); + } + + protected final ByteString convertToProtoFormat(ByteBuffer byteBuffer) { + return ProtoUtils.convertToProtoFormat(byteBuffer); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java new file mode 100644 index 0000000000000..3c07a50cf3d83 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java @@ -0,0 +1,102 @@ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProtoOrBuilder; + +import java.io.ByteArrayInputStream; +import java.io.DataInputStream; +import java.io.IOException; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class RouterStoreTokenPBImpl extends RouterStoreToken { + + private RouterStoreTokenProto proto = RouterStoreTokenProto.getDefaultInstance(); + + private RouterStoreTokenProto.Builder builder = null; + + private boolean viaProto = false; + + public RouterStoreTokenPBImpl() { + builder = RouterStoreTokenProto.newBuilder(); + } + + public RouterStoreTokenPBImpl(RouterStoreTokenProto storeTokenProto) { + this.proto = storeTokenProto; + viaProto = true; + } + + public RouterStoreTokenProto getProto() { + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RouterStoreTokenProto.newBuilder(proto); + } + viaProto = false; + } + + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public RMDelegationTokenIdentifier getTokenIdentifier() throws IOException { + RouterStoreTokenProtoOrBuilder p = viaProto ? proto : builder; + YARNDelegationTokenIdentifierProto identifierProto = p.getTokenIdentifier(); + ByteArrayInputStream in = new ByteArrayInputStream(identifierProto.toByteArray()); + RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(); + identifier.readFields(new DataInputStream(in)); + return identifier; + } + + @Override + public Long getRenewDate() { + RouterStoreTokenProtoOrBuilder p = viaProto ? proto : builder; + return p.getRenewDate(); + } + + @Override + public void setIdentifier(YARNDelegationTokenIdentifier identifier) { + maybeInitBuilder(); + if(identifier == null) { + builder.clearTokenIdentifier(); + } + builder.setTokenIdentifier(identifier.getProto()); + } + + @Override + public void setRenewDate(Long renewDate) { + maybeInitBuilder(); + if(renewDate == null) { + builder.clearRenewDate(); + } + builder.setRenewDate(renewDate); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyRequestPBImpl.java new file mode 100644 index 0000000000000..7f9b63c874de9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyRequestPBImpl.java @@ -0,0 +1,130 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; +import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyRequestProtoOrBuilder; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class StoreNewMasterKeyRequestPBImpl extends StoreNewMasterKeyRequest { + + private StoreNewMasterKeyRequestProto proto = StoreNewMasterKeyRequestProto.getDefaultInstance(); + + private StoreNewMasterKeyRequestProto.Builder builder = null; + private boolean viaProto = false; + + private RouterMasterKey routerMasterKey = null; + + public StoreNewMasterKeyRequestPBImpl() { + builder = StoreNewMasterKeyRequestProto.newBuilder(); + } + + public StoreNewMasterKeyRequestPBImpl(StoreNewMasterKeyRequestProto proto) { + this.proto = proto; + viaProto = true; + } + + public StoreNewMasterKeyRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = StoreNewMasterKeyRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + RouterMasterKeyPBImpl masterKeyRequest = + (RouterMasterKeyPBImpl) this.routerMasterKey; + RouterMasterKeyProto routerMasterKeyProto = builder.getRouterMasterKey(); + if (this.routerMasterKey != null && !masterKeyRequest.getProto().equals(routerMasterKeyProto)) { + builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); + } + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public RouterMasterKey getRouterMasterKey() { + StoreNewMasterKeyRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerMasterKey != null) { + return this.routerMasterKey; + } + if (!p.hasRouterMasterKey()) { + return null; + } + this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey()); + return this.routerMasterKey; + } + + @Override + public void setRouterMasterKey(RouterMasterKey masterKey) { + maybeInitBuilder(); + if (masterKey == null) { + builder.clearRouterMasterKey(); + } + this.routerMasterKey = masterKey; + } + + private RouterMasterKey convertFromProtoFormat(RouterMasterKeyProto masterKeyProto) { + return new RouterMasterKeyPBImpl(masterKeyProto); + } + + private RouterMasterKeyProto convertToProtoFormat(RouterMasterKey masterKey) { + return ((RouterMasterKeyPBImpl) masterKey).getProto(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index 56aeba8144057..51c8f48aa9994 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -41,6 +41,7 @@ import org.apache.hadoop.io.retry.RetryPolicies; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.io.retry.RetryProxy; +import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ReservationId; @@ -388,6 +389,16 @@ public SubClusterId getApplicationHomeSubCluster(ApplicationId appId) return response.getApplicationHomeSubCluster().getHomeSubCluster(); } + /** + * The Router supports saving the master key + * + * @param newKey DelegationKey + */ + public void storeNewMasterKey(DelegationKey newKey) { + LOG.info("Storing master key with keyID {}.", newKey.getKeyId()); + + } + /** * Get the singleton instance of SubClusterResolver. * diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto index 33f5cb3fc14e0..bd12bf63be6a3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto @@ -25,6 +25,8 @@ package hadoop.yarn; import "yarn_protos.proto"; import "yarn_server_common_protos.proto"; +import "yarn_security_token.proto"; +import "RpcHeader.proto"; message SubClusterIdProto { optional string id = 1; @@ -213,4 +215,35 @@ message GetSubClusterPoliciesConfigurationsRequestProto { message GetSubClusterPoliciesConfigurationsResponseProto { repeated SubClusterPolicyConfigurationProto policies_configurations = 1; - } +} + +message RouterMasterKeyProto { + optional int32 key_id = 1; + optional bytes key_bytes = 2; + optional uint64 expiry_date = 3; +} + +message StoreNewMasterKeyRequestProto { + optional RouterMasterKeyProto router_master_key = 1; +} + +message StoreNewMasterKeyResponseProto { + optional RouterMasterKeyProto router_master_key = 1; +} + +message RemoveStoredMasterKeyRequestProto { + optional RouterMasterKeyProto router_master_key = 1; +} + +message RemoveStoredMasterKeyResponseProto { + optional RouterMasterKeyProto router_master_key = 1; +} + +message RouterStoreTokenProto { + optional YARNDelegationTokenIdentifierProto token_identifier = 1; + optional int64 renew_date = 2; +} + +message RouterStoreNewTokenRequestProto { + optional RouterStoreTokenProto router_store_token = 1; +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java index d2d69231f5ef6..89341344da6b3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java @@ -66,6 +66,12 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -325,4 +331,22 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( GetReservationsHomeSubClusterRequest request) throws YarnException { return stateStoreClient.getReservationsHomeSubCluster(request); } + + @Override + public StoreNewMasterKeyResponse storeNewMasterKey( + StoreNewMasterKeyRequest request) throws Exception { + return stateStoreClient.storeNewMasterKey(request); + } + + @Override + public RemoveStoredMasterKeyResponse removeStoredMasterKey( + RemoveStoredMasterKeyRequest request) throws Exception { + return stateStoreClient.removeStoredMasterKey(request); + } + + @Override + public RouterStoreNewTokenResponse storeNewToken( + RouterStoreNewTokenRequest request) throws Exception { + return stateStoreClient.storeNewToken(request); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java new file mode 100644 index 0000000000000..3732d201a6aef --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java @@ -0,0 +1,164 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.security; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.VisibleForTesting; +import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; +import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.federation.store.records.*; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * A Router specific delegation token secret manager. + * The secret manager is responsible for generating and accepting the password + * for each token. + */ +public class RouterDelegationTokenSecretManager + extends AbstractDelegationTokenSecretManager { + + private static final Logger LOG = LoggerFactory + .getLogger(RouterDelegationTokenSecretManager.class); + + private FederationStateStoreFacade federationFacade; + + /** + * Create a secret manager. + * + * @param delegationKeyUpdateInterval the number of milliseconds for rolling + * new secret keys. + * @param delegationTokenMaxLifetime the maximum lifetime of the delegation + * tokens in milliseconds + * @param delegationTokenRenewInterval how often the tokens must be renewed + * in milliseconds + * @param delegationTokenRemoverScanInterval how often the tokens are scanned + */ + public RouterDelegationTokenSecretManager(long delegationKeyUpdateInterval, + long delegationTokenMaxLifetime, + long delegationTokenRenewInterval, + long delegationTokenRemoverScanInterval) { + super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, + delegationTokenRenewInterval, delegationTokenRemoverScanInterval); + federationFacade = FederationStateStoreFacade.getInstance(); + } + + @Override + public RMDelegationTokenIdentifier createIdentifier() { + return new RMDelegationTokenIdentifier(); + } + + private boolean shouldIgnoreException(Exception e) { + return !running && e.getCause() instanceof InterruptedException; + } + + @Override + protected void storeNewMasterKey(DelegationKey newKey) { + try { + LOG.info("Storing master key with keyID {}.", newKey.getKeyId()); + ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey()); + RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(), + keyBytes, newKey.getExpiryDate()); + StoreNewMasterKeyRequest keyRequest = StoreNewMasterKeyRequest.newInstance(masterKey); + federationFacade.getStateStore().storeNewMasterKey(keyRequest); + } catch (Exception e) { + if (!shouldIgnoreException(e)) { + LOG.error("Error in storing master key with KeyID: {}.", newKey.getKeyId()); + ExitUtil.terminate(1, e); + } + } + } + + @Override + protected void removeStoredMasterKey(DelegationKey key) { + try { + LOG.info("removing master key with keyID {}.", key.getKeyId()); + ByteBuffer keyBytes = ByteBuffer.wrap(key.getEncodedKey()); + RouterMasterKey masterKey = RouterMasterKey.newInstance(key.getKeyId(), + keyBytes, key.getExpiryDate()); + RemoveStoredMasterKeyRequest keyRequest = RemoveStoredMasterKeyRequest.newInstance(masterKey); + federationFacade.getStateStore().removeStoredMasterKey(keyRequest); + } catch (Exception e) { + if (!shouldIgnoreException(e)) { + LOG.error("Error in removing master key with KeyID: {}.", key.getKeyId()); + ExitUtil.terminate(1, e); + } + } + } + + @Override + protected void storeNewToken(RMDelegationTokenIdentifier identifier, + long renewDate) throws IOException { + try { + LOG.info("storing RMDelegation token with sequence number: {}.", + identifier.getSequenceNumber()); + RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate); + RouterStoreNewTokenRequest request = RouterStoreNewTokenRequest.newInstance(storeToken); + federationFacade.getStateStore().storeNewToken(request); + } catch (Exception e) { + if (!shouldIgnoreException(e)) { + LOG.error("Error in storing RMDelegationToken with sequence number: {}.", + identifier.getSequenceNumber()); + ExitUtil.terminate(1, e); + } + } + } + + @Override + protected void updateStoredToken(RMDelegationTokenIdentifier id, long renewDate) { + try { + LOG.info("updating RMDelegation token with sequence number: {}.", + id.getSequenceNumber()); + // rm.getRMContext().getStateStore().updateRMDelegationToken(id, renewDate); + } catch (Exception e) { + if (!shouldIgnoreException(e)) { + LOG.error("Error in updating persisted RMDelegationToken with sequence number: {}.", + id.getSequenceNumber()); + ExitUtil.terminate(1, e); + } + } + } + + @Override + protected void removeStoredToken(RMDelegationTokenIdentifier + identifier) throws IOException { + try { + LOG.info("removing RMDelegation token with sequence number: {}", + identifier.getSequenceNumber()); + // rm.getRMContext().getStateStore().removeRMDelegationToken(ident); + } catch (Exception e) { + if (!shouldIgnoreException(e)) { + LOG.error("Error in removing RMDelegationToken with sequence number: {}", + identifier.getSequenceNumber()); + ExitUtil.terminate(1, e); + } + } + } + + @InterfaceAudience.Private + @VisibleForTesting + public int getLatestDTSequenceNumber() { + return delegationTokenSequenceNumber; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java new file mode 100644 index 0000000000000..b29c029c36d03 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Router WebApp package. **/ +package org.apache.hadoop.yarn.server.router.security; From 25228878a8cabbf0ec33155ed9d7f1b3624c220c Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sat, 13 Aug 2022 21:35:17 -0700 Subject: [PATCH 02/12] YARN-9708. Yarn Router Support DelegationToken. --- .../FederationDelegationTokenStateStore.java | 18 +++- .../impl/MemoryFederationStateStore.java | 24 +++++ .../store/impl/SQLFederationStateStore.java | 16 ++++ .../impl/ZookeeperFederationStateStore.java | 16 ++++ .../RouterRemoveStoredTokenRequest.java | 28 ++++++ .../RouterRemoveStoredTokenResponse.java | 27 ++++++ .../RouterUpdateStoredTokenRequest.java | 28 ++++++ .../RouterUpdateStoredTokenResponse.java | 27 ++++++ .../utils/FederationStateStoreFacade.java | 90 ++++++++++++------- .../FederationStateStoreService.java | 16 ++++ .../RouterDelegationTokenSecretManager.java | 30 ++----- 11 files changed, 261 insertions(+), 59 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenResponse.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java index 34e6a0f70109d..5e553eee04437 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java @@ -18,8 +18,16 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; -import org.apache.hadoop.yarn.server.federation.store.records.*; +import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenResponse; @InterfaceAudience.Private @InterfaceStability.Unstable @@ -31,4 +39,10 @@ RemoveStoredMasterKeyResponse removeStoredMasterKey(RemoveStoredMasterKeyRequest throws Exception; RouterStoreNewTokenResponse storeNewToken(RouterStoreNewTokenRequest request) throws Exception; + + RouterUpdateStoredTokenResponse updateStoredToken(RouterUpdateStoredTokenRequest request) + throws Exception; + + RouterRemoveStoredTokenResponse removeStoredToken(RouterRemoveStoredTokenRequest request) + throws Exception; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java index c95b9a2f42734..2aa403f231e92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java @@ -86,6 +86,10 @@ import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils; +import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenResponse; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.util.MonotonicClock; import org.slf4j.Logger; @@ -437,6 +441,26 @@ public RouterStoreNewTokenResponse storeNewToken( return RouterStoreNewTokenResponse.newInstance(storeToken); } + @Override + public RouterUpdateStoredTokenResponse updateStoredToken( + RouterUpdateStoredTokenRequest request) throws Exception { + RouterStoreToken storeToken = request.getRouterStoreToken(); + RMDelegationTokenIdentifier tokenIdentifier = storeToken.getTokenIdentifier(); + Long renewDate = storeToken.getRenewDate(); + storeOrUpdateRouterRMDT(tokenIdentifier, renewDate, true); + return RouterUpdateStoredTokenResponse.newInstance(storeToken); + } + + @Override + public RouterRemoveStoredTokenResponse removeStoredToken( + RouterRemoveStoredTokenRequest request) throws Exception { + RouterStoreToken storeToken = request.getRouterStoreToken(); + RMDelegationTokenIdentifier tokenIdentifier = storeToken.getTokenIdentifier(); + Map rmDTState = routerRMSecretManagerState.getTokenState(); + rmDTState.remove(tokenIdentifier); + return RouterRemoveStoredTokenResponse.newInstance(storeToken); + } + private void storeOrUpdateRouterRMDT(RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate, boolean isUpdate) throws Exception { Map rmDTState = routerRMSecretManagerState.getTokenState(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java index 97148dc0bc46c..d02f8562cc09a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java @@ -84,6 +84,10 @@ import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils; +import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenResponse; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.MonotonicClock; @@ -1051,4 +1055,16 @@ public RouterStoreNewTokenResponse storeNewToken( RouterStoreNewTokenRequest request) throws Exception { throw new NotImplementedException("Code is not implemented"); } + + @Override + public RouterUpdateStoredTokenResponse updateStoredToken( + RouterUpdateStoredTokenRequest request) throws Exception { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterRemoveStoredTokenResponse removeStoredToken( + RouterRemoveStoredTokenRequest request) throws Exception { + throw new NotImplementedException("Code is not implemented"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java index b8d134f3b79e9..8e2520fe982bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java @@ -85,6 +85,10 @@ import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenResponse; import org.apache.hadoop.yarn.server.records.Version; import org.apache.zookeeper.data.ACL; import org.slf4j.Logger; @@ -686,4 +690,16 @@ public RouterStoreNewTokenResponse storeNewToken( RouterStoreNewTokenRequest request) throws Exception { throw new NotImplementedException("Code is not implemented"); } + + @Override + public RouterUpdateStoredTokenResponse updateStoredToken( + RouterUpdateStoredTokenRequest request) throws Exception { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterRemoveStoredTokenResponse removeStoredToken( + RouterRemoveStoredTokenRequest request) throws Exception { + throw new NotImplementedException("Code is not implemented"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenRequest.java new file mode 100644 index 0000000000000..049746563adb4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenRequest.java @@ -0,0 +1,28 @@ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterRemoveStoredTokenRequest { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterRemoveStoredTokenRequest newInstance(RouterStoreToken routerStoreToken) { + RouterRemoveStoredTokenRequest request = + Records.newRecord(RouterRemoveStoredTokenRequest.class); + request.setRouterStoreToken(routerStoreToken); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterStoreToken getRouterStoreToken(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenResponse.java new file mode 100644 index 0000000000000..0447d4fc33744 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenResponse.java @@ -0,0 +1,27 @@ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterRemoveStoredTokenResponse { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterRemoveStoredTokenResponse newInstance(RouterStoreToken routerStoreToken) { + RouterRemoveStoredTokenResponse request = Records.newRecord(RouterRemoveStoredTokenResponse.class); + request.setRouterStoreToken(routerStoreToken); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterStoreToken getRouterStoreToken(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenRequest.java new file mode 100644 index 0000000000000..ece2918100287 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenRequest.java @@ -0,0 +1,28 @@ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterUpdateStoredTokenRequest { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterUpdateStoredTokenRequest newInstance(RouterStoreToken routerStoreToken) { + RouterUpdateStoredTokenRequest request = + Records.newRecord(RouterUpdateStoredTokenRequest.class); + request.setRouterStoreToken(routerStoreToken); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterStoreToken getRouterStoreToken(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenResponse.java new file mode 100644 index 0000000000000..9a6e89a987539 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenResponse.java @@ -0,0 +1,27 @@ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterUpdateStoredTokenResponse { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterUpdateStoredTokenResponse newInstance(RouterStoreToken routerStoreToken) { + RouterUpdateStoredTokenResponse request = Records.newRecord(RouterUpdateStoredTokenResponse.class); + request.setRouterStoreToken(routerStoreToken); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterStoreToken getRouterStoreToken(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index 51c8f48aa9994..9df4e533e50a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.federation.utils; +import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -51,28 +52,8 @@ import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; import org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreRetriableException; -import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest; -import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse; -import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest; -import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse; -import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; -import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest; -import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse; -import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; -import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse; -import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest; -import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse; -import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest; -import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse; -import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest; -import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse; -import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest; -import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse; -import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration; -import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.*; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -389,16 +370,6 @@ public SubClusterId getApplicationHomeSubCluster(ApplicationId appId) return response.getApplicationHomeSubCluster().getHomeSubCluster(); } - /** - * The Router supports saving the master key - * - * @param newKey DelegationKey - */ - public void storeNewMasterKey(DelegationKey newKey) { - LOG.info("Storing master key with keyID {}.", newKey.getKeyId()); - - } - /** * Get the singleton instance of SubClusterResolver. * @@ -446,6 +417,61 @@ public SubClusterId getReservationHomeSubCluster(ReservationId reservationId) return response.getReservationHomeSubCluster().getHomeSubCluster(); } + /** + * The Router supports saving the master key. + * + * @param newKey DelegationKey + */ + public void storeNewMasterKey(DelegationKey newKey) throws Exception { + LOG.info("Storing master key with keyID {}.", newKey.getKeyId()); + ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey()); + RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(), + keyBytes, newKey.getExpiryDate()); + StoreNewMasterKeyRequest keyRequest = StoreNewMasterKeyRequest.newInstance(masterKey); + stateStore.storeNewMasterKey(keyRequest); + } + + /** + * The Router supports remove the master key. + * + * @param newKey DelegationKey + * @throws Exception An error occurred + */ + public void removeStoredMasterKey(DelegationKey newKey) throws Exception { + LOG.info("Removing master key with keyID {}.", newKey.getKeyId()); + ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey()); + RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(), + keyBytes, newKey.getExpiryDate()); + RemoveStoredMasterKeyRequest keyRequest = RemoveStoredMasterKeyRequest.newInstance(masterKey); + stateStore.removeStoredMasterKey(keyRequest); + } + + public void storeNewToken(RMDelegationTokenIdentifier id, + long renewDate) throws Exception { + LOG.info("storing RMDelegation token with sequence number: {}.", id.getSequenceNumber()); + RouterStoreToken storeToken = RouterStoreToken.newInstance(id, renewDate); + RouterStoreNewTokenRequest request = RouterStoreNewTokenRequest.newInstance(storeToken); + stateStore.storeNewToken(request); + } + + public void updateStoredToken(RMDelegationTokenIdentifier id, + long renewDate) throws Exception { + LOG.info("updating RMDelegation token with sequence number: {}.", id.getSequenceNumber()); + RouterStoreToken storeToken = RouterStoreToken.newInstance(id, renewDate); + RouterUpdateStoredTokenRequest request = + RouterUpdateStoredTokenRequest.newInstance(storeToken); + stateStore.updateStoredToken(request); + } + + public void removeStoredToken(RMDelegationTokenIdentifier id) + throws Exception{ + LOG.info("removing RMDelegation token with sequence number: {}", id.getSequenceNumber()); + RouterStoreToken storeToken = RouterStoreToken.newInstance(id, 0L); + RouterRemoveStoredTokenRequest request = + RouterRemoveStoredTokenRequest.newInstance(storeToken); + stateStore.removeStoredToken(request); + } + /** * Helper method to create instances of Object using the class name defined in * the configuration object. The instances creates {@link RetryProxy} using diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java index 89341344da6b3..b84ec7a3ea6ac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java @@ -72,6 +72,10 @@ import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenResponse; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -349,4 +353,16 @@ public RouterStoreNewTokenResponse storeNewToken( RouterStoreNewTokenRequest request) throws Exception { return stateStoreClient.storeNewToken(request); } + + @Override + public RouterUpdateStoredTokenResponse updateStoredToken( + RouterUpdateStoredTokenRequest request) throws Exception { + return stateStoreClient.updateStoredToken(request); + } + + @Override + public RouterRemoveStoredTokenResponse removeStoredToken( + RouterRemoveStoredTokenRequest request) throws Exception { + return stateStoreClient.removeStoredToken(request); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java index 3732d201a6aef..4396e6c28bfe1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java @@ -23,13 +23,11 @@ import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; -import org.apache.hadoop.yarn.server.federation.store.records.*; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.nio.ByteBuffer; /** * A Router specific delegation token secret manager. @@ -76,12 +74,7 @@ private boolean shouldIgnoreException(Exception e) { @Override protected void storeNewMasterKey(DelegationKey newKey) { try { - LOG.info("Storing master key with keyID {}.", newKey.getKeyId()); - ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey()); - RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(), - keyBytes, newKey.getExpiryDate()); - StoreNewMasterKeyRequest keyRequest = StoreNewMasterKeyRequest.newInstance(masterKey); - federationFacade.getStateStore().storeNewMasterKey(keyRequest); + federationFacade.storeNewMasterKey(newKey); } catch (Exception e) { if (!shouldIgnoreException(e)) { LOG.error("Error in storing master key with KeyID: {}.", newKey.getKeyId()); @@ -93,12 +86,7 @@ protected void storeNewMasterKey(DelegationKey newKey) { @Override protected void removeStoredMasterKey(DelegationKey key) { try { - LOG.info("removing master key with keyID {}.", key.getKeyId()); - ByteBuffer keyBytes = ByteBuffer.wrap(key.getEncodedKey()); - RouterMasterKey masterKey = RouterMasterKey.newInstance(key.getKeyId(), - keyBytes, key.getExpiryDate()); - RemoveStoredMasterKeyRequest keyRequest = RemoveStoredMasterKeyRequest.newInstance(masterKey); - federationFacade.getStateStore().removeStoredMasterKey(keyRequest); + federationFacade.removeStoredMasterKey(key); } catch (Exception e) { if (!shouldIgnoreException(e)) { LOG.error("Error in removing master key with KeyID: {}.", key.getKeyId()); @@ -111,11 +99,7 @@ protected void removeStoredMasterKey(DelegationKey key) { protected void storeNewToken(RMDelegationTokenIdentifier identifier, long renewDate) throws IOException { try { - LOG.info("storing RMDelegation token with sequence number: {}.", - identifier.getSequenceNumber()); - RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate); - RouterStoreNewTokenRequest request = RouterStoreNewTokenRequest.newInstance(storeToken); - federationFacade.getStateStore().storeNewToken(request); + federationFacade.storeNewToken(identifier, renewDate); } catch (Exception e) { if (!shouldIgnoreException(e)) { LOG.error("Error in storing RMDelegationToken with sequence number: {}.", @@ -128,9 +112,7 @@ protected void storeNewToken(RMDelegationTokenIdentifier identifier, @Override protected void updateStoredToken(RMDelegationTokenIdentifier id, long renewDate) { try { - LOG.info("updating RMDelegation token with sequence number: {}.", - id.getSequenceNumber()); - // rm.getRMContext().getStateStore().updateRMDelegationToken(id, renewDate); + federationFacade.updateStoredToken(id, renewDate); } catch (Exception e) { if (!shouldIgnoreException(e)) { LOG.error("Error in updating persisted RMDelegationToken with sequence number: {}.", @@ -144,9 +126,7 @@ protected void updateStoredToken(RMDelegationTokenIdentifier id, long renewDate) protected void removeStoredToken(RMDelegationTokenIdentifier identifier) throws IOException { try { - LOG.info("removing RMDelegation token with sequence number: {}", - identifier.getSequenceNumber()); - // rm.getRMContext().getStateStore().removeRMDelegationToken(ident); + federationFacade.removeStoredToken(identifier); } catch (Exception e) { if (!shouldIgnoreException(e)) { LOG.error("Error in removing RMDelegationToken with sequence number: {}", From 606a2c95938507c765b1f680e98027b122707414 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sat, 13 Aug 2022 23:43:04 -0700 Subject: [PATCH 03/12] YARN-9708. Yarn Router Support DelegationToken. --- .../FederationDelegationTokenStateStore.java | 63 +++++++++++++++++++ .../utils/FederationStateStoreFacade.java | 42 ++++++++++--- .../AbstractClientRequestInterceptor.java | 12 ++++ .../clientrm/ClientRequestInterceptor.java | 15 +++++ .../clientrm/RouterClientRMService.java | 37 ++++++++++- .../RouterDelegationTokenSecretManager.java | 42 +++++++++++-- 6 files changed, 195 insertions(+), 16 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java index 5e553eee04437..9dc0f09d19778 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java @@ -29,20 +29,83 @@ import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenRequest; import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenResponse; +/** + * FederationDelegationTokenStateStore maintains the state of all + * DelegationToken that have been submitted to the federated cluster. + * + *

+ * It mainly includes the following operations: + *

+ * + * + */ @InterfaceAudience.Private @InterfaceStability.Unstable public interface FederationDelegationTokenStateStore { + /** + * The Router Supports Store the new master key. + * + * @param request DelegationKey. + * @return StoreNewMasterKeyResponse. + * @throws Exception exception occurred. + */ StoreNewMasterKeyResponse storeNewMasterKey(StoreNewMasterKeyRequest request) throws Exception; + /** + * The Router Supports Remove the master key. + * + * @param request DelegationKey. + * @return RemoveStoredMasterKeyResponse. + * @throws Exception exception occurred. + */ RemoveStoredMasterKeyResponse removeStoredMasterKey(RemoveStoredMasterKeyRequest request) throws Exception; + /** + * The Router Supports Store new Token. + * + * @param request DelegationKey. + * @return RouterStoreNewTokenResponse. + * @throws Exception exception occurred. + */ RouterStoreNewTokenResponse storeNewToken(RouterStoreNewTokenRequest request) throws Exception; + /** + * The Router Supports Update Token. + * + * @param request DelegationKey. + * @return RouterUpdateStoredTokenResponse. + * @throws Exception exception occurred. + */ RouterUpdateStoredTokenResponse updateStoredToken(RouterUpdateStoredTokenRequest request) throws Exception; + /** + * The Router Supports Remove Token. + * + * @param request DelegationKey. + * @return RouterRemoveStoredTokenResponse. + * @throws Exception exception occurred. + */ RouterRemoveStoredTokenResponse removeStoredToken(RouterRemoveStoredTokenRequest request) throws Exception; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index 9df4e533e50a7..591e3b5ce6509 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.federation.utils; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.List; @@ -446,27 +447,50 @@ public void removeStoredMasterKey(DelegationKey newKey) throws Exception { stateStore.removeStoredMasterKey(keyRequest); } - public void storeNewToken(RMDelegationTokenIdentifier id, + /** + * The Router Supports Store new Token. + * + * @param identifier Delegation Token + * @param renewDate renewDate + * @throws IOException IO exception occurred. + */ + public void storeNewToken(RMDelegationTokenIdentifier identifier, long renewDate) throws Exception { - LOG.info("storing RMDelegation token with sequence number: {}.", id.getSequenceNumber()); - RouterStoreToken storeToken = RouterStoreToken.newInstance(id, renewDate); + LOG.info("storing RMDelegation token with sequence number: {}.", + identifier.getSequenceNumber()); + RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate); RouterStoreNewTokenRequest request = RouterStoreNewTokenRequest.newInstance(storeToken); stateStore.storeNewToken(request); } - public void updateStoredToken(RMDelegationTokenIdentifier id, + /** + * The Router Supports Update Token. + * + * @param identifier Delegation Token + * @param renewDate renewDate + * @throws IOException IO exception occurred. + */ + public void updateStoredToken(RMDelegationTokenIdentifier identifier, long renewDate) throws Exception { - LOG.info("updating RMDelegation token with sequence number: {}.", id.getSequenceNumber()); - RouterStoreToken storeToken = RouterStoreToken.newInstance(id, renewDate); + LOG.info("updating RMDelegation token with sequence number: {}.", + identifier.getSequenceNumber()); + RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate); RouterUpdateStoredTokenRequest request = RouterUpdateStoredTokenRequest.newInstance(storeToken); stateStore.updateStoredToken(request); } - public void removeStoredToken(RMDelegationTokenIdentifier id) + /** + * The Router Supports Remove Token. + * + * @param identifier Delegation Token + * @throws IOException IO exception occurred. + */ + public void removeStoredToken(RMDelegationTokenIdentifier identifier) throws Exception{ - LOG.info("removing RMDelegation token with sequence number: {}", id.getSequenceNumber()); - RouterStoreToken storeToken = RouterStoreToken.newInstance(id, 0L); + LOG.info("removing RMDelegation token with sequence number: {}", + identifier.getSequenceNumber()); + RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, 0L); RouterRemoveStoredTokenRequest request = RouterRemoveStoredTokenRequest.newInstance(storeToken); stateStore.removeStoredToken(request); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java index 961026d014611..c71be3c46f755 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/AbstractClientRequestInterceptor.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,6 +45,8 @@ public abstract class AbstractClientRequestInterceptor @SuppressWarnings("checkstyle:visibilitymodifier") protected UserGroupInformation user = null; + private RouterDelegationTokenSecretManager tokenSecretManager = null; + /** * Sets the {@link ClientRequestInterceptor} in the chain. */ @@ -125,4 +128,13 @@ private void setupUser(String userName) { } } + @Override + public RouterDelegationTokenSecretManager getTokenSecretManager() { + return tokenSecretManager; + } + + @Override + public void setTokenSecretManager(RouterDelegationTokenSecretManager tokenSecretManager) { + this.tokenSecretManager = tokenSecretManager; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java index 3e3ffce5f4b3f..6e19cbadf9d38 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/ClientRequestInterceptor.java @@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.yarn.api.ApplicationClientProtocol; +import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager; /** * Defines the contract to be implemented by the request interceptor classes, @@ -62,4 +63,18 @@ public interface ClientRequestInterceptor */ ClientRequestInterceptor getNextInterceptor(); + /** + * Set RouterDelegationTokenSecretManager for specific interceptor to support Token operations, + * including create Token, update Token, and delete Token. + * + * @param tokenSecretManager Router DelegationTokenSecretManager + */ + void setTokenSecretManager(RouterDelegationTokenSecretManager tokenSecretManager); + + /** + * Get RouterDelegationTokenSecretManager. + * + * @return Router DelegationTokenSecretManager. + */ + RouterDelegationTokenSecretManager getTokenSecretManager(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index b60a267746e4f..1c8550ef26a08 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -105,6 +105,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.router.RouterServerUtil; +import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager; import org.apache.hadoop.yarn.server.router.security.authorize.RouterPolicyProvider; import org.apache.hadoop.yarn.util.LRUCacheHashMap; import org.slf4j.Logger; @@ -136,6 +137,8 @@ public class RouterClientRMService extends AbstractService // and remove the oldest used ones. private Map userPipelineMap; + private RouterDelegationTokenSecretManager routerDTSecretManager; + public RouterClientRMService() { super(RouterClientRMService.class.getName()); } @@ -164,8 +167,11 @@ protected void serviceStart() throws Exception { serverConf.getInt(YarnConfiguration.RM_CLIENT_THREAD_COUNT, YarnConfiguration.DEFAULT_RM_CLIENT_THREAD_COUNT); + // Initialize RouterRMDelegationTokenSecretManager. + routerDTSecretManager = createRouterRMDelegationTokenSecretManager(conf); + this.server = rpc.getServer(ApplicationClientProtocol.class, this, - listenerEndpoint, serverConf, null, numWorkerThreads); + listenerEndpoint, serverConf, routerDTSecretManager, numWorkerThreads); // Enable service authorization? if (conf.getBoolean( @@ -508,6 +514,12 @@ private RequestInterceptorChainWrapper initializePipeline(String user) { ClientRequestInterceptor interceptorChain = this.createRequestInterceptorChain(); interceptorChain.init(user); + + // + if (routerDTSecretManager != null) { + interceptorChain.setTokenSecretManager(routerDTSecretManager); + } + chainWrapper.init(interceptorChain); } catch (Exception e) { LOG.error("Init ClientRequestInterceptor error for user: {}.", user, e); @@ -558,4 +570,27 @@ protected void finalize() { public Map getUserPipelineMap() { return userPipelineMap; } + + /** + * Create RouterRMDelegationTokenSecretManager. + * In the YARN federation, the Router will replace the RM to + * manage the RMDelegationToken (generate, update, cancel), + * so the relevant configuration parameters still obtain the configuration parameters of the RM. + * + * @param conf Configuration + * @return RouterDelegationTokenSecretManager. + */ + protected RouterDelegationTokenSecretManager createRouterRMDelegationTokenSecretManager( + Configuration conf) { + + long secretKeyInterval = conf.getLong(YarnConfiguration.RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY, + YarnConfiguration.RM_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT); + long tokenMaxLifetime = conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_KEY, + YarnConfiguration.RM_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT); + long tokenRenewInterval = conf.getLong(YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, + YarnConfiguration.RM_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT); + + return new RouterDelegationTokenSecretManager(secretKeyInterval, + tokenMaxLifetime, tokenRenewInterval, 3600000); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java index 4396e6c28bfe1..3f934f6cd916b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java @@ -43,7 +43,7 @@ public class RouterDelegationTokenSecretManager private FederationStateStoreFacade federationFacade; /** - * Create a secret manager. + * Create a Router Secret manager. * * @param delegationKeyUpdateInterval the number of milliseconds for rolling * new secret keys. @@ -59,7 +59,6 @@ public RouterDelegationTokenSecretManager(long delegationKeyUpdateInterval, long delegationTokenRemoverScanInterval) { super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, delegationTokenRenewInterval, delegationTokenRemoverScanInterval); - federationFacade = FederationStateStoreFacade.getInstance(); } @Override @@ -71,6 +70,11 @@ private boolean shouldIgnoreException(Exception e) { return !running && e.getCause() instanceof InterruptedException; } + /** + * The Router Supports Store the new master key. + * + * @param newKey DelegationKey + */ @Override protected void storeNewMasterKey(DelegationKey newKey) { try { @@ -83,18 +87,30 @@ protected void storeNewMasterKey(DelegationKey newKey) { } } + /** + * The Router Supports Remove the master key. + * + * @param delegationKey DelegationKey + */ @Override - protected void removeStoredMasterKey(DelegationKey key) { + protected void removeStoredMasterKey(DelegationKey delegationKey) { try { - federationFacade.removeStoredMasterKey(key); + federationFacade.removeStoredMasterKey(delegationKey); } catch (Exception e) { if (!shouldIgnoreException(e)) { - LOG.error("Error in removing master key with KeyID: {}.", key.getKeyId()); + LOG.error("Error in removing master key with KeyID: {}.", delegationKey.getKeyId()); ExitUtil.terminate(1, e); } } } + /** + * The Router Supports Store new Token. + * + * @param identifier Delegation Token + * @param renewDate renewDate + * @throws IOException IO exception occurred. + */ @Override protected void storeNewToken(RMDelegationTokenIdentifier identifier, long renewDate) throws IOException { @@ -109,8 +125,16 @@ protected void storeNewToken(RMDelegationTokenIdentifier identifier, } } + /** + * The Router Supports Update Token. + * + * @param id Delegation Token + * @param renewDate renewDate + * @throws IOException IO exception occurred. + */ @Override - protected void updateStoredToken(RMDelegationTokenIdentifier id, long renewDate) { + protected void updateStoredToken(RMDelegationTokenIdentifier id, long renewDate) + throws IOException { try { federationFacade.updateStoredToken(id, renewDate); } catch (Exception e) { @@ -122,6 +146,12 @@ protected void updateStoredToken(RMDelegationTokenIdentifier id, long renewDate) } } + /** + * The Router Supports Remove Token. + * + * @param identifier Delegation Token + * @throws IOException IO exception occurred. + */ @Override protected void removeStoredToken(RMDelegationTokenIdentifier identifier) throws IOException { From 2ca1d47361577ccc2970ec031295afc550b7edbd Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sun, 14 Aug 2022 03:21:21 -0700 Subject: [PATCH 04/12] YARN-11252. Fix CheckStyle. --- .../GetRouterDelegationTokenResponse.java | 8 ------- .../store/records/RouterMasterKey.java | 3 ++- .../records/RouterRMDTSecretManagerState.java | 20 ++++++++++++++++- .../RouterRemoveStoredTokenRequest.java | 17 ++++++++++++++ .../RouterRemoveStoredTokenResponse.java | 21 ++++++++++++++++-- .../records/RouterStoreNewTokenRequest.java | 17 ++++++++++++++ .../records/RouterStoreNewTokenResponse.java | 18 ++++++++++++++- .../RouterUpdateStoredTokenRequest.java | 17 ++++++++++++++ .../RouterUpdateStoredTokenResponse.java | 21 ++++++++++++++++-- .../impl/pb/RouterMasterKeyPBImpl.java | 17 ++++++++++++++ .../impl/pb/RouterStoreTokenPBImpl.java | 22 +++++++++++++++++++ .../utils/FederationStateStoreFacade.java | 1 + 12 files changed, 167 insertions(+), 15 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenResponse.java index b9f304cc6909f..71a0e3225cbc9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenResponse.java @@ -23,17 +23,9 @@ import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; -import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.util.Records; - -/** - * Response to a {@link GetDelegationTokenRequest} request - * from the client. The response contains the token that - * can be used by the containers to talk to ClientRMService. - * - */ @Public @Stable public abstract class GetRouterDelegationTokenResponse { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java index b7c7f0a3ddf52..f9cb89354a2ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKey.java @@ -120,7 +120,8 @@ public boolean equals(Object right) { return false; } else { RouterMasterKey r = (RouterMasterKey) right; - return getKeyId() == r.getKeyId() && getExpiryDate() == r.getExpiryDate() && + return getKeyId().intValue() == r.getKeyId().intValue() && + getExpiryDate().longValue() == r.getExpiryDate().longValue() && Arrays.equals(getKeyBytes().array(), r.getKeyBytes().array()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java index ecf39c16a77c8..e5010945286e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMDTSecretManagerState.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.federation.store.records; import org.apache.hadoop.security.token.delegation.DelegationKey; @@ -11,7 +28,8 @@ public class RouterRMDTSecretManagerState { // DTIdentifier -> renewDate - private Map delegationTokenState = new HashMap(); + private Map delegationTokenState = + new HashMap(); private Set masterKeyState = new HashSet(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenRequest.java index 049746563adb4..d616ec0fe6699 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenRequest.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.federation.store.records; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenResponse.java index 0447d4fc33744..1a3a8092d61b7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenResponse.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.federation.store.records; import org.apache.hadoop.classification.InterfaceAudience; @@ -11,7 +28,8 @@ public abstract class RouterRemoveStoredTokenResponse { @InterfaceAudience.Private @InterfaceStability.Unstable public static RouterRemoveStoredTokenResponse newInstance(RouterStoreToken routerStoreToken) { - RouterRemoveStoredTokenResponse request = Records.newRecord(RouterRemoveStoredTokenResponse.class); + RouterRemoveStoredTokenResponse request = + Records.newRecord(RouterRemoveStoredTokenResponse.class); request.setRouterStoreToken(routerStoreToken); return request; } @@ -23,5 +41,4 @@ public static RouterRemoveStoredTokenResponse newInstance(RouterStoreToken route @InterfaceAudience.Private @InterfaceStability.Unstable public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenRequest.java index 0cbe02a156e90..b3a4608e9c393 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenRequest.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.federation.store.records; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenResponse.java index 357773ae79e6d..c119661f6876d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenResponse.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.federation.store.records; import org.apache.hadoop.classification.InterfaceAudience; @@ -23,5 +40,4 @@ public static RouterStoreNewTokenResponse newInstance(RouterStoreToken routerSto @InterfaceAudience.Private @InterfaceStability.Unstable public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenRequest.java index ece2918100287..af2eb7adc6563 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenRequest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenRequest.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.federation.store.records; import org.apache.hadoop.classification.InterfaceAudience; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenResponse.java index 9a6e89a987539..600cc9fcb3ea6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenResponse.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenResponse.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.federation.store.records; import org.apache.hadoop.classification.InterfaceAudience; @@ -11,7 +28,8 @@ public abstract class RouterUpdateStoredTokenResponse { @InterfaceAudience.Private @InterfaceStability.Unstable public static RouterUpdateStoredTokenResponse newInstance(RouterStoreToken routerStoreToken) { - RouterUpdateStoredTokenResponse request = Records.newRecord(RouterUpdateStoredTokenResponse.class); + RouterUpdateStoredTokenResponse request = + Records.newRecord(RouterUpdateStoredTokenResponse.class); request.setRouterStoreToken(routerStoreToken); return request; } @@ -23,5 +41,4 @@ public static RouterUpdateStoredTokenResponse newInstance(RouterStoreToken route @InterfaceAudience.Private @InterfaceStability.Unstable public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyPBImpl.java index 1388c51aef457..e2adccc955205 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyPBImpl.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; import org.apache.hadoop.thirdparty.protobuf.ByteString; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java index 3c07a50cf3d83..60668bd917747 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; import org.apache.hadoop.classification.InterfaceAudience; @@ -14,6 +31,9 @@ import java.io.DataInputStream; import java.io.IOException; +/** + * Protocol buffer based implementation of {@link RouterStoreToken}. + */ @InterfaceAudience.Private @InterfaceStability.Unstable public class RouterStoreTokenPBImpl extends RouterStoreToken { @@ -87,6 +107,7 @@ public void setIdentifier(YARNDelegationTokenIdentifier identifier) { maybeInitBuilder(); if(identifier == null) { builder.clearTokenIdentifier(); + return; } builder.setTokenIdentifier(identifier.getProto()); } @@ -96,6 +117,7 @@ public void setRenewDate(Long renewDate) { maybeInitBuilder(); if(renewDate == null) { builder.clearRenewDate(); + return; } builder.setRenewDate(renewDate); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index 591e3b5ce6509..9ed1452c89fc2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -422,6 +422,7 @@ public SubClusterId getReservationHomeSubCluster(ReservationId reservationId) * The Router supports saving the master key. * * @param newKey DelegationKey + * @throws Exception An error occurred */ public void storeNewMasterKey(DelegationKey newKey) throws Exception { LOG.info("Storing master key with keyID {}.", newKey.getKeyId()); From 70d0c6ae907d1e9519fda8bf75b77e05fa581714 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Sun, 14 Aug 2022 04:56:48 -0700 Subject: [PATCH 05/12] YARN-9708. Fix CheckStyle. --- .../store/records/RouterStoreToken.java | 6 +- ...> RemoveStoredMasterKeyRequestPBImpl.java} | 33 ++--- .../RemoveStoredMasterKeyResponsePBImpl.java | 108 +++++++++++++++ .../pb/StoreNewMasterKeyRequestPBImpl.java | 8 +- .../pb/StoreNewMasterKeyResponsePBImpl.java | 126 ++++++++++++++++++ .../TestFederationProtocolRecords.java | 7 + 6 files changed, 265 insertions(+), 23 deletions(-) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/{RemoveNewMasterKeyRequestPBImpl.java => RemoveStoredMasterKeyRequestPBImpl.java} (77%) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyResponsePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyResponsePBImpl.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java index e77301bd92e74..a1a895ab8eeb1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java @@ -62,9 +62,9 @@ public byte[] toByteArray() throws IOException { public RMDelegationTokenIdentifier getTokenIdentifier() throws IOException { ByteArrayInputStream in = new ByteArrayInputStream(builder.getTokenIdentifier().toByteArray()); - RMDelegationTokenIdentifier identifer = new RMDelegationTokenIdentifier(); - identifer.readFields(new DataInputStream(in)); - return identifer; + RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(); + identifier.readFields(new DataInputStream(in)); + return identifier; } public Long getRenewDate() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveNewMasterKeyRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyRequestPBImpl.java similarity index 77% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveNewMasterKeyRequestPBImpl.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyRequestPBImpl.java index 9555a4874db9a..6f3f119082d89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveNewMasterKeyRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyRequestPBImpl.java @@ -16,36 +16,36 @@ */ package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.thirdparty.protobuf.TextFormat; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyRequestProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyRequestProtoOrBuilder; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RemoveStoredMasterKeyRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RemoveStoredMasterKeyRequestProtoOrBuilder; +import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyRequest; import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; -import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; -@InterfaceAudience.Private -@InterfaceStability.Unstable -public class RemoveNewMasterKeyRequestPBImpl extends StoreNewMasterKeyRequest { +@Private +@Unstable +public class RemoveStoredMasterKeyRequestPBImpl extends RemoveStoredMasterKeyRequest { - private StoreNewMasterKeyRequestProto proto = StoreNewMasterKeyRequestProto.getDefaultInstance(); + private RemoveStoredMasterKeyRequestProto proto = RemoveStoredMasterKeyRequestProto.getDefaultInstance(); - private StoreNewMasterKeyRequestProto.Builder builder = null; + private RemoveStoredMasterKeyRequestProto.Builder builder = null; private boolean viaProto = false; private RouterMasterKey routerMasterKey = null; - public RemoveNewMasterKeyRequestPBImpl() { - builder = StoreNewMasterKeyRequestProto.newBuilder(); + public RemoveStoredMasterKeyRequestPBImpl() { + builder = RemoveStoredMasterKeyRequestProto.newBuilder(); } - public RemoveNewMasterKeyRequestPBImpl(StoreNewMasterKeyRequestProto proto) { + public RemoveStoredMasterKeyRequestPBImpl(RemoveStoredMasterKeyRequestProto proto) { this.proto = proto; viaProto = true; } - public StoreNewMasterKeyRequestProto getProto() { + public RemoveStoredMasterKeyRequestProto getProto() { mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; @@ -63,7 +63,7 @@ private void mergeLocalToProto() { private void maybeInitBuilder() { if (viaProto || builder == null) { - builder = StoreNewMasterKeyRequestProto.newBuilder(proto); + builder = RemoveStoredMasterKeyRequestProto.newBuilder(proto); } viaProto = false; } @@ -100,7 +100,7 @@ public String toString() { @Override public RouterMasterKey getRouterMasterKey() { - StoreNewMasterKeyRequestProtoOrBuilder p = viaProto ? proto : builder; + RemoveStoredMasterKeyRequestProtoOrBuilder p = viaProto ? proto : builder; if (this.routerMasterKey != null) { return this.routerMasterKey; } @@ -116,6 +116,7 @@ public void setRouterMasterKey(RouterMasterKey masterKey) { maybeInitBuilder(); if (masterKey == null) { builder.clearRouterMasterKey(); + return; } this.routerMasterKey = masterKey; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyResponsePBImpl.java new file mode 100644 index 0000000000000..3a136d96abda7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyResponsePBImpl.java @@ -0,0 +1,108 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RemoveStoredMasterKeyResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RemoveStoredMasterKeyResponseProtoOrBuilder; +import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; + +public class RemoveStoredMasterKeyResponsePBImpl extends RemoveStoredMasterKeyResponse { + + private RemoveStoredMasterKeyResponseProto proto = + RemoveStoredMasterKeyResponseProto.getDefaultInstance(); + + private RemoveStoredMasterKeyResponseProto.Builder builder = null; + + private boolean viaProto = false; + + private RouterMasterKey routerMasterKey = null; + + public RemoveStoredMasterKeyResponsePBImpl() { + builder = RemoveStoredMasterKeyResponseProto.newBuilder(); + } + + public RemoveStoredMasterKeyResponsePBImpl(RemoveStoredMasterKeyResponseProto responseProto) { + this.proto = responseProto; + viaProto = true; + } + + public RemoveStoredMasterKeyResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RemoveStoredMasterKeyResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.routerMasterKey != null && !((RouterMasterKeyPBImpl) this.routerMasterKey). + getProto().equals(builder.getRouterMasterKey())) { + builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); + } + } + + private YarnServerFederationProtos.RouterMasterKeyProto convertToProtoFormat( + RouterMasterKey masterKey) { + return ((RouterMasterKeyPBImpl) masterKey).getProto(); + } + + + @Override + public RouterMasterKey getRouterMasterKey() { + RemoveStoredMasterKeyResponseProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerMasterKey != null) { + return this.routerMasterKey; + } + if (!p.hasRouterMasterKey()) { + return null; + } + this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey()); + return this.routerMasterKey; + } + + @Override + public void setRouterMasterKey(RouterMasterKey masterKey) { + maybeInitBuilder(); + if (masterKey == null) { + builder.clearRouterMasterKey(); + return; + } + this.routerMasterKey = masterKey; + } + + private RouterMasterKey convertFromProtoFormat(RouterMasterKeyProto masterKey) { + return new RouterMasterKeyPBImpl(masterKey); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyRequestPBImpl.java index 7f9b63c874de9..ff569bee65397 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyRequestPBImpl.java @@ -16,8 +16,8 @@ */ package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.thirdparty.protobuf.TextFormat; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyRequestProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; @@ -25,8 +25,8 @@ import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyRequestProtoOrBuilder; -@InterfaceAudience.Private -@InterfaceStability.Unstable +@Private +@Unstable public class StoreNewMasterKeyRequestPBImpl extends StoreNewMasterKeyRequest { private StoreNewMasterKeyRequestProto proto = StoreNewMasterKeyRequestProto.getDefaultInstance(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyResponsePBImpl.java new file mode 100644 index 0000000000000..61d7af86f041f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyResponsePBImpl.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyResponseProtoOrBuilder; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; +import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyResponse; + +@Private +@Unstable +public class StoreNewMasterKeyResponsePBImpl extends StoreNewMasterKeyResponse { + + private StoreNewMasterKeyResponseProto proto = + StoreNewMasterKeyResponseProto.getDefaultInstance(); + + private StoreNewMasterKeyResponseProto.Builder builder = null; + + private boolean viaProto = false; + + private RouterMasterKey routerMasterKey = null; + + public StoreNewMasterKeyResponsePBImpl() { + builder = StoreNewMasterKeyResponseProto.newBuilder(); + } + + public StoreNewMasterKeyResponsePBImpl(StoreNewMasterKeyResponseProto responseProto) { + this.proto = responseProto; + viaProto = true; + } + + public StoreNewMasterKeyResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = StoreNewMasterKeyResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.routerMasterKey != null && !((RouterMasterKeyPBImpl) this.routerMasterKey). + getProto().equals(builder.getRouterMasterKey())) { + builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); + } + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + private RouterMasterKeyProto convertToProtoFormat( + RouterMasterKey masterKey) { + return ((RouterMasterKeyPBImpl) masterKey).getProto(); + } + + @Override + public RouterMasterKey getRouterMasterKey() { + StoreNewMasterKeyResponseProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerMasterKey != null) { + return this.routerMasterKey; + } + if (!p.hasRouterMasterKey()) { + return null; + } + this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey()); + return this.routerMasterKey; + } + + @Override + public void setRouterMasterKey(RouterMasterKey masterKey) { + maybeInitBuilder(); + if (masterKey == null) { + builder.clearRouterMasterKey(); + return; + } + this.routerMasterKey = masterKey; + } + + private RouterMasterKey convertFromProtoFormat(RouterMasterKeyProto masterKey) { + return new RouterMasterKeyPBImpl(masterKey); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java index cf8cf719d01d5..85b619c48512e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterResponseProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterRequestProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyRequestProto; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.AddApplicationHomeSubClusterRequestPBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.AddApplicationHomeSubClusterResponsePBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.DeleteApplicationHomeSubClusterRequestPBImpl; @@ -75,6 +76,7 @@ import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterRegisterResponsePBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.UpdateApplicationHomeSubClusterRequestPBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.UpdateApplicationHomeSubClusterResponsePBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.StoreNewMasterKeyRequestPBImpl; import org.apache.hadoop.yarn.server.records.Version; import org.junit.BeforeClass; import org.junit.Test; @@ -262,4 +264,9 @@ public void testGetSubClusterPoliciesConfigurationsResponse() GetSubClusterPoliciesConfigurationsResponsePBImpl.class, GetSubClusterPoliciesConfigurationsResponseProto.class); } + + @Test + public void testStoreNewMasterKeyRequest() throws Exception { + + } } From 535f50fbe774804c89d6a2d39ef16bc08705b193 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Mon, 15 Aug 2022 21:14:35 +0800 Subject: [PATCH 06/12] YARN-9708. Yarn Router Support DelegationToken. --- .../client/YARNDelegationTokenIdentifier.java | 13 ++ .../YARNDelegationTokenIdentifierPBImpl.java | 201 ++++++++++++++++++ .../security/client/impl/pb/package-info.java | 21 ++ .../yarn/api/BasePBImplRecordsTest.java | 6 +- .../impl/MemoryFederationStateStore.java | 6 +- .../store/records/RouterStoreToken.java | 61 ++---- .../RemoveStoredMasterKeyResponsePBImpl.java | 16 ++ .../RouterRemoveStoredTokenRequestPBImpl.java | 124 +++++++++++ ...RouterRemoveStoredTokenResponsePBImpl.java | 111 ++++++++++ .../pb/RouterStoreNewTokenRequestPBImpl.java | 124 +++++++++++ .../pb/RouterStoreNewTokenResponsePBImpl.java | 110 ++++++++++ .../impl/pb/RouterStoreTokenPBImpl.java | 52 ++++- .../RouterUpdateStoredTokenRequestPBImpl.java | 124 +++++++++++ ...RouterUpdateStoredTokenResponsePBImpl.java | 124 +++++++++++ .../proto/yarn_server_federation_protos.proto | 23 +- .../TestFederationProtocolRecords.java | 92 ++++++++ 16 files changed, 1155 insertions(+), 53 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenResponsePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenResponsePBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenRequestPBImpl.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenResponsePBImpl.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java index 6d8bc4bc1da88..0020b3c407627 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/YARNDelegationTokenIdentifier.java @@ -22,11 +22,14 @@ import java.io.DataOutputStream; import java.io.IOException; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto; +import org.apache.hadoop.yarn.util.Records; @Private public abstract class YARNDelegationTokenIdentifier extends @@ -112,4 +115,14 @@ public YARNDelegationTokenIdentifierProto getProto() { setBuilderFields(); return builder.build(); } + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static YARNDelegationTokenIdentifier newInstance(Text owner, Text renewer, Text realUser) { + YARNDelegationTokenIdentifier policy = Records.newRecord(YARNDelegationTokenIdentifier.class); + policy.setOwner(owner); + policy.setRenewer(renewer); + policy.setRenewer(realUser); + return policy; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java new file mode 100644 index 0000000000000..d7b25463fff3d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java @@ -0,0 +1,201 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.security.client.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProto; +import org.apache.hadoop.yarn.proto.YarnSecurityTokenProtos.YARNDelegationTokenIdentifierProtoOrBuilder; +import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public class YARNDelegationTokenIdentifierPBImpl extends YARNDelegationTokenIdentifier { + + private YARNDelegationTokenIdentifierProto proto = + YARNDelegationTokenIdentifierProto.getDefaultInstance(); + private YARNDelegationTokenIdentifierProto.Builder builder = null; + private boolean viaProto = false; + + public YARNDelegationTokenIdentifierPBImpl() { + builder = YARNDelegationTokenIdentifierProto.newBuilder(); + } + + public YARNDelegationTokenIdentifierPBImpl(YARNDelegationTokenIdentifierProto identifierProto) { + this.proto = identifierProto; + viaProto = true; + } + + public YARNDelegationTokenIdentifierProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + // mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + if (proto == null) { + proto = YARNDelegationTokenIdentifierProto.getDefaultInstance(); + } + builder = YARNDelegationTokenIdentifierProto.newBuilder(proto); + } + viaProto = false; + } + + @Override + public Text getOwner() { + YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder; + return new Text(p.getOwner()); + } + + @Override + public void setOwner(Text owner) { + super.setOwner(owner); + maybeInitBuilder(); + if (owner == null) { + builder.clearOwner(); + return; + } + builder.setOwner(owner.toString()); + } + + @Override + public Text getRenewer() { + YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder; + return new Text(p.getRenewer()); + } + + @Override + public void setRenewer(Text renewer) { + super.setRenewer(renewer); + maybeInitBuilder(); + if (renewer == null) { + builder.clearRenewer(); + return; + } + builder.setOwner(renewer.toString()); + } + + @Override + public Text getRealUser() { + YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder; + return new Text(p.getRealUser()); + } + + @Override + public void setRealUser(Text realUser) { + super.setRealUser(realUser); + maybeInitBuilder(); + if (realUser == null) { + builder.clearRealUser(); + return; + } + builder.setRealUser(realUser.toString()); + } + + @Override + public void setIssueDate(long issueDate) { + super.setIssueDate(issueDate); + maybeInitBuilder(); + builder.setIssueDate(issueDate); + } + + @Override + public long getIssueDate() { + YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder; + return p.getIssueDate(); + } + + @Override + public void setMaxDate(long maxDate) { + super.setMaxDate(maxDate); + maybeInitBuilder(); + builder.setMaxDate(maxDate); + } + + @Override + public long getMaxDate() { + YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder; + return p.getMaxDate(); + } + + @Override + public void setSequenceNumber(int seqNum) { + super.setSequenceNumber(seqNum); + maybeInitBuilder(); + builder.setSequenceNumber(seqNum); + } + + @Override + public int getSequenceNumber() { + YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder; + return p.getSequenceNumber(); + } + + @Override + public void setMasterKeyId(int newId) { + super.setMasterKeyId(newId); + maybeInitBuilder(); + builder.setMasterKeyId(newId); + } + + @Override + public int getMasterKeyId() { + YARNDelegationTokenIdentifierProtoOrBuilder p = viaProto ? proto : builder; + return p.getMasterKeyId(); + } + + @Override + public Text getKind() { + return null; + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java new file mode 100644 index 0000000000000..ce2fd41dccd86 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@InterfaceAudience.Public +package org.apache.hadoop.yarn.security.client.impl.pb; +import org.apache.hadoop.classification.InterfaceAudience; + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java index 5697923c9974d..63a5b352eef02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java @@ -18,10 +18,12 @@ package org.apache.hadoop.yarn.api; import org.apache.commons.lang3.Range; +import org.apache.hadoop.io.Text; import org.apache.hadoop.util.Lists; import org.apache.hadoop.util.Sets; import org.apache.hadoop.yarn.api.resource.PlacementConstraint; import org.apache.hadoop.yarn.api.resource.PlacementConstraints; +import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; import org.apache.hadoop.thirdparty.com.google.common.collect.Maps; @@ -80,6 +82,8 @@ private static Object genTypeValue(Type type) { 'a' + rand.nextInt(26)); } else if (type.equals(Float.class)) { return rand.nextFloat(); + } else if (type.equals(Text.class)) { + return new Text('a' + String.valueOf(rand.nextInt(1000000))); } else if (type instanceof Class) { Class clazz = (Class)type; if (clazz.isArray()) { @@ -167,7 +171,7 @@ protected static Object generateByNewInstance(Class clazz) throws Exception { " does not have newInstance method"); } Object [] args = new Object[paramTypes.length]; - for (int i=0;i rmDTState = routerRMSecretManagerState.getTokenState(); rmDTState.remove(tokenIdentifier); return RouterRemoveStoredTokenResponse.newInstance(storeToken); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java index a1a895ab8eeb1..d6eade878020f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java @@ -17,32 +17,19 @@ */ package org.apache.hadoop.yarn.server.federation.store.records; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; -import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; import org.apache.hadoop.yarn.util.Records; -import java.io.ByteArrayInputStream; -import java.io.DataInput; -import java.io.DataInputStream; import java.io.IOException; -@InterfaceAudience.Private -@InterfaceStability.Unstable +@Private +@Unstable public abstract class RouterStoreToken { - private RouterStoreTokenProto.Builder builder = RouterStoreTokenProto.newBuilder(); - public RouterStoreToken() {} - - public RouterStoreToken(YARNDelegationTokenIdentifier identifier, Long renewdate) { - builder.setTokenIdentifier(identifier.getProto()); - builder.setRenewDate(renewdate); - } - - @InterfaceAudience.Private - @InterfaceStability.Unstable + @Private + @Unstable public static RouterStoreToken newInstance(YARNDelegationTokenIdentifier identifier, Long renewdate) { RouterStoreToken storeToken = Records.newRecord(RouterStoreToken.class); @@ -51,31 +38,19 @@ public static RouterStoreToken newInstance(YARNDelegationTokenIdentifier identif return storeToken; } - public void readFields(DataInput in) throws IOException { - builder.mergeFrom((DataInputStream) in); - } + @Private + @Unstable + public abstract YARNDelegationTokenIdentifier getTokenIdentifier() throws IOException; - public byte[] toByteArray() throws IOException { - return builder.build().toByteArray(); - } + @Private + @Unstable + public abstract void setIdentifier(YARNDelegationTokenIdentifier identifier); - public RMDelegationTokenIdentifier getTokenIdentifier() throws IOException { - ByteArrayInputStream in = - new ByteArrayInputStream(builder.getTokenIdentifier().toByteArray()); - RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(); - identifier.readFields(new DataInputStream(in)); - return identifier; - } + @Private + @Unstable + public abstract Long getRenewDate(); - public Long getRenewDate() { - return builder.getRenewDate(); - } - - public void setIdentifier(YARNDelegationTokenIdentifier identifier) { - builder.setTokenIdentifier(identifier.getProto()); - } - - public void setRenewDate(Long renewDate) { - builder.setRenewDate(renewDate); - } + @Private + @Unstable + public abstract void setRenewDate(Long renewDate); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyResponsePBImpl.java index 3a136d96abda7..6f06e76d0fb74 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyResponsePBImpl.java @@ -105,4 +105,20 @@ public void setRouterMasterKey(RouterMasterKey masterKey) { private RouterMasterKey convertFromProtoFormat(RouterMasterKeyProto masterKey) { return new RouterMasterKeyPBImpl(masterKey); } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenRequestPBImpl.java new file mode 100644 index 0000000000000..c8a9a280d30d3 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenRequestPBImpl.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRemoveStoredTokenRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRemoveStoredTokenRequestProtoOrBuilder; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; + +public class RouterRemoveStoredTokenRequestPBImpl extends RouterStoreNewTokenRequest { + + private RouterRemoveStoredTokenRequestProto proto = RouterRemoveStoredTokenRequestProto.getDefaultInstance(); + private RouterRemoveStoredTokenRequestProto.Builder builder = null; + private boolean viaProto = false; + private RouterStoreToken routerStoreToken = null; + + public RouterRemoveStoredTokenRequestPBImpl() { + builder = RouterRemoveStoredTokenRequestProto.newBuilder(); + } + + public RouterRemoveStoredTokenRequestPBImpl(RouterRemoveStoredTokenRequestProto requestProto) { + this.proto = requestProto; + viaProto = true; + } + + public RouterRemoveStoredTokenRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RouterRemoveStoredTokenRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.routerStoreToken != null && + !((RouterStoreTokenPBImpl) this.routerStoreToken).getProto().equals(builder.getRouterStoreToken())) { + builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); + } + } + + @Override + public RouterStoreToken getRouterStoreToken() { + RouterRemoveStoredTokenRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerStoreToken != null) { + return this.routerStoreToken; + } + if (!p.hasRouterStoreToken()) { + return null; + } + this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken()); + return this.routerStoreToken; + } + + @Override + public void setRouterStoreToken(RouterStoreToken storeToken) { + maybeInitBuilder(); + if (storeToken == null) { + builder.clearRouterStoreToken(); + } + this.routerStoreToken = storeToken; + this.builder.setRouterStoreToken(convertToProtoFormat(storeToken)); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private RouterStoreTokenProto convertToProtoFormat(RouterStoreToken storeToken) { + return ((RouterStoreTokenPBImpl) storeToken).getProto(); + } + + private RouterStoreToken convertFromProtoFormat(RouterStoreTokenProto storeTokenProto) { + return new RouterStoreTokenPBImpl(storeTokenProto); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenResponsePBImpl.java new file mode 100644 index 0000000000000..19b1266a624b6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenResponsePBImpl.java @@ -0,0 +1,111 @@ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRemoveStoredTokenResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRemoveStoredTokenResponseProtoOrBuilder; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; + +@Private +@Unstable +public class RouterRemoveStoredTokenResponsePBImpl extends RouterStoreNewTokenResponse { + + private RouterRemoveStoredTokenResponseProto proto = + RouterRemoveStoredTokenResponseProto.getDefaultInstance(); + private RouterRemoveStoredTokenResponseProto.Builder builder = null; + private boolean viaProto = false; + private RouterStoreToken routerStoreToken = null; + + public RouterRemoveStoredTokenResponsePBImpl() { + builder = RouterRemoveStoredTokenResponseProto.newBuilder(); + } + + public RouterRemoveStoredTokenResponsePBImpl(RouterRemoveStoredTokenResponseProto requestProto) { + this.proto = requestProto; + viaProto = true; + } + + public RouterRemoveStoredTokenResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RouterRemoveStoredTokenResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.routerStoreToken != null && + !((RouterStoreTokenPBImpl) this.routerStoreToken).getProto().equals(builder.getRouterStoreToken())) { + builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); + } + } + + @Override + public RouterStoreToken getRouterStoreToken() { + RouterRemoveStoredTokenResponseProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerStoreToken != null) { + return this.routerStoreToken; + } + if (!p.hasRouterStoreToken()) { + return null; + } + this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken()); + return this.routerStoreToken; + } + + @Override + public void setRouterStoreToken(RouterStoreToken storeToken) { + maybeInitBuilder(); + if (storeToken == null) { + builder.clearRouterStoreToken(); + } + this.routerStoreToken = storeToken; + } + + private RouterStoreTokenProto convertToProtoFormat(RouterStoreToken storeToken) { + return ((RouterStoreTokenPBImpl) storeToken).getProto(); + } + + private RouterStoreToken convertFromProtoFormat(RouterStoreTokenProto storeTokenProto) { + return new RouterStoreTokenPBImpl(storeTokenProto); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenRequestPBImpl.java new file mode 100644 index 0000000000000..0dafebc1c4a33 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenRequestPBImpl.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreNewTokenRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreNewTokenRequestProtoOrBuilder; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; + +public class RouterStoreNewTokenRequestPBImpl extends RouterStoreNewTokenRequest { + + private RouterStoreNewTokenRequestProto proto = RouterStoreNewTokenRequestProto.getDefaultInstance(); + private RouterStoreNewTokenRequestProto.Builder builder = null; + private boolean viaProto = false; + private RouterStoreToken routerStoreToken = null; + + public RouterStoreNewTokenRequestPBImpl() { + builder = RouterStoreNewTokenRequestProto.newBuilder(); + } + + public RouterStoreNewTokenRequestPBImpl(RouterStoreNewTokenRequestProto requestProto) { + this.proto = requestProto; + viaProto = true; + } + + public RouterStoreNewTokenRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RouterStoreNewTokenRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.routerStoreToken != null && + !((RouterStoreTokenPBImpl) this.routerStoreToken).getProto().equals(builder.getRouterStoreToken())) { + builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); + } + } + + @Override + public RouterStoreToken getRouterStoreToken() { + RouterStoreNewTokenRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerStoreToken != null) { + return this.routerStoreToken; + } + if (!p.hasRouterStoreToken()) { + return null; + } + this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken()); + return this.routerStoreToken; + } + + @Override + public void setRouterStoreToken(RouterStoreToken storeToken) { + maybeInitBuilder(); + if (storeToken == null) { + builder.clearRouterStoreToken(); + } + this.routerStoreToken = storeToken; + this.builder.setRouterStoreToken(convertToProtoFormat(storeToken)); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + private RouterStoreTokenProto convertToProtoFormat(RouterStoreToken storeToken) { + return ((RouterStoreTokenPBImpl) storeToken).getProto(); + } + + private RouterStoreToken convertFromProtoFormat(RouterStoreTokenProto storeTokenProto) { + return new RouterStoreTokenPBImpl(storeTokenProto); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenResponsePBImpl.java new file mode 100644 index 0000000000000..0a33705029cec --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenResponsePBImpl.java @@ -0,0 +1,110 @@ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.classification.InterfaceAudience.Private; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreNewTokenResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreNewTokenResponseProtoOrBuilder; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; + +@Private +@Unstable +public class RouterStoreNewTokenResponsePBImpl extends RouterStoreNewTokenResponse { + + private RouterStoreNewTokenResponseProto proto = RouterStoreNewTokenResponseProto.getDefaultInstance(); + private RouterStoreNewTokenResponseProto.Builder builder = null; + private boolean viaProto = false; + private RouterStoreToken routerStoreToken = null; + + public RouterStoreNewTokenResponsePBImpl() { + builder = RouterStoreNewTokenResponseProto.newBuilder(); + } + + public RouterStoreNewTokenResponsePBImpl(RouterStoreNewTokenResponseProto requestProto) { + this.proto = requestProto; + viaProto = true; + } + + public RouterStoreNewTokenResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RouterStoreNewTokenResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.routerStoreToken != null && + !((RouterStoreTokenPBImpl) this.routerStoreToken).getProto().equals(builder.getRouterStoreToken())) { + builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); + } + } + + @Override + public RouterStoreToken getRouterStoreToken() { + RouterStoreNewTokenResponseProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerStoreToken != null) { + return this.routerStoreToken; + } + if (!p.hasRouterStoreToken()) { + return null; + } + this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken()); + return this.routerStoreToken; + } + + @Override + public void setRouterStoreToken(RouterStoreToken storeToken) { + maybeInitBuilder(); + if (storeToken == null) { + builder.clearRouterStoreToken(); + } + this.routerStoreToken = storeToken; + } + + private RouterStoreTokenProto convertToProtoFormat(RouterStoreToken storeToken) { + return ((RouterStoreTokenPBImpl) storeToken).getProto(); + } + + private RouterStoreToken convertFromProtoFormat(RouterStoreTokenProto storeTokenProto) { + return new RouterStoreTokenPBImpl(storeTokenProto); + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java index 60668bd917747..7003a1b702bb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java @@ -44,6 +44,9 @@ public class RouterStoreTokenPBImpl extends RouterStoreToken { private boolean viaProto = false; + private YARNDelegationTokenIdentifier rMDelegationTokenIdentifier = null; + private Long renewDate; + public RouterStoreTokenPBImpl() { builder = RouterStoreTokenProto.newBuilder(); } @@ -54,11 +57,31 @@ public RouterStoreTokenPBImpl(RouterStoreTokenProto storeTokenProto) { } public RouterStoreTokenProto getProto() { + mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; return proto; } + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (this.rMDelegationTokenIdentifier != null + && !(this.rMDelegationTokenIdentifier.getProto().equals(builder.getTokenIdentifier()))) { + builder.setTokenIdentifier(convertToProtoFormat(this.rMDelegationTokenIdentifier)); + } + if (this.renewDate != null) { + builder.setRenewDate(this.renewDate); + } + } + private void maybeInitBuilder() { if (viaProto || builder == null) { builder = RouterStoreTokenProto.newBuilder(proto); @@ -87,19 +110,33 @@ public String toString() { } @Override - public RMDelegationTokenIdentifier getTokenIdentifier() throws IOException { + public YARNDelegationTokenIdentifier getTokenIdentifier() throws IOException { RouterStoreTokenProtoOrBuilder p = viaProto ? proto : builder; + if (rMDelegationTokenIdentifier != null) { + return rMDelegationTokenIdentifier; + } + if(!p.hasTokenIdentifier()){ + return null; + } YARNDelegationTokenIdentifierProto identifierProto = p.getTokenIdentifier(); ByteArrayInputStream in = new ByteArrayInputStream(identifierProto.toByteArray()); RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(); identifier.readFields(new DataInputStream(in)); + this.rMDelegationTokenIdentifier = identifier; return identifier; } @Override public Long getRenewDate() { RouterStoreTokenProtoOrBuilder p = viaProto ? proto : builder; - return p.getRenewDate(); + if (this.renewDate != null) { + return this.renewDate; + } + if (!p.hasRenewDate()) { + return null; + } + this.renewDate = p.getRenewDate(); + return this.renewDate; } @Override @@ -109,7 +146,8 @@ public void setIdentifier(YARNDelegationTokenIdentifier identifier) { builder.clearTokenIdentifier(); return; } - builder.setTokenIdentifier(identifier.getProto()); + this.rMDelegationTokenIdentifier = identifier; + this.builder.setTokenIdentifier(identifier.getProto()); } @Override @@ -119,6 +157,12 @@ public void setRenewDate(Long renewDate) { builder.clearRenewDate(); return; } - builder.setRenewDate(renewDate); + this.renewDate = renewDate; + this.builder.setRenewDate(renewDate); + } + + private YARNDelegationTokenIdentifierProto convertToProtoFormat( + YARNDelegationTokenIdentifier rMDelegationTokenIdentifier) { + return rMDelegationTokenIdentifier.getProto(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenRequestPBImpl.java new file mode 100644 index 0000000000000..3f90c00601761 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenRequestPBImpl.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterUpdateStoredTokenRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterUpdateStoredTokenRequestProtoOrBuilder; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; +import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenRequest; + +public class RouterUpdateStoredTokenRequestPBImpl extends RouterUpdateStoredTokenRequest { + + private RouterUpdateStoredTokenRequestProto proto = RouterUpdateStoredTokenRequestProto.getDefaultInstance(); + private RouterUpdateStoredTokenRequestProto.Builder builder = null; + private boolean viaProto = false; + private RouterStoreToken routerStoreToken = null; + + public RouterUpdateStoredTokenRequestPBImpl() { + builder = RouterUpdateStoredTokenRequestProto.newBuilder(); + } + + public RouterUpdateStoredTokenRequestPBImpl(RouterUpdateStoredTokenRequestProto requestProto) { + this.proto = requestProto; + viaProto = true; + } + + public RouterUpdateStoredTokenRequestProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RouterUpdateStoredTokenRequestProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.routerStoreToken != null && + !((RouterStoreTokenPBImpl) this.routerStoreToken).getProto().equals(builder.getRouterStoreToken())) { + builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); + } + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public RouterStoreToken getRouterStoreToken() { + RouterUpdateStoredTokenRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerStoreToken != null) { + return this.routerStoreToken; + } + if (!p.hasRouterStoreToken()) { + return null; + } + this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken()); + return this.routerStoreToken; + } + + @Override + public void setRouterStoreToken(RouterStoreToken storeToken) { + maybeInitBuilder(); + if (storeToken == null) { + builder.clearRouterStoreToken(); + } + this.routerStoreToken = storeToken; + this.builder.setRouterStoreToken(convertToProtoFormat(storeToken)); + } + + private RouterStoreTokenProto convertToProtoFormat(RouterStoreToken storeToken) { + return ((RouterStoreTokenPBImpl) storeToken).getProto(); + } + + private RouterStoreToken convertFromProtoFormat(RouterStoreTokenProto storeTokenProto) { + return new RouterStoreTokenPBImpl(storeTokenProto); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenResponsePBImpl.java new file mode 100644 index 0000000000000..f4884ffaea088 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenResponsePBImpl.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; + +import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterUpdateStoredTokenResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterUpdateStoredTokenResponseProtoOrBuilder; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; +import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenRequest; + +public class RouterUpdateStoredTokenResponsePBImpl extends RouterUpdateStoredTokenRequest { + + private RouterUpdateStoredTokenResponseProto proto = RouterUpdateStoredTokenResponseProto.getDefaultInstance(); + private RouterUpdateStoredTokenResponseProto.Builder builder = null; + private boolean viaProto = false; + private RouterStoreToken routerStoreToken = null; + + public RouterUpdateStoredTokenResponsePBImpl() { + builder = RouterUpdateStoredTokenResponseProto.newBuilder(); + } + + public RouterUpdateStoredTokenResponsePBImpl(RouterUpdateStoredTokenResponseProto requestProto) { + this.proto = requestProto; + viaProto = true; + } + + public RouterUpdateStoredTokenResponseProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; + } + + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void maybeInitBuilder() { + if (viaProto || builder == null) { + builder = RouterUpdateStoredTokenResponseProto.newBuilder(proto); + } + viaProto = false; + } + + private void mergeLocalToBuilder() { + if (this.routerStoreToken != null && + !((RouterStoreTokenPBImpl) this.routerStoreToken).getProto().equals(builder.getRouterStoreToken())) { + builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); + } + } + + @Override + public int hashCode() { + return getProto().hashCode(); + } + + @Override + public boolean equals(Object other) { + if (other == null) { + return false; + } + if (other.getClass().isAssignableFrom(this.getClass())) { + return this.getProto().equals(this.getClass().cast(other).getProto()); + } + return false; + } + + @Override + public String toString() { + return TextFormat.shortDebugString(getProto()); + } + + @Override + public RouterStoreToken getRouterStoreToken() { + RouterUpdateStoredTokenResponseProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerStoreToken != null) { + return this.routerStoreToken; + } + if (!p.hasRouterStoreToken()) { + return null; + } + this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken()); + return this.routerStoreToken; + } + + @Override + public void setRouterStoreToken(RouterStoreToken storeToken) { + maybeInitBuilder(); + if (storeToken == null) { + builder.clearRouterStoreToken(); + } + this.routerStoreToken = storeToken; + this.builder.setRouterStoreToken(convertToProtoFormat(storeToken)); + } + + private RouterStoreTokenProto convertToProtoFormat(RouterStoreToken storeToken) { + return ((RouterStoreTokenPBImpl) storeToken).getProto(); + } + + private RouterStoreToken convertFromProtoFormat(RouterStoreTokenProto storeTokenProto) { + return new RouterStoreTokenPBImpl(storeTokenProto); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto index bd12bf63be6a3..9c0c387503b9a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto @@ -26,7 +26,6 @@ package hadoop.yarn; import "yarn_protos.proto"; import "yarn_server_common_protos.proto"; import "yarn_security_token.proto"; -import "RpcHeader.proto"; message SubClusterIdProto { optional string id = 1; @@ -246,4 +245,24 @@ message RouterStoreTokenProto { message RouterStoreNewTokenRequestProto { optional RouterStoreTokenProto router_store_token = 1; -} \ No newline at end of file +} + +message RouterStoreNewTokenResponseProto { + optional RouterStoreTokenProto router_store_token = 1; +} + +message RouterUpdateStoredTokenRequestProto { + optional RouterStoreTokenProto router_store_token = 1; +} + +message RouterUpdateStoredTokenResponseProto { + optional RouterStoreTokenProto router_store_token = 1; +} + +message RouterRemoveStoredTokenRequestProto { + optional RouterStoreTokenProto router_store_token = 1; +} + +message RouterRemoveStoredTokenResponseProto { + optional RouterStoreTokenProto router_store_token = 1; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java index 85b619c48512e..f441f0a44c84d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java @@ -19,6 +19,7 @@ import org.apache.hadoop.yarn.api.BasePBImplRecordsTest; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterRequestProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.AddApplicationHomeSubClusterResponseProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.DeleteApplicationHomeSubClusterRequestProto; @@ -48,6 +49,17 @@ import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterRequestProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterResponseProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RemoveStoredMasterKeyRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RemoveStoredMasterKeyResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreNewTokenRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreNewTokenResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterUpdateStoredTokenRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterUpdateStoredTokenResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRemoveStoredTokenRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRemoveStoredTokenResponseProto; +import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.AddApplicationHomeSubClusterRequestPBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.AddApplicationHomeSubClusterResponsePBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.DeleteApplicationHomeSubClusterRequestPBImpl; @@ -76,7 +88,18 @@ import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterRegisterResponsePBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.UpdateApplicationHomeSubClusterRequestPBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.UpdateApplicationHomeSubClusterResponsePBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterMasterKeyPBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.StoreNewMasterKeyRequestPBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.StoreNewMasterKeyResponsePBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RemoveStoredMasterKeyRequestPBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RemoveStoredMasterKeyResponsePBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterStoreTokenPBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterStoreNewTokenRequestPBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterStoreNewTokenResponsePBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterUpdateStoredTokenRequestPBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterUpdateStoredTokenResponsePBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterRemoveStoredTokenRequestPBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterRemoveStoredTokenResponsePBImpl; import org.apache.hadoop.yarn.server.records.Version; import org.junit.BeforeClass; import org.junit.Test; @@ -94,6 +117,9 @@ public static void setup() throws Exception { generateByNewInstance(SubClusterInfo.class); generateByNewInstance(ApplicationHomeSubCluster.class); generateByNewInstance(SubClusterPolicyConfiguration.class); + generateByNewInstance(RouterMasterKey.class); + generateByNewInstance(YARNDelegationTokenIdentifier.class); + generateByNewInstance(RouterStoreToken.class); } @Test @@ -265,8 +291,74 @@ public void testGetSubClusterPoliciesConfigurationsResponse() GetSubClusterPoliciesConfigurationsResponseProto.class); } + @Test + public void testRouterMasterKey() throws Exception { + validatePBImplRecord(RouterMasterKeyPBImpl.class, RouterMasterKeyProto.class); + } + @Test public void testStoreNewMasterKeyRequest() throws Exception { + validatePBImplRecord(StoreNewMasterKeyRequestPBImpl.class, + StoreNewMasterKeyRequestProto.class); + } + + @Test + public void testStoreNewMasterKeyResponse() throws Exception { + validatePBImplRecord(StoreNewMasterKeyResponsePBImpl.class, + StoreNewMasterKeyResponseProto.class); + } + + @Test + public void testRemoveStoredMasterKeyRequest() throws Exception { + validatePBImplRecord(RemoveStoredMasterKeyRequestPBImpl.class, + RemoveStoredMasterKeyRequestProto.class); + } + + @Test + public void testRemoveStoredMasterKeyResponse() throws Exception { + validatePBImplRecord(RemoveStoredMasterKeyResponsePBImpl.class, + RemoveStoredMasterKeyResponseProto.class); + } + + @Test + public void testRouterStoreToken() throws Exception { + validatePBImplRecord(RouterStoreTokenPBImpl.class, + YarnServerFederationProtos.RouterStoreTokenProto.class); + } + @Test + public void testRouterStoreNewTokenRequest() throws Exception { + validatePBImplRecord(RouterStoreNewTokenRequestPBImpl.class, + RouterStoreNewTokenRequestProto.class); + } + + @Test + public void testRouterStoreNewTokenResponse() throws Exception { + validatePBImplRecord(RouterStoreNewTokenResponsePBImpl.class, + RouterStoreNewTokenResponseProto.class); + } + + @Test + public void testRouterUpdateStoredTokenRequest() throws Exception { + validatePBImplRecord(RouterUpdateStoredTokenRequestPBImpl.class, + RouterUpdateStoredTokenRequestProto.class); + } + + @Test + public void testRouterUpdateStoredTokenResponse() throws Exception { + validatePBImplRecord(RouterUpdateStoredTokenResponsePBImpl.class, + RouterUpdateStoredTokenResponseProto.class); + } + + @Test + public void testRouterRemoveStoredTokenRequest() throws Exception { + validatePBImplRecord(RouterRemoveStoredTokenRequestPBImpl.class, + RouterRemoveStoredTokenRequestProto.class); + } + + @Test + public void testRouterRemoveStoredTokenResponse() throws Exception { + validatePBImplRecord(RouterRemoveStoredTokenResponsePBImpl.class, + RouterRemoveStoredTokenResponseProto.class); } } From 0ca4e0da12edb2557d5eab0778e7c4ee4d2a79d3 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Mon, 15 Aug 2022 15:58:58 -0700 Subject: [PATCH 07/12] YARN-9708. Fix CheckStyle. --- .../YARNDelegationTokenIdentifierPBImpl.java | 3 +- .../security/client/impl/pb/package-info.java | 2 +- .../FederationDelegationTokenStateStore.java | 31 +++++------- .../impl/MemoryFederationStateStore.java | 50 ++++++++----------- .../store/impl/SQLFederationStateStore.java | 31 ++++-------- .../impl/ZookeeperFederationStateStore.java | 31 ++++-------- .../store/records/RouterMasterKeyRequest.java | 43 ++++++++++++++++ .../records/RouterMasterKeyResponse.java | 44 ++++++++++++++++ .../store/records/RouterRMTokenRequest.java | 43 ++++++++++++++++ .../store/records/RouterRMTokenResponse.java | 43 ++++++++++++++++ .../RemoveStoredMasterKeyRequestPBImpl.java | 5 +- .../RouterRemoveStoredTokenRequestPBImpl.java | 23 +++++---- .../pb/RouterStoreNewTokenRequestPBImpl.java | 23 +++++---- .../impl/pb/RouterStoreTokenPBImpl.java | 13 +++-- .../RouterUpdateStoredTokenRequestPBImpl.java | 13 +++-- .../utils/FederationStateStoreFacade.java | 39 ++++++++++++--- .../proto/yarn_server_federation_protos.proto | 8 +++ 17 files changed, 317 insertions(+), 128 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyResponse.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenRequest.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenResponse.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java index d7b25463fff3d..aea2d0eb7c29d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/YARNDelegationTokenIdentifierPBImpl.java @@ -179,7 +179,7 @@ public int getMasterKeyId() { @Override public Text getKind() { - return null; + return null; } @Override @@ -191,7 +191,6 @@ public boolean equals(Object other) { return this.getProto().equals(this.getClass().cast(other).getProto()); } return false; - } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java index ce2fd41dccd86..a481ba4197a6e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java index 9dc0f09d19778..17ee028431070 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java @@ -18,16 +18,10 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; -import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; /** * FederationDelegationTokenStateStore maintains the state of all @@ -68,7 +62,7 @@ public interface FederationDelegationTokenStateStore { * @return StoreNewMasterKeyResponse. * @throws Exception exception occurred. */ - StoreNewMasterKeyResponse storeNewMasterKey(StoreNewMasterKeyRequest request) throws Exception; + RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) throws Exception; /** * The Router Supports Remove the master key. @@ -77,35 +71,34 @@ public interface FederationDelegationTokenStateStore { * @return RemoveStoredMasterKeyResponse. * @throws Exception exception occurred. */ - RemoveStoredMasterKeyResponse removeStoredMasterKey(RemoveStoredMasterKeyRequest request) - throws Exception; + RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) throws Exception; /** * The Router Supports Store new Token. * * @param request DelegationKey. - * @return RouterStoreNewTokenResponse. + * @return RouterRMTokenResponse. * @throws Exception exception occurred. */ - RouterStoreNewTokenResponse storeNewToken(RouterStoreNewTokenRequest request) throws Exception; + RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws Exception; /** * The Router Supports Update Token. * * @param request DelegationKey. - * @return RouterUpdateStoredTokenResponse. + * @return RouterRMTokenResponse. * @throws Exception exception occurred. */ - RouterUpdateStoredTokenResponse updateStoredToken(RouterUpdateStoredTokenRequest request) + RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) throws Exception; /** * The Router Supports Remove Token. * * @param request DelegationKey. - * @return RouterRemoveStoredTokenResponse. + * @return RouterRMTokenResponse. * @throws Exception exception occurred. */ - RouterRemoveStoredTokenResponse removeStoredToken(RouterRemoveStoredTokenRequest request) + RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws Exception; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java index 32ab915686390..d26e3a654287d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java @@ -71,12 +71,10 @@ import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyRequest; -import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyResponse; -import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; import org.apache.hadoop.yarn.server.federation.store.records.RouterRMDTSecretManagerState; import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; @@ -86,10 +84,6 @@ import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils; -import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenResponse; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.util.MonotonicClock; import org.slf4j.Logger; @@ -387,8 +381,8 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( } @Override - public StoreNewMasterKeyResponse storeNewMasterKey( - StoreNewMasterKeyRequest request) throws Exception { + public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) + throws Exception { // Restore the DelegationKey from the request RouterMasterKey masterKey = request.getRouterMasterKey(); @@ -409,12 +403,12 @@ public StoreNewMasterKeyResponse storeNewMasterKey( LOG.info("Store Router-RMDT master key with key id: {}. Currently rmDTMasterKeyState size: {}", delegationKey.getKeyId(), rmDTMasterKeyState.size()); - return StoreNewMasterKeyResponse.newInstance(masterKey); + return RouterMasterKeyResponse.newInstance(masterKey); } @Override - public RemoveStoredMasterKeyResponse removeStoredMasterKey( - RemoveStoredMasterKeyRequest request) throws Exception { + public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) + throws Exception { // Restore the DelegationKey from the request RouterMasterKey masterKey = request.getRouterMasterKey(); @@ -428,37 +422,37 @@ public RemoveStoredMasterKeyResponse removeStoredMasterKey( Set rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); rmDTMasterKeyState.remove(delegationKey); - return RemoveStoredMasterKeyResponse.newInstance(masterKey); + return RouterMasterKeyResponse.newInstance(masterKey); } @Override - public RouterStoreNewTokenResponse storeNewToken( - RouterStoreNewTokenRequest request) throws Exception { + public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws Exception { RouterStoreToken storeToken = request.getRouterStoreToken(); - RMDelegationTokenIdentifier tokenIdentifier = (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); + RMDelegationTokenIdentifier tokenIdentifier = + (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); Long renewDate = storeToken.getRenewDate(); storeOrUpdateRouterRMDT(tokenIdentifier, renewDate, false); - return RouterStoreNewTokenResponse.newInstance(storeToken); + return RouterRMTokenResponse.newInstance(storeToken); } @Override - public RouterUpdateStoredTokenResponse updateStoredToken( - RouterUpdateStoredTokenRequest request) throws Exception { + public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) throws Exception { RouterStoreToken storeToken = request.getRouterStoreToken(); - RMDelegationTokenIdentifier tokenIdentifier = (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); + RMDelegationTokenIdentifier tokenIdentifier = + (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); Long renewDate = storeToken.getRenewDate(); storeOrUpdateRouterRMDT(tokenIdentifier, renewDate, true); - return RouterUpdateStoredTokenResponse.newInstance(storeToken); + return RouterRMTokenResponse.newInstance(storeToken); } @Override - public RouterRemoveStoredTokenResponse removeStoredToken( - RouterRemoveStoredTokenRequest request) throws Exception { + public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws Exception { RouterStoreToken storeToken = request.getRouterStoreToken(); - RMDelegationTokenIdentifier tokenIdentifier = (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); + RMDelegationTokenIdentifier tokenIdentifier = + (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); Map rmDTState = routerRMSecretManagerState.getTokenState(); rmDTState.remove(tokenIdentifier); - return RouterRemoveStoredTokenResponse.newInstance(storeToken); + return RouterRMTokenResponse.newInstance(storeToken); } private void storeOrUpdateRouterRMDT(RMDelegationTokenIdentifier rmDTIdentifier, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java index d02f8562cc09a..6725ab74076da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java @@ -74,20 +74,14 @@ import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterResponse; import org.apache.hadoop.yarn.server.federation.store.records.GetReservationsHomeSubClusterRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyRequest; -import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyResponse; -import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; import org.apache.hadoop.yarn.server.federation.store.utils.FederationApplicationHomeSubClusterStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils; -import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.util.Clock; import org.apache.hadoop.yarn.util.MonotonicClock; @@ -1039,32 +1033,29 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( } @Override - public StoreNewMasterKeyResponse storeNewMasterKey( - StoreNewMasterKeyRequest request) throws Exception{ + public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) + throws Exception { throw new NotImplementedException("Code is not implemented"); } @Override - public RemoveStoredMasterKeyResponse removeStoredMasterKey( - RemoveStoredMasterKeyRequest request) throws Exception { + public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) + throws Exception { throw new NotImplementedException("Code is not implemented"); } @Override - public RouterStoreNewTokenResponse storeNewToken( - RouterStoreNewTokenRequest request) throws Exception { + public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws Exception { throw new NotImplementedException("Code is not implemented"); } @Override - public RouterUpdateStoredTokenResponse updateStoredToken( - RouterUpdateStoredTokenRequest request) throws Exception { + public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) throws Exception { throw new NotImplementedException("Code is not implemented"); } @Override - public RouterRemoveStoredTokenResponse removeStoredToken( - RouterRemoveStoredTokenRequest request) throws Exception { + public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws Exception { throw new NotImplementedException("Code is not implemented"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java index 8e2520fe982bc..c8a3a99ea5c61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java @@ -75,20 +75,14 @@ import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterIdPBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterInfoPBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterPolicyConfigurationPBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; import org.apache.hadoop.yarn.server.federation.store.utils.FederationApplicationHomeSubClusterStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator; import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils; -import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyRequest; -import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyResponse; -import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenResponse; import org.apache.hadoop.yarn.server.records.Version; import org.apache.zookeeper.data.ACL; import org.slf4j.Logger; @@ -674,32 +668,29 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( } @Override - public StoreNewMasterKeyResponse storeNewMasterKey( - StoreNewMasterKeyRequest request) throws Exception{ + public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) + throws Exception{ throw new NotImplementedException("Code is not implemented"); } @Override - public RemoveStoredMasterKeyResponse removeStoredMasterKey( - RemoveStoredMasterKeyRequest request) throws Exception { + public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) + throws Exception { throw new NotImplementedException("Code is not implemented"); } @Override - public RouterStoreNewTokenResponse storeNewToken( - RouterStoreNewTokenRequest request) throws Exception { + public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws Exception { throw new NotImplementedException("Code is not implemented"); } @Override - public RouterUpdateStoredTokenResponse updateStoredToken( - RouterUpdateStoredTokenRequest request) throws Exception { + public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) throws Exception { throw new NotImplementedException("Code is not implemented"); } @Override - public RouterRemoveStoredTokenResponse removeStoredToken( - RouterRemoveStoredTokenRequest request) throws Exception { + public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws Exception { throw new NotImplementedException("Code is not implemented"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyRequest.java new file mode 100644 index 0000000000000..2bf30535162d5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyRequest.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterMasterKeyRequest { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterMasterKeyRequest newInstance(RouterMasterKey routerMasterKey) { + RouterMasterKeyRequest request = Records.newRecord(RouterMasterKeyRequest.class); + request.setRouterMasterKey(routerMasterKey); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterMasterKey getRouterMasterKey(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterMasterKey(RouterMasterKey routerMasterKey); + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyResponse.java new file mode 100644 index 0000000000000..6625e4c1911d8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterMasterKeyResponse.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterMasterKeyResponse { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterMasterKeyResponse newInstance(RouterMasterKey masterKey) { + RouterMasterKeyResponse request = + Records.newRecord(RouterMasterKeyResponse.class); + request.setRouterMasterKey(masterKey); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterMasterKey getRouterMasterKey(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterMasterKey(RouterMasterKey masterKey); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenRequest.java new file mode 100644 index 0000000000000..0d4908de3539f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenRequest.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterRMTokenRequest { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterRMTokenRequest newInstance(RouterStoreToken routerStoreToken) { + RouterRMTokenRequest request = Records.newRecord(RouterRMTokenRequest.class); + request.setRouterStoreToken(routerStoreToken); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterStoreToken getRouterStoreToken(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenResponse.java new file mode 100644 index 0000000000000..57671bda419f4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRMTokenResponse.java @@ -0,0 +1,43 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.federation.store.records; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.util.Records; + +@InterfaceAudience.Private +@InterfaceStability.Unstable +public abstract class RouterRMTokenResponse { + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public static RouterRMTokenResponse newInstance(RouterStoreToken routerStoreToken) { + RouterRMTokenResponse request = Records.newRecord(RouterRMTokenResponse.class); + request.setRouterStoreToken(routerStoreToken); + return request; + } + + @InterfaceAudience.Public + @InterfaceStability.Unstable + public abstract RouterStoreToken getRouterStoreToken(); + + @InterfaceAudience.Private + @InterfaceStability.Unstable + public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyRequestPBImpl.java index 6f3f119082d89..10db1aa700ab0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyRequestPBImpl.java @@ -29,11 +29,10 @@ @Unstable public class RemoveStoredMasterKeyRequestPBImpl extends RemoveStoredMasterKeyRequest { - private RemoveStoredMasterKeyRequestProto proto = RemoveStoredMasterKeyRequestProto.getDefaultInstance(); - + private RemoveStoredMasterKeyRequestProto proto = + RemoveStoredMasterKeyRequestProto.getDefaultInstance(); private RemoveStoredMasterKeyRequestProto.Builder builder = null; private boolean viaProto = false; - private RouterMasterKey routerMasterKey = null; public RemoveStoredMasterKeyRequestPBImpl() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenRequestPBImpl.java index c8a9a280d30d3..6a841a3bccc21 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenRequestPBImpl.java @@ -26,7 +26,8 @@ public class RouterRemoveStoredTokenRequestPBImpl extends RouterStoreNewTokenRequest { - private RouterRemoveStoredTokenRequestProto proto = RouterRemoveStoredTokenRequestProto.getDefaultInstance(); + private RouterRemoveStoredTokenRequestProto proto = + RouterRemoveStoredTokenRequestProto.getDefaultInstance(); private RouterRemoveStoredTokenRequestProto.Builder builder = null; private boolean viaProto = false; private RouterStoreToken routerStoreToken = null; @@ -58,15 +59,19 @@ private void mergeLocalToProto() { private void maybeInitBuilder() { if (viaProto || builder == null) { - builder = RouterRemoveStoredTokenRequestProto.newBuilder(proto); + builder = RouterRemoveStoredTokenRequestProto.newBuilder(proto); } viaProto = false; } private void mergeLocalToBuilder() { - if (this.routerStoreToken != null && - !((RouterStoreTokenPBImpl) this.routerStoreToken).getProto().equals(builder.getRouterStoreToken())) { + if (this.routerStoreToken != null) { + RouterStoreTokenPBImpl routerStoreTokenPBImpl = + (RouterStoreTokenPBImpl) this.routerStoreToken; + RouterStoreTokenProto storeTokenProto = routerStoreTokenPBImpl.getProto(); + if (!storeTokenProto.equals(builder.getRouterStoreToken())) { builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); + } } } @@ -74,10 +79,10 @@ private void mergeLocalToBuilder() { public RouterStoreToken getRouterStoreToken() { RouterRemoveStoredTokenRequestProtoOrBuilder p = viaProto ? proto : builder; if (this.routerStoreToken != null) { - return this.routerStoreToken; + return this.routerStoreToken; } if (!p.hasRouterStoreToken()) { - return null; + return null; } this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken()); return this.routerStoreToken; @@ -87,7 +92,7 @@ public RouterStoreToken getRouterStoreToken() { public void setRouterStoreToken(RouterStoreToken storeToken) { maybeInitBuilder(); if (storeToken == null) { - builder.clearRouterStoreToken(); + builder.clearRouterStoreToken(); } this.routerStoreToken = storeToken; this.builder.setRouterStoreToken(convertToProtoFormat(storeToken)); @@ -101,10 +106,10 @@ public int hashCode() { @Override public boolean equals(Object other) { if (other == null) { - return false; + return false; } if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); + return this.getProto().equals(this.getClass().cast(other).getProto()); } return false; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenRequestPBImpl.java index 0dafebc1c4a33..72f6624d8b2bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenRequestPBImpl.java @@ -26,7 +26,8 @@ public class RouterStoreNewTokenRequestPBImpl extends RouterStoreNewTokenRequest { - private RouterStoreNewTokenRequestProto proto = RouterStoreNewTokenRequestProto.getDefaultInstance(); + private RouterStoreNewTokenRequestProto proto = + RouterStoreNewTokenRequestProto.getDefaultInstance(); private RouterStoreNewTokenRequestProto.Builder builder = null; private boolean viaProto = false; private RouterStoreToken routerStoreToken = null; @@ -58,15 +59,19 @@ private void mergeLocalToProto() { private void maybeInitBuilder() { if (viaProto || builder == null) { - builder = RouterStoreNewTokenRequestProto.newBuilder(proto); + builder = RouterStoreNewTokenRequestProto.newBuilder(proto); } viaProto = false; } private void mergeLocalToBuilder() { - if (this.routerStoreToken != null && - !((RouterStoreTokenPBImpl) this.routerStoreToken).getProto().equals(builder.getRouterStoreToken())) { + if (this.routerStoreToken != null) { + RouterStoreTokenPBImpl routerStoreTokenPBImpl = + (RouterStoreTokenPBImpl) this.routerStoreToken; + RouterStoreTokenProto storeTokenProto = routerStoreTokenPBImpl.getProto(); + if (!storeTokenProto.equals(builder.getRouterStoreToken())) { builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); + } } } @@ -74,10 +79,10 @@ private void mergeLocalToBuilder() { public RouterStoreToken getRouterStoreToken() { RouterStoreNewTokenRequestProtoOrBuilder p = viaProto ? proto : builder; if (this.routerStoreToken != null) { - return this.routerStoreToken; + return this.routerStoreToken; } if (!p.hasRouterStoreToken()) { - return null; + return null; } this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken()); return this.routerStoreToken; @@ -87,7 +92,7 @@ public RouterStoreToken getRouterStoreToken() { public void setRouterStoreToken(RouterStoreToken storeToken) { maybeInitBuilder(); if (storeToken == null) { - builder.clearRouterStoreToken(); + builder.clearRouterStoreToken(); } this.routerStoreToken = storeToken; this.builder.setRouterStoreToken(convertToProtoFormat(storeToken)); @@ -101,10 +106,10 @@ public int hashCode() { @Override public boolean equals(Object other) { if (other == null) { - return false; + return false; } if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); + return this.getProto().equals(this.getClass().cast(other).getProto()); } return false; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java index 7003a1b702bb7..4df7e40730531 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreTokenPBImpl.java @@ -73,10 +73,13 @@ private void mergeLocalToProto() { } private void mergeLocalToBuilder() { - if (this.rMDelegationTokenIdentifier != null - && !(this.rMDelegationTokenIdentifier.getProto().equals(builder.getTokenIdentifier()))) { - builder.setTokenIdentifier(convertToProtoFormat(this.rMDelegationTokenIdentifier)); + if (this.rMDelegationTokenIdentifier != null) { + YARNDelegationTokenIdentifierProto idProto = this.rMDelegationTokenIdentifier.getProto(); + if (!idProto.equals(builder.getTokenIdentifier())) { + builder.setTokenIdentifier(convertToProtoFormat(this.rMDelegationTokenIdentifier)); + } } + if (this.renewDate != null) { builder.setRenewDate(this.renewDate); } @@ -162,7 +165,7 @@ public void setRenewDate(Long renewDate) { } private YARNDelegationTokenIdentifierProto convertToProtoFormat( - YARNDelegationTokenIdentifier rMDelegationTokenIdentifier) { - return rMDelegationTokenIdentifier.getProto(); + YARNDelegationTokenIdentifier delegationTokenIdentifier) { + return delegationTokenIdentifier.getProto(); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenRequestPBImpl.java index 3f90c00601761..c35856dbb4991 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenRequestPBImpl.java @@ -26,7 +26,8 @@ public class RouterUpdateStoredTokenRequestPBImpl extends RouterUpdateStoredTokenRequest { - private RouterUpdateStoredTokenRequestProto proto = RouterUpdateStoredTokenRequestProto.getDefaultInstance(); + private RouterUpdateStoredTokenRequestProto proto = + RouterUpdateStoredTokenRequestProto.getDefaultInstance(); private RouterUpdateStoredTokenRequestProto.Builder builder = null; private boolean viaProto = false; private RouterStoreToken routerStoreToken = null; @@ -64,9 +65,13 @@ private void maybeInitBuilder() { } private void mergeLocalToBuilder() { - if (this.routerStoreToken != null && - !((RouterStoreTokenPBImpl) this.routerStoreToken).getProto().equals(builder.getRouterStoreToken())) { - builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); + if (this.routerStoreToken != null) { + RouterStoreTokenPBImpl routerStoreTokenPBImpl = + (RouterStoreTokenPBImpl) this.routerStoreToken; + RouterStoreTokenProto storeTokenProto = routerStoreTokenPBImpl.getProto(); + if (!storeTokenProto.equals(builder.getRouterStoreToken())) { + builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index 9ed1452c89fc2..844b63bb66297 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -53,7 +53,32 @@ import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; import org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreRetriableException; -import org.apache.hadoop.yarn.server.federation.store.records.*; +import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetReservationHomeSubClusterResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse; +import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest; +import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse; +import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration; +import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; +import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -429,7 +454,7 @@ public void storeNewMasterKey(DelegationKey newKey) throws Exception { ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey()); RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(), keyBytes, newKey.getExpiryDate()); - StoreNewMasterKeyRequest keyRequest = StoreNewMasterKeyRequest.newInstance(masterKey); + RouterMasterKeyRequest keyRequest = RouterMasterKeyRequest.newInstance(masterKey); stateStore.storeNewMasterKey(keyRequest); } @@ -444,7 +469,7 @@ public void removeStoredMasterKey(DelegationKey newKey) throws Exception { ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey()); RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(), keyBytes, newKey.getExpiryDate()); - RemoveStoredMasterKeyRequest keyRequest = RemoveStoredMasterKeyRequest.newInstance(masterKey); + RouterMasterKeyRequest keyRequest = RouterMasterKeyRequest.newInstance(masterKey); stateStore.removeStoredMasterKey(keyRequest); } @@ -460,7 +485,7 @@ public void storeNewToken(RMDelegationTokenIdentifier identifier, LOG.info("storing RMDelegation token with sequence number: {}.", identifier.getSequenceNumber()); RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate); - RouterStoreNewTokenRequest request = RouterStoreNewTokenRequest.newInstance(storeToken); + RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken); stateStore.storeNewToken(request); } @@ -476,8 +501,7 @@ public void updateStoredToken(RMDelegationTokenIdentifier identifier, LOG.info("updating RMDelegation token with sequence number: {}.", identifier.getSequenceNumber()); RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate); - RouterUpdateStoredTokenRequest request = - RouterUpdateStoredTokenRequest.newInstance(storeToken); + RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken); stateStore.updateStoredToken(request); } @@ -492,8 +516,7 @@ public void removeStoredToken(RMDelegationTokenIdentifier identifier) LOG.info("removing RMDelegation token with sequence number: {}", identifier.getSequenceNumber()); RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, 0L); - RouterRemoveStoredTokenRequest request = - RouterRemoveStoredTokenRequest.newInstance(storeToken); + RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken); stateStore.removeStoredToken(request); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto index 9c0c387503b9a..d8c9f60ca170f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto @@ -266,3 +266,11 @@ message RouterRemoveStoredTokenRequestProto { message RouterRemoveStoredTokenResponseProto { optional RouterStoreTokenProto router_store_token = 1; } + +message RouterRMTokenRequestProto { + optional RouterStoreTokenProto router_store_token = 1; +} + +message RouterRMTokenResponseProto { + optional RouterStoreTokenProto router_store_token = 1; +} \ No newline at end of file From b97fd231db28b6f07fcb1753373567fd0dd0f65d Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 16 Aug 2022 09:40:50 +0800 Subject: [PATCH 08/12] YARN-9708. Yarn Router Support DelegationToken. --- .../RemoveStoredMasterKeyResponsePBImpl.java | 124 ----------------- ...java => RouterMasterKeyRequestPBImpl.java} | 104 +++++++------- ...ava => RouterMasterKeyResponsePBImpl.java} | 100 +++++++------- ...l.java => RouterRMTokenRequestPBImpl.java} | 25 ++-- ....java => RouterRMTokenResponsePBImpl.java} | 24 ++-- .../RouterRemoveStoredTokenRequestPBImpl.java | 129 ------------------ ...RouterRemoveStoredTokenResponsePBImpl.java | 111 --------------- .../RouterUpdateStoredTokenRequestPBImpl.java | 129 ------------------ ...RouterUpdateStoredTokenResponsePBImpl.java | 124 ----------------- .../pb/StoreNewMasterKeyResponsePBImpl.java | 126 ----------------- .../proto/yarn_server_federation_protos.proto | 36 +---- .../TestFederationProtocolRecords.java | 88 +++--------- .../FederationStateStoreService.java | 31 ++--- 13 files changed, 155 insertions(+), 996 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyResponsePBImpl.java rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/{StoreNewMasterKeyRequestPBImpl.java => RouterMasterKeyRequestPBImpl.java} (61%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/{RemoveStoredMasterKeyRequestPBImpl.java => RouterMasterKeyResponsePBImpl.java} (60%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/{RouterStoreNewTokenRequestPBImpl.java => RouterRMTokenRequestPBImpl.java} (82%) rename hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/{RouterStoreNewTokenResponsePBImpl.java => RouterRMTokenResponsePBImpl.java} (78%) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenRequestPBImpl.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenResponsePBImpl.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenRequestPBImpl.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenResponsePBImpl.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyResponsePBImpl.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyResponsePBImpl.java deleted file mode 100644 index 6f06e76d0fb74..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyResponsePBImpl.java +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; - -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RemoveStoredMasterKeyResponseProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RemoveStoredMasterKeyResponseProtoOrBuilder; -import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; - -public class RemoveStoredMasterKeyResponsePBImpl extends RemoveStoredMasterKeyResponse { - - private RemoveStoredMasterKeyResponseProto proto = - RemoveStoredMasterKeyResponseProto.getDefaultInstance(); - - private RemoveStoredMasterKeyResponseProto.Builder builder = null; - - private boolean viaProto = false; - - private RouterMasterKey routerMasterKey = null; - - public RemoveStoredMasterKeyResponsePBImpl() { - builder = RemoveStoredMasterKeyResponseProto.newBuilder(); - } - - public RemoveStoredMasterKeyResponsePBImpl(RemoveStoredMasterKeyResponseProto responseProto) { - this.proto = responseProto; - viaProto = true; - } - - public RemoveStoredMasterKeyResponseProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void mergeLocalToProto() { - if (viaProto) { - maybeInitBuilder(); - } - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = RemoveStoredMasterKeyResponseProto.newBuilder(proto); - } - viaProto = false; - } - - private void mergeLocalToBuilder() { - if (this.routerMasterKey != null && !((RouterMasterKeyPBImpl) this.routerMasterKey). - getProto().equals(builder.getRouterMasterKey())) { - builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); - } - } - - private YarnServerFederationProtos.RouterMasterKeyProto convertToProtoFormat( - RouterMasterKey masterKey) { - return ((RouterMasterKeyPBImpl) masterKey).getProto(); - } - - - @Override - public RouterMasterKey getRouterMasterKey() { - RemoveStoredMasterKeyResponseProtoOrBuilder p = viaProto ? proto : builder; - if (this.routerMasterKey != null) { - return this.routerMasterKey; - } - if (!p.hasRouterMasterKey()) { - return null; - } - this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey()); - return this.routerMasterKey; - } - - @Override - public void setRouterMasterKey(RouterMasterKey masterKey) { - maybeInitBuilder(); - if (masterKey == null) { - builder.clearRouterMasterKey(); - return; - } - this.routerMasterKey = masterKey; - } - - private RouterMasterKey convertFromProtoFormat(RouterMasterKeyProto masterKey) { - return new RouterMasterKeyPBImpl(masterKey); - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) { - return false; - } - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyRequestPBImpl.java similarity index 61% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyRequestPBImpl.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyRequestPBImpl.java index ff569bee65397..93fcdd8e6166d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyRequestPBImpl.java @@ -1,51 +1,50 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.thirdparty.protobuf.TextFormat; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyRequestProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; -import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyRequestProtoOrBuilder; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyRequestProtoOrBuilder; +import org.apache.hadoop.thirdparty.protobuf.TextFormat; @Private @Unstable -public class StoreNewMasterKeyRequestPBImpl extends StoreNewMasterKeyRequest { - - private StoreNewMasterKeyRequestProto proto = StoreNewMasterKeyRequestProto.getDefaultInstance(); +public class RouterMasterKeyRequestPBImpl extends RouterMasterKeyRequest { - private StoreNewMasterKeyRequestProto.Builder builder = null; + private RouterMasterKeyRequestProto proto = RouterMasterKeyRequestProto.getDefaultInstance(); + private RouterMasterKeyRequestProto.Builder builder = null; private boolean viaProto = false; - private RouterMasterKey routerMasterKey = null; - public StoreNewMasterKeyRequestPBImpl() { - builder = StoreNewMasterKeyRequestProto.newBuilder(); + public RouterMasterKeyRequestPBImpl() { + builder = RouterMasterKeyRequestProto.newBuilder(); } - public StoreNewMasterKeyRequestPBImpl(StoreNewMasterKeyRequestProto proto) { + public RouterMasterKeyRequestPBImpl(RouterMasterKeyRequestProto proto) { this.proto = proto; viaProto = true; } - public StoreNewMasterKeyRequestProto getProto() { + public RouterMasterKeyRequestProto getProto() { mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; @@ -63,18 +62,39 @@ private void mergeLocalToProto() { private void maybeInitBuilder() { if (viaProto || builder == null) { - builder = StoreNewMasterKeyRequestProto.newBuilder(proto); + builder = RouterMasterKeyRequestProto.newBuilder(proto); } viaProto = false; } private void mergeLocalToBuilder() { - RouterMasterKeyPBImpl masterKeyRequest = - (RouterMasterKeyPBImpl) this.routerMasterKey; + RouterMasterKeyPBImpl masterKeyRequest = (RouterMasterKeyPBImpl) this.routerMasterKey; RouterMasterKeyProto routerMasterKeyProto = builder.getRouterMasterKey(); if (this.routerMasterKey != null && !masterKeyRequest.getProto().equals(routerMasterKeyProto)) { - builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); + builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); + } + } + + @Override + public RouterMasterKey getRouterMasterKey() { + RouterMasterKeyRequestProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerMasterKey != null) { + return this.routerMasterKey; + } + if (!p.hasRouterMasterKey()) { + return null; + } + this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey()); + return this.routerMasterKey; + } + + @Override + public void setRouterMasterKey(RouterMasterKey masterKey) { + maybeInitBuilder(); + if (masterKey == null) { + builder.clearRouterMasterKey(); } + this.routerMasterKey = masterKey; } @Override @@ -98,28 +118,6 @@ public String toString() { return TextFormat.shortDebugString(getProto()); } - @Override - public RouterMasterKey getRouterMasterKey() { - StoreNewMasterKeyRequestProtoOrBuilder p = viaProto ? proto : builder; - if (this.routerMasterKey != null) { - return this.routerMasterKey; - } - if (!p.hasRouterMasterKey()) { - return null; - } - this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey()); - return this.routerMasterKey; - } - - @Override - public void setRouterMasterKey(RouterMasterKey masterKey) { - maybeInitBuilder(); - if (masterKey == null) { - builder.clearRouterMasterKey(); - } - this.routerMasterKey = masterKey; - } - private RouterMasterKey convertFromProtoFormat(RouterMasterKeyProto masterKeyProto) { return new RouterMasterKeyPBImpl(masterKeyProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyResponsePBImpl.java similarity index 60% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyRequestPBImpl.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyResponsePBImpl.java index 10db1aa700ab0..3f0f3fbde7e22 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RemoveStoredMasterKeyRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterMasterKeyResponsePBImpl.java @@ -1,18 +1,19 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; @@ -20,31 +21,30 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.thirdparty.protobuf.TextFormat; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RemoveStoredMasterKeyRequestProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RemoveStoredMasterKeyRequestProtoOrBuilder; -import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyRequest; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyResponseProtoOrBuilder; import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; @Private @Unstable -public class RemoveStoredMasterKeyRequestPBImpl extends RemoveStoredMasterKeyRequest { +public class RouterMasterKeyResponsePBImpl extends RouterMasterKeyResponse { - private RemoveStoredMasterKeyRequestProto proto = - RemoveStoredMasterKeyRequestProto.getDefaultInstance(); - private RemoveStoredMasterKeyRequestProto.Builder builder = null; + private RouterMasterKeyResponseProto proto = RouterMasterKeyResponseProto.getDefaultInstance(); + private RouterMasterKeyResponseProto.Builder builder = null; private boolean viaProto = false; private RouterMasterKey routerMasterKey = null; - public RemoveStoredMasterKeyRequestPBImpl() { - builder = RemoveStoredMasterKeyRequestProto.newBuilder(); + public RouterMasterKeyResponsePBImpl() { + builder = RouterMasterKeyResponseProto.newBuilder(); } - public RemoveStoredMasterKeyRequestPBImpl(RemoveStoredMasterKeyRequestProto proto) { + public RouterMasterKeyResponsePBImpl(RouterMasterKeyResponseProto proto) { this.proto = proto; viaProto = true; } - public RemoveStoredMasterKeyRequestProto getProto() { + public RouterMasterKeyResponseProto getProto() { mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; @@ -62,20 +62,41 @@ private void mergeLocalToProto() { private void maybeInitBuilder() { if (viaProto || builder == null) { - builder = RemoveStoredMasterKeyRequestProto.newBuilder(proto); + builder = RouterMasterKeyResponseProto.newBuilder(proto); } viaProto = false; } private void mergeLocalToBuilder() { - RouterMasterKeyPBImpl masterKeyRequest = - (RouterMasterKeyPBImpl) this.routerMasterKey; + RouterMasterKeyPBImpl masterKeyRequest = (RouterMasterKeyPBImpl) this.routerMasterKey; RouterMasterKeyProto routerMasterKeyProto = builder.getRouterMasterKey(); if (this.routerMasterKey != null && !masterKeyRequest.getProto().equals(routerMasterKeyProto)) { - builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); + builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); } } + @Override + public RouterMasterKey getRouterMasterKey() { + RouterMasterKeyResponseProtoOrBuilder p = viaProto ? proto : builder; + if (this.routerMasterKey != null) { + return this.routerMasterKey; + } + if (!p.hasRouterMasterKey()) { + return null; + } + this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey()); + return this.routerMasterKey; + } + + @Override + public void setRouterMasterKey(RouterMasterKey masterKey) { + maybeInitBuilder(); + if (masterKey == null) { + builder.clearRouterMasterKey(); + } + this.routerMasterKey = masterKey; + } + @Override public int hashCode() { return getProto().hashCode(); @@ -97,29 +118,6 @@ public String toString() { return TextFormat.shortDebugString(getProto()); } - @Override - public RouterMasterKey getRouterMasterKey() { - RemoveStoredMasterKeyRequestProtoOrBuilder p = viaProto ? proto : builder; - if (this.routerMasterKey != null) { - return this.routerMasterKey; - } - if (!p.hasRouterMasterKey()) { - return null; - } - this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey()); - return this.routerMasterKey; - } - - @Override - public void setRouterMasterKey(RouterMasterKey masterKey) { - maybeInitBuilder(); - if (masterKey == null) { - builder.clearRouterMasterKey(); - return; - } - this.routerMasterKey = masterKey; - } - private RouterMasterKey convertFromProtoFormat(RouterMasterKeyProto masterKeyProto) { return new RouterMasterKeyPBImpl(masterKeyProto); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenRequestPBImpl.java similarity index 82% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenRequestPBImpl.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenRequestPBImpl.java index 72f6624d8b2bd..756d4745507e0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenRequestPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenRequestPBImpl.java @@ -18,30 +18,29 @@ package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; import org.apache.hadoop.thirdparty.protobuf.TextFormat; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenRequestProtoOrBuilder; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreNewTokenRequestProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreNewTokenRequestProtoOrBuilder; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; -public class RouterStoreNewTokenRequestPBImpl extends RouterStoreNewTokenRequest { +public class RouterRMTokenRequestPBImpl extends RouterRMTokenRequest { - private RouterStoreNewTokenRequestProto proto = - RouterStoreNewTokenRequestProto.getDefaultInstance(); - private RouterStoreNewTokenRequestProto.Builder builder = null; + private RouterRMTokenRequestProto proto = RouterRMTokenRequestProto.getDefaultInstance(); + private RouterRMTokenRequestProto.Builder builder = null; private boolean viaProto = false; private RouterStoreToken routerStoreToken = null; - public RouterStoreNewTokenRequestPBImpl() { - builder = RouterStoreNewTokenRequestProto.newBuilder(); + public RouterRMTokenRequestPBImpl() { + builder = RouterRMTokenRequestProto.newBuilder(); } - public RouterStoreNewTokenRequestPBImpl(RouterStoreNewTokenRequestProto requestProto) { + public RouterRMTokenRequestPBImpl(RouterRMTokenRequestProto requestProto) { this.proto = requestProto; viaProto = true; } - public RouterStoreNewTokenRequestProto getProto() { + public RouterRMTokenRequestProto getProto() { mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; @@ -59,7 +58,7 @@ private void mergeLocalToProto() { private void maybeInitBuilder() { if (viaProto || builder == null) { - builder = RouterStoreNewTokenRequestProto.newBuilder(proto); + builder = RouterRMTokenRequestProto.newBuilder(proto); } viaProto = false; } @@ -77,7 +76,7 @@ private void mergeLocalToBuilder() { @Override public RouterStoreToken getRouterStoreToken() { - RouterStoreNewTokenRequestProtoOrBuilder p = viaProto ? proto : builder; + RouterRMTokenRequestProtoOrBuilder p = viaProto ? proto : builder; if (this.routerStoreToken != null) { return this.routerStoreToken; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java similarity index 78% rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenResponsePBImpl.java rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java index 0a33705029cec..0c87a1a78c432 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterStoreNewTokenResponsePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRMTokenResponsePBImpl.java @@ -3,31 +3,31 @@ import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Unstable; import org.apache.hadoop.thirdparty.protobuf.TextFormat; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreNewTokenResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenResponseProtoOrBuilder; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreNewTokenResponseProtoOrBuilder; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; @Private @Unstable -public class RouterStoreNewTokenResponsePBImpl extends RouterStoreNewTokenResponse { +public class RouterRMTokenResponsePBImpl extends RouterRMTokenResponse { - private RouterStoreNewTokenResponseProto proto = RouterStoreNewTokenResponseProto.getDefaultInstance(); - private RouterStoreNewTokenResponseProto.Builder builder = null; + private RouterRMTokenResponseProto proto = RouterRMTokenResponseProto.getDefaultInstance(); + private RouterRMTokenResponseProto.Builder builder = null; private boolean viaProto = false; private RouterStoreToken routerStoreToken = null; - public RouterStoreNewTokenResponsePBImpl() { - builder = RouterStoreNewTokenResponseProto.newBuilder(); + public RouterRMTokenResponsePBImpl() { + builder = RouterRMTokenResponseProto.newBuilder(); } - public RouterStoreNewTokenResponsePBImpl(RouterStoreNewTokenResponseProto requestProto) { + public RouterRMTokenResponsePBImpl(RouterRMTokenResponseProto requestProto) { this.proto = requestProto; viaProto = true; } - public RouterStoreNewTokenResponseProto getProto() { + public RouterRMTokenResponseProto getProto() { mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; @@ -45,7 +45,7 @@ private void mergeLocalToProto() { private void maybeInitBuilder() { if (viaProto || builder == null) { - builder = RouterStoreNewTokenResponseProto.newBuilder(proto); + builder = RouterRMTokenResponseProto.newBuilder(proto); } viaProto = false; } @@ -59,7 +59,7 @@ private void mergeLocalToBuilder() { @Override public RouterStoreToken getRouterStoreToken() { - RouterStoreNewTokenResponseProtoOrBuilder p = viaProto ? proto : builder; + RouterRMTokenResponseProtoOrBuilder p = viaProto ? proto : builder; if (this.routerStoreToken != null) { return this.routerStoreToken; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenRequestPBImpl.java deleted file mode 100644 index 6a841a3bccc21..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenRequestPBImpl.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; - -import org.apache.hadoop.thirdparty.protobuf.TextFormat; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRemoveStoredTokenRequestProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRemoveStoredTokenRequestProtoOrBuilder; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; - -public class RouterRemoveStoredTokenRequestPBImpl extends RouterStoreNewTokenRequest { - - private RouterRemoveStoredTokenRequestProto proto = - RouterRemoveStoredTokenRequestProto.getDefaultInstance(); - private RouterRemoveStoredTokenRequestProto.Builder builder = null; - private boolean viaProto = false; - private RouterStoreToken routerStoreToken = null; - - public RouterRemoveStoredTokenRequestPBImpl() { - builder = RouterRemoveStoredTokenRequestProto.newBuilder(); - } - - public RouterRemoveStoredTokenRequestPBImpl(RouterRemoveStoredTokenRequestProto requestProto) { - this.proto = requestProto; - viaProto = true; - } - - public RouterRemoveStoredTokenRequestProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void mergeLocalToProto() { - if (viaProto) { - maybeInitBuilder(); - } - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = RouterRemoveStoredTokenRequestProto.newBuilder(proto); - } - viaProto = false; - } - - private void mergeLocalToBuilder() { - if (this.routerStoreToken != null) { - RouterStoreTokenPBImpl routerStoreTokenPBImpl = - (RouterStoreTokenPBImpl) this.routerStoreToken; - RouterStoreTokenProto storeTokenProto = routerStoreTokenPBImpl.getProto(); - if (!storeTokenProto.equals(builder.getRouterStoreToken())) { - builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); - } - } - } - - @Override - public RouterStoreToken getRouterStoreToken() { - RouterRemoveStoredTokenRequestProtoOrBuilder p = viaProto ? proto : builder; - if (this.routerStoreToken != null) { - return this.routerStoreToken; - } - if (!p.hasRouterStoreToken()) { - return null; - } - this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken()); - return this.routerStoreToken; - } - - @Override - public void setRouterStoreToken(RouterStoreToken storeToken) { - maybeInitBuilder(); - if (storeToken == null) { - builder.clearRouterStoreToken(); - } - this.routerStoreToken = storeToken; - this.builder.setRouterStoreToken(convertToProtoFormat(storeToken)); - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) { - return false; - } - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return TextFormat.shortDebugString(getProto()); - } - - private RouterStoreTokenProto convertToProtoFormat(RouterStoreToken storeToken) { - return ((RouterStoreTokenPBImpl) storeToken).getProto(); - } - - private RouterStoreToken convertFromProtoFormat(RouterStoreTokenProto storeTokenProto) { - return new RouterStoreTokenPBImpl(storeTokenProto); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenResponsePBImpl.java deleted file mode 100644 index 19b1266a624b6..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterRemoveStoredTokenResponsePBImpl.java +++ /dev/null @@ -1,111 +0,0 @@ -package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.thirdparty.protobuf.TextFormat; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRemoveStoredTokenResponseProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRemoveStoredTokenResponseProtoOrBuilder; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; - -@Private -@Unstable -public class RouterRemoveStoredTokenResponsePBImpl extends RouterStoreNewTokenResponse { - - private RouterRemoveStoredTokenResponseProto proto = - RouterRemoveStoredTokenResponseProto.getDefaultInstance(); - private RouterRemoveStoredTokenResponseProto.Builder builder = null; - private boolean viaProto = false; - private RouterStoreToken routerStoreToken = null; - - public RouterRemoveStoredTokenResponsePBImpl() { - builder = RouterRemoveStoredTokenResponseProto.newBuilder(); - } - - public RouterRemoveStoredTokenResponsePBImpl(RouterRemoveStoredTokenResponseProto requestProto) { - this.proto = requestProto; - viaProto = true; - } - - public RouterRemoveStoredTokenResponseProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void mergeLocalToProto() { - if (viaProto) { - maybeInitBuilder(); - } - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = RouterRemoveStoredTokenResponseProto.newBuilder(proto); - } - viaProto = false; - } - - private void mergeLocalToBuilder() { - if (this.routerStoreToken != null && - !((RouterStoreTokenPBImpl) this.routerStoreToken).getProto().equals(builder.getRouterStoreToken())) { - builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); - } - } - - @Override - public RouterStoreToken getRouterStoreToken() { - RouterRemoveStoredTokenResponseProtoOrBuilder p = viaProto ? proto : builder; - if (this.routerStoreToken != null) { - return this.routerStoreToken; - } - if (!p.hasRouterStoreToken()) { - return null; - } - this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken()); - return this.routerStoreToken; - } - - @Override - public void setRouterStoreToken(RouterStoreToken storeToken) { - maybeInitBuilder(); - if (storeToken == null) { - builder.clearRouterStoreToken(); - } - this.routerStoreToken = storeToken; - } - - private RouterStoreTokenProto convertToProtoFormat(RouterStoreToken storeToken) { - return ((RouterStoreTokenPBImpl) storeToken).getProto(); - } - - private RouterStoreToken convertFromProtoFormat(RouterStoreTokenProto storeTokenProto) { - return new RouterStoreTokenPBImpl(storeTokenProto); - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) { - return false; - } - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return TextFormat.shortDebugString(getProto()); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenRequestPBImpl.java deleted file mode 100644 index c35856dbb4991..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenRequestPBImpl.java +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; - -import org.apache.hadoop.thirdparty.protobuf.TextFormat; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterUpdateStoredTokenRequestProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterUpdateStoredTokenRequestProtoOrBuilder; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; -import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenRequest; - -public class RouterUpdateStoredTokenRequestPBImpl extends RouterUpdateStoredTokenRequest { - - private RouterUpdateStoredTokenRequestProto proto = - RouterUpdateStoredTokenRequestProto.getDefaultInstance(); - private RouterUpdateStoredTokenRequestProto.Builder builder = null; - private boolean viaProto = false; - private RouterStoreToken routerStoreToken = null; - - public RouterUpdateStoredTokenRequestPBImpl() { - builder = RouterUpdateStoredTokenRequestProto.newBuilder(); - } - - public RouterUpdateStoredTokenRequestPBImpl(RouterUpdateStoredTokenRequestProto requestProto) { - this.proto = requestProto; - viaProto = true; - } - - public RouterUpdateStoredTokenRequestProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void mergeLocalToProto() { - if (viaProto) { - maybeInitBuilder(); - } - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = RouterUpdateStoredTokenRequestProto.newBuilder(proto); - } - viaProto = false; - } - - private void mergeLocalToBuilder() { - if (this.routerStoreToken != null) { - RouterStoreTokenPBImpl routerStoreTokenPBImpl = - (RouterStoreTokenPBImpl) this.routerStoreToken; - RouterStoreTokenProto storeTokenProto = routerStoreTokenPBImpl.getProto(); - if (!storeTokenProto.equals(builder.getRouterStoreToken())) { - builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); - } - } - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) { - return false; - } - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return TextFormat.shortDebugString(getProto()); - } - - @Override - public RouterStoreToken getRouterStoreToken() { - RouterUpdateStoredTokenRequestProtoOrBuilder p = viaProto ? proto : builder; - if (this.routerStoreToken != null) { - return this.routerStoreToken; - } - if (!p.hasRouterStoreToken()) { - return null; - } - this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken()); - return this.routerStoreToken; - } - - @Override - public void setRouterStoreToken(RouterStoreToken storeToken) { - maybeInitBuilder(); - if (storeToken == null) { - builder.clearRouterStoreToken(); - } - this.routerStoreToken = storeToken; - this.builder.setRouterStoreToken(convertToProtoFormat(storeToken)); - } - - private RouterStoreTokenProto convertToProtoFormat(RouterStoreToken storeToken) { - return ((RouterStoreTokenPBImpl) storeToken).getProto(); - } - - private RouterStoreToken convertFromProtoFormat(RouterStoreTokenProto storeTokenProto) { - return new RouterStoreTokenPBImpl(storeTokenProto); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenResponsePBImpl.java deleted file mode 100644 index f4884ffaea088..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/RouterUpdateStoredTokenResponsePBImpl.java +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; - -import org.apache.hadoop.thirdparty.protobuf.TextFormat; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterUpdateStoredTokenResponseProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterUpdateStoredTokenResponseProtoOrBuilder; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; -import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenRequest; - -public class RouterUpdateStoredTokenResponsePBImpl extends RouterUpdateStoredTokenRequest { - - private RouterUpdateStoredTokenResponseProto proto = RouterUpdateStoredTokenResponseProto.getDefaultInstance(); - private RouterUpdateStoredTokenResponseProto.Builder builder = null; - private boolean viaProto = false; - private RouterStoreToken routerStoreToken = null; - - public RouterUpdateStoredTokenResponsePBImpl() { - builder = RouterUpdateStoredTokenResponseProto.newBuilder(); - } - - public RouterUpdateStoredTokenResponsePBImpl(RouterUpdateStoredTokenResponseProto requestProto) { - this.proto = requestProto; - viaProto = true; - } - - public RouterUpdateStoredTokenResponseProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void mergeLocalToProto() { - if (viaProto) { - maybeInitBuilder(); - } - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = RouterUpdateStoredTokenResponseProto.newBuilder(proto); - } - viaProto = false; - } - - private void mergeLocalToBuilder() { - if (this.routerStoreToken != null && - !((RouterStoreTokenPBImpl) this.routerStoreToken).getProto().equals(builder.getRouterStoreToken())) { - builder.setRouterStoreToken(convertToProtoFormat(this.routerStoreToken)); - } - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) { - return false; - } - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - @Override - public String toString() { - return TextFormat.shortDebugString(getProto()); - } - - @Override - public RouterStoreToken getRouterStoreToken() { - RouterUpdateStoredTokenResponseProtoOrBuilder p = viaProto ? proto : builder; - if (this.routerStoreToken != null) { - return this.routerStoreToken; - } - if (!p.hasRouterStoreToken()) { - return null; - } - this.routerStoreToken = convertFromProtoFormat(p.getRouterStoreToken()); - return this.routerStoreToken; - } - - @Override - public void setRouterStoreToken(RouterStoreToken storeToken) { - maybeInitBuilder(); - if (storeToken == null) { - builder.clearRouterStoreToken(); - } - this.routerStoreToken = storeToken; - this.builder.setRouterStoreToken(convertToProtoFormat(storeToken)); - } - - private RouterStoreTokenProto convertToProtoFormat(RouterStoreToken storeToken) { - return ((RouterStoreTokenPBImpl) storeToken).getProto(); - } - - private RouterStoreToken convertFromProtoFormat(RouterStoreTokenProto storeTokenProto) { - return new RouterStoreTokenPBImpl(storeTokenProto); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyResponsePBImpl.java deleted file mode 100644 index 61d7af86f041f..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/impl/pb/StoreNewMasterKeyResponsePBImpl.java +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.yarn.server.federation.store.records.impl.pb; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyResponseProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyResponseProtoOrBuilder; -import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; -import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyResponse; - -@Private -@Unstable -public class StoreNewMasterKeyResponsePBImpl extends StoreNewMasterKeyResponse { - - private StoreNewMasterKeyResponseProto proto = - StoreNewMasterKeyResponseProto.getDefaultInstance(); - - private StoreNewMasterKeyResponseProto.Builder builder = null; - - private boolean viaProto = false; - - private RouterMasterKey routerMasterKey = null; - - public StoreNewMasterKeyResponsePBImpl() { - builder = StoreNewMasterKeyResponseProto.newBuilder(); - } - - public StoreNewMasterKeyResponsePBImpl(StoreNewMasterKeyResponseProto responseProto) { - this.proto = responseProto; - viaProto = true; - } - - public StoreNewMasterKeyResponseProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - private void mergeLocalToProto() { - if (viaProto) { - maybeInitBuilder(); - } - mergeLocalToBuilder(); - proto = builder.build(); - viaProto = true; - } - - private void maybeInitBuilder() { - if (viaProto || builder == null) { - builder = StoreNewMasterKeyResponseProto.newBuilder(proto); - } - viaProto = false; - } - - private void mergeLocalToBuilder() { - if (this.routerMasterKey != null && !((RouterMasterKeyPBImpl) this.routerMasterKey). - getProto().equals(builder.getRouterMasterKey())) { - builder.setRouterMasterKey(convertToProtoFormat(this.routerMasterKey)); - } - } - - @Override - public int hashCode() { - return getProto().hashCode(); - } - - @Override - public boolean equals(Object other) { - if (other == null) { - return false; - } - if (other.getClass().isAssignableFrom(this.getClass())) { - return this.getProto().equals(this.getClass().cast(other).getProto()); - } - return false; - } - - private RouterMasterKeyProto convertToProtoFormat( - RouterMasterKey masterKey) { - return ((RouterMasterKeyPBImpl) masterKey).getProto(); - } - - @Override - public RouterMasterKey getRouterMasterKey() { - StoreNewMasterKeyResponseProtoOrBuilder p = viaProto ? proto : builder; - if (this.routerMasterKey != null) { - return this.routerMasterKey; - } - if (!p.hasRouterMasterKey()) { - return null; - } - this.routerMasterKey = convertFromProtoFormat(p.getRouterMasterKey()); - return this.routerMasterKey; - } - - @Override - public void setRouterMasterKey(RouterMasterKey masterKey) { - maybeInitBuilder(); - if (masterKey == null) { - builder.clearRouterMasterKey(); - return; - } - this.routerMasterKey = masterKey; - } - - private RouterMasterKey convertFromProtoFormat(RouterMasterKeyProto masterKey) { - return new RouterMasterKeyPBImpl(masterKey); - } -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto index d8c9f60ca170f..ff2b97091bf43 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_federation_protos.proto @@ -222,19 +222,11 @@ message RouterMasterKeyProto { optional uint64 expiry_date = 3; } -message StoreNewMasterKeyRequestProto { +message RouterMasterKeyRequestProto { optional RouterMasterKeyProto router_master_key = 1; } -message StoreNewMasterKeyResponseProto { - optional RouterMasterKeyProto router_master_key = 1; -} - -message RemoveStoredMasterKeyRequestProto { - optional RouterMasterKeyProto router_master_key = 1; -} - -message RemoveStoredMasterKeyResponseProto { +message RouterMasterKeyResponseProto { optional RouterMasterKeyProto router_master_key = 1; } @@ -243,30 +235,6 @@ message RouterStoreTokenProto { optional int64 renew_date = 2; } -message RouterStoreNewTokenRequestProto { - optional RouterStoreTokenProto router_store_token = 1; -} - -message RouterStoreNewTokenResponseProto { - optional RouterStoreTokenProto router_store_token = 1; -} - -message RouterUpdateStoredTokenRequestProto { - optional RouterStoreTokenProto router_store_token = 1; -} - -message RouterUpdateStoredTokenResponseProto { - optional RouterStoreTokenProto router_store_token = 1; -} - -message RouterRemoveStoredTokenRequestProto { - optional RouterStoreTokenProto router_store_token = 1; -} - -message RouterRemoveStoredTokenResponseProto { - optional RouterStoreTokenProto router_store_token = 1; -} - message RouterRMTokenRequestProto { optional RouterStoreTokenProto router_store_token = 1; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java index f441f0a44c84d..f87d370224309 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/records/TestFederationProtocolRecords.java @@ -48,17 +48,12 @@ import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterRegisterResponseProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterRequestProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.UpdateApplicationHomeSubClusterResponseProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyRequestProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.StoreNewMasterKeyResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreTokenProto; import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RemoveStoredMasterKeyRequestProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RemoveStoredMasterKeyResponseProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreNewTokenRequestProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterStoreNewTokenResponseProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterUpdateStoredTokenRequestProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterUpdateStoredTokenResponseProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRemoveStoredTokenRequestProto; -import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRemoveStoredTokenResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterRMTokenResponseProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyRequestProto; +import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.RouterMasterKeyResponseProto; import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.AddApplicationHomeSubClusterRequestPBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.AddApplicationHomeSubClusterResponsePBImpl; @@ -89,17 +84,11 @@ import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.UpdateApplicationHomeSubClusterRequestPBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.UpdateApplicationHomeSubClusterResponsePBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterMasterKeyPBImpl; -import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.StoreNewMasterKeyRequestPBImpl; -import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.StoreNewMasterKeyResponsePBImpl; -import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RemoveStoredMasterKeyRequestPBImpl; -import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RemoveStoredMasterKeyResponsePBImpl; import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterStoreTokenPBImpl; -import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterStoreNewTokenRequestPBImpl; -import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterStoreNewTokenResponsePBImpl; -import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterUpdateStoredTokenRequestPBImpl; -import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterUpdateStoredTokenResponsePBImpl; -import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterRemoveStoredTokenRequestPBImpl; -import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterRemoveStoredTokenResponsePBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterRMTokenRequestPBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterRMTokenResponsePBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterMasterKeyRequestPBImpl; +import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.RouterMasterKeyResponsePBImpl; import org.apache.hadoop.yarn.server.records.Version; import org.junit.BeforeClass; import org.junit.Test; @@ -296,69 +285,28 @@ public void testRouterMasterKey() throws Exception { validatePBImplRecord(RouterMasterKeyPBImpl.class, RouterMasterKeyProto.class); } - @Test - public void testStoreNewMasterKeyRequest() throws Exception { - validatePBImplRecord(StoreNewMasterKeyRequestPBImpl.class, - StoreNewMasterKeyRequestProto.class); - } - - @Test - public void testStoreNewMasterKeyResponse() throws Exception { - validatePBImplRecord(StoreNewMasterKeyResponsePBImpl.class, - StoreNewMasterKeyResponseProto.class); - } - - @Test - public void testRemoveStoredMasterKeyRequest() throws Exception { - validatePBImplRecord(RemoveStoredMasterKeyRequestPBImpl.class, - RemoveStoredMasterKeyRequestProto.class); - } - - @Test - public void testRemoveStoredMasterKeyResponse() throws Exception { - validatePBImplRecord(RemoveStoredMasterKeyResponsePBImpl.class, - RemoveStoredMasterKeyResponseProto.class); - } - @Test public void testRouterStoreToken() throws Exception { - validatePBImplRecord(RouterStoreTokenPBImpl.class, - YarnServerFederationProtos.RouterStoreTokenProto.class); - } - - @Test - public void testRouterStoreNewTokenRequest() throws Exception { - validatePBImplRecord(RouterStoreNewTokenRequestPBImpl.class, - RouterStoreNewTokenRequestProto.class); - } - - @Test - public void testRouterStoreNewTokenResponse() throws Exception { - validatePBImplRecord(RouterStoreNewTokenResponsePBImpl.class, - RouterStoreNewTokenResponseProto.class); + validatePBImplRecord(RouterStoreTokenPBImpl.class, RouterStoreTokenProto.class); } @Test - public void testRouterUpdateStoredTokenRequest() throws Exception { - validatePBImplRecord(RouterUpdateStoredTokenRequestPBImpl.class, - RouterUpdateStoredTokenRequestProto.class); + public void testRouterRMTokenRequest() throws Exception { + validatePBImplRecord(RouterRMTokenRequestPBImpl.class, RouterRMTokenRequestProto.class); } @Test - public void testRouterUpdateStoredTokenResponse() throws Exception { - validatePBImplRecord(RouterUpdateStoredTokenResponsePBImpl.class, - RouterUpdateStoredTokenResponseProto.class); + public void testRouterRMTokenResponse() throws Exception { + validatePBImplRecord(RouterRMTokenResponsePBImpl.class, RouterRMTokenResponseProto.class); } @Test - public void testRouterRemoveStoredTokenRequest() throws Exception { - validatePBImplRecord(RouterRemoveStoredTokenRequestPBImpl.class, - RouterRemoveStoredTokenRequestProto.class); + public void testRouterMasterKeyRequest() throws Exception { + validatePBImplRecord(RouterMasterKeyRequestPBImpl.class, RouterMasterKeyRequestProto.class); } @Test - public void testRouterRemoveStoredTokenResponse() throws Exception { - validatePBImplRecord(RouterRemoveStoredTokenResponsePBImpl.class, - RouterRemoveStoredTokenResponseProto.class); + public void testRouterMasterKeyResponse() throws Exception { + validatePBImplRecord(RouterMasterKeyResponsePBImpl.class, RouterMasterKeyResponseProto.class); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java index b84ec7a3ea6ac..b7c8288055abe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java @@ -66,16 +66,10 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RemoveStoredMasterKeyRequest; -import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyResponse; -import org.apache.hadoop.yarn.server.federation.store.records.StoreNewMasterKeyRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreNewTokenResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterUpdateStoredTokenResponse; -import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenRequest; -import org.apache.hadoop.yarn.server.federation.store.records.RouterRemoveStoredTokenResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.records.Version; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; @@ -337,32 +331,29 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( } @Override - public StoreNewMasterKeyResponse storeNewMasterKey( - StoreNewMasterKeyRequest request) throws Exception { + public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) + throws Exception { return stateStoreClient.storeNewMasterKey(request); } @Override - public RemoveStoredMasterKeyResponse removeStoredMasterKey( - RemoveStoredMasterKeyRequest request) throws Exception { + public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) + throws Exception { return stateStoreClient.removeStoredMasterKey(request); } @Override - public RouterStoreNewTokenResponse storeNewToken( - RouterStoreNewTokenRequest request) throws Exception { + public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws Exception { return stateStoreClient.storeNewToken(request); } @Override - public RouterUpdateStoredTokenResponse updateStoredToken( - RouterUpdateStoredTokenRequest request) throws Exception { + public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) throws Exception { return stateStoreClient.updateStoredToken(request); } @Override - public RouterRemoveStoredTokenResponse removeStoredToken( - RouterRemoveStoredTokenRequest request) throws Exception { + public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws Exception { return stateStoreClient.removeStoredToken(request); } } From bcb5783d2c68dafc79ef3305aeb82c857e50b42c Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 16 Aug 2022 09:43:22 +0800 Subject: [PATCH 09/12] YARN-9708. Yarn Router Support DelegationToken. --- .../records/RemoveStoredMasterKeyRequest.java | 42 ----------------- .../RemoveStoredMasterKeyResponse.java | 44 ------------------ .../RouterRemoveStoredTokenRequest.java | 45 ------------------- .../RouterRemoveStoredTokenResponse.java | 44 ------------------ .../records/RouterStoreNewTokenRequest.java | 44 ------------------ .../records/RouterStoreNewTokenResponse.java | 43 ------------------ .../RouterUpdateStoredTokenRequest.java | 45 ------------------- .../RouterUpdateStoredTokenResponse.java | 44 ------------------ .../records/StoreNewMasterKeyRequest.java | 43 ------------------ .../records/StoreNewMasterKeyResponse.java | 45 ------------------- 10 files changed, 439 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyRequest.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyResponse.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenRequest.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenResponse.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenRequest.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenResponse.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenRequest.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenResponse.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyRequest.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyResponse.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyRequest.java deleted file mode 100644 index c44945e467dca..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyRequest.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.yarn.server.federation.store.records; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.util.Records; - -@InterfaceAudience.Private -@InterfaceStability.Unstable -public abstract class RemoveStoredMasterKeyRequest { - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public static RemoveStoredMasterKeyRequest newInstance(RouterMasterKey routerMasterKey) { - RemoveStoredMasterKeyRequest request = Records.newRecord(RemoveStoredMasterKeyRequest.class); - request.setRouterMasterKey(routerMasterKey); - return request; - } - - @InterfaceAudience.Public - @InterfaceStability.Unstable - public abstract RouterMasterKey getRouterMasterKey(); - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public abstract void setRouterMasterKey(RouterMasterKey routerMasterKey); -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyResponse.java deleted file mode 100644 index 6ff44d3948b1f..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RemoveStoredMasterKeyResponse.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.federation.store.records; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.util.Records; - -@InterfaceAudience.Private -@InterfaceStability.Unstable -public abstract class RemoveStoredMasterKeyResponse { - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public static RemoveStoredMasterKeyResponse newInstance(RouterMasterKey masterKey) { - RemoveStoredMasterKeyResponse response = Records.newRecord(RemoveStoredMasterKeyResponse.class); - response.setRouterMasterKey(masterKey); - return response; - } - - @InterfaceAudience.Public - @InterfaceStability.Unstable - public abstract RouterMasterKey getRouterMasterKey(); - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public abstract void setRouterMasterKey(RouterMasterKey routerMasterKey); -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenRequest.java deleted file mode 100644 index d616ec0fe6699..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenRequest.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.federation.store.records; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.util.Records; - -@InterfaceAudience.Private -@InterfaceStability.Unstable -public abstract class RouterRemoveStoredTokenRequest { - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public static RouterRemoveStoredTokenRequest newInstance(RouterStoreToken routerStoreToken) { - RouterRemoveStoredTokenRequest request = - Records.newRecord(RouterRemoveStoredTokenRequest.class); - request.setRouterStoreToken(routerStoreToken); - return request; - } - - @InterfaceAudience.Public - @InterfaceStability.Unstable - public abstract RouterStoreToken getRouterStoreToken(); - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenResponse.java deleted file mode 100644 index 1a3a8092d61b7..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterRemoveStoredTokenResponse.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.federation.store.records; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.util.Records; - -@InterfaceAudience.Private -@InterfaceStability.Unstable -public abstract class RouterRemoveStoredTokenResponse { - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public static RouterRemoveStoredTokenResponse newInstance(RouterStoreToken routerStoreToken) { - RouterRemoveStoredTokenResponse request = - Records.newRecord(RouterRemoveStoredTokenResponse.class); - request.setRouterStoreToken(routerStoreToken); - return request; - } - - @InterfaceAudience.Public - @InterfaceStability.Unstable - public abstract RouterStoreToken getRouterStoreToken(); - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenRequest.java deleted file mode 100644 index b3a4608e9c393..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenRequest.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.federation.store.records; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.util.Records; - -@InterfaceAudience.Private -@InterfaceStability.Unstable -public abstract class RouterStoreNewTokenRequest { - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public static RouterStoreNewTokenRequest newInstance(RouterStoreToken routerStoreToken) { - RouterStoreNewTokenRequest request = Records.newRecord(RouterStoreNewTokenRequest.class); - request.setRouterStoreToken(routerStoreToken); - return request; - } - - @InterfaceAudience.Public - @InterfaceStability.Unstable - public abstract RouterStoreToken getRouterStoreToken(); - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenResponse.java deleted file mode 100644 index c119661f6876d..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreNewTokenResponse.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.federation.store.records; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.util.Records; - -@InterfaceAudience.Private -@InterfaceStability.Unstable -public abstract class RouterStoreNewTokenResponse { - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public static RouterStoreNewTokenResponse newInstance(RouterStoreToken routerStoreToken) { - RouterStoreNewTokenResponse request = Records.newRecord(RouterStoreNewTokenResponse.class); - request.setRouterStoreToken(routerStoreToken); - return request; - } - - @InterfaceAudience.Public - @InterfaceStability.Unstable - public abstract RouterStoreToken getRouterStoreToken(); - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenRequest.java deleted file mode 100644 index af2eb7adc6563..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenRequest.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.federation.store.records; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.util.Records; - -@InterfaceAudience.Private -@InterfaceStability.Unstable -public abstract class RouterUpdateStoredTokenRequest { - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public static RouterUpdateStoredTokenRequest newInstance(RouterStoreToken routerStoreToken) { - RouterUpdateStoredTokenRequest request = - Records.newRecord(RouterUpdateStoredTokenRequest.class); - request.setRouterStoreToken(routerStoreToken); - return request; - } - - @InterfaceAudience.Public - @InterfaceStability.Unstable - public abstract RouterStoreToken getRouterStoreToken(); - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenResponse.java deleted file mode 100644 index 600cc9fcb3ea6..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterUpdateStoredTokenResponse.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.federation.store.records; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.util.Records; - -@InterfaceAudience.Private -@InterfaceStability.Unstable -public abstract class RouterUpdateStoredTokenResponse { - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public static RouterUpdateStoredTokenResponse newInstance(RouterStoreToken routerStoreToken) { - RouterUpdateStoredTokenResponse request = - Records.newRecord(RouterUpdateStoredTokenResponse.class); - request.setRouterStoreToken(routerStoreToken); - return request; - } - - @InterfaceAudience.Public - @InterfaceStability.Unstable - public abstract RouterStoreToken getRouterStoreToken(); - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public abstract void setRouterStoreToken(RouterStoreToken routerStoreToken); -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyRequest.java deleted file mode 100644 index d3dd7ba3f3d52..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyRequest.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.yarn.server.federation.store.records; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.util.Records; - -@InterfaceAudience.Private -@InterfaceStability.Unstable -public abstract class StoreNewMasterKeyRequest { - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public static StoreNewMasterKeyRequest newInstance(RouterMasterKey routerMasterKey) { - StoreNewMasterKeyRequest request = Records.newRecord(StoreNewMasterKeyRequest.class); - request.setRouterMasterKey(routerMasterKey); - return request; - } - - @InterfaceAudience.Public - @InterfaceStability.Unstable - public abstract RouterMasterKey getRouterMasterKey(); - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public abstract void setRouterMasterKey(RouterMasterKey routerMasterKey); - -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyResponse.java deleted file mode 100644 index 3c491bfc7342b..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/StoreNewMasterKeyResponse.java +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.federation.store.records; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.yarn.util.Records; - -@InterfaceAudience.Private -@InterfaceStability.Unstable -public abstract class StoreNewMasterKeyResponse { - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public static StoreNewMasterKeyResponse newInstance(RouterMasterKey masterKey) { - StoreNewMasterKeyResponse response = Records.newRecord(StoreNewMasterKeyResponse.class); - response.setRouterMasterKey(masterKey); - return response; - } - - @InterfaceAudience.Public - @InterfaceStability.Unstable - public abstract RouterMasterKey getRouterMasterKey(); - - @InterfaceAudience.Private - @InterfaceStability.Unstable - public abstract void setRouterMasterKey(RouterMasterKey routerMasterKey); - -} From 2317e0d9643ee498d0cb679d075c07629d474d8d Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 16 Aug 2022 09:44:30 +0800 Subject: [PATCH 10/12] YARN-9708. Yarn Router Support DelegationToken. --- .../GetRouterDelegationTokenRequest.java | 50 ----------------- .../GetRouterDelegationTokenResponse.java | 54 ------------------- 2 files changed, 104 deletions(-) delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenRequest.java delete mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenResponse.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenRequest.java deleted file mode 100644 index faba59c322442..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenRequest.java +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.federation.store.records; - -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.yarn.util.Records; - -/** - * The request issued by the client to get a delegation token from - * the {@code ResourceManager}. - * for more information. - */ -@Public -@Stable -public abstract class GetRouterDelegationTokenRequest { - - @Public - @Stable - public static GetRouterDelegationTokenRequest newInstance(String renewer) { - GetRouterDelegationTokenRequest request = - Records.newRecord(GetRouterDelegationTokenRequest.class); - request.setRenewer(renewer); - return request; - } - - @Public - @Stable - public abstract String getRenewer(); - - @Public - @Stable - public abstract void setRenewer(String renewer); -} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenResponse.java deleted file mode 100644 index 71a0e3225cbc9..0000000000000 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/GetRouterDelegationTokenResponse.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.yarn.server.federation.store.records; - -import org.apache.hadoop.classification.InterfaceAudience.Private; -import org.apache.hadoop.classification.InterfaceAudience.Public; -import org.apache.hadoop.classification.InterfaceStability.Stable; -import org.apache.hadoop.classification.InterfaceStability.Unstable; -import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier; -import org.apache.hadoop.yarn.api.records.Token; -import org.apache.hadoop.yarn.util.Records; - -@Public -@Stable -public abstract class GetRouterDelegationTokenResponse { - - @Private - @Unstable - public static GetRouterDelegationTokenResponse newInstance(Token rmDTToken) { - GetRouterDelegationTokenResponse response = - Records.newRecord(GetRouterDelegationTokenResponse.class); - response.setRMDelegationToken(rmDTToken); - return response; - } - - /** - * The Delegation tokens have a identifier which maps to - * {@link AbstractDelegationTokenIdentifier}. - * @return the delegation tokens - */ - @Public - @Stable - public abstract Token getRMDelegationToken(); - - @Private - @Unstable - public abstract void setRMDelegationToken(Token rmDTToken); -} From d3948d055e138842ae9dbbe111a8c439ab335ce5 Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 16 Aug 2022 10:16:28 +0800 Subject: [PATCH 11/12] YARN-9708. Yarn Router Support DelegationToken. --- .../security/client/impl/pb/package-info.java | 3 +- .../FederationDelegationTokenStateStore.java | 26 ++++++----- .../impl/MemoryFederationStateStore.java | 17 +++++--- .../store/impl/SQLFederationStateStore.java | 14 +++--- .../impl/ZookeeperFederationStateStore.java | 13 +++--- .../utils/FederationStateStoreFacade.java | 11 +++-- .../utils/TestFederationStateStoreFacade.java | 43 +++++++++++++++---- .../server/router/security/package-info.java | 3 +- 8 files changed, 86 insertions(+), 44 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java index a481ba4197a6e..5751632e316ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/impl/pb/package-info.java @@ -17,5 +17,4 @@ */ @InterfaceAudience.Public package org.apache.hadoop.yarn.security.client.impl.pb; -import org.apache.hadoop.classification.InterfaceAudience; - +import org.apache.hadoop.classification.InterfaceAudience; \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java index 17ee028431070..9ab2746222616 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java @@ -18,11 +18,14 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; +import java.io.IOException; + /** * FederationDelegationTokenStateStore maintains the state of all * DelegationToken that have been submitted to the federated cluster. @@ -60,45 +63,48 @@ public interface FederationDelegationTokenStateStore { * * @param request DelegationKey. * @return StoreNewMasterKeyResponse. - * @throws Exception exception occurred. + * @throws YarnException exception occurred. */ - RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) throws Exception; + RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) + throws YarnException, IOException; /** * The Router Supports Remove the master key. * * @param request DelegationKey. * @return RemoveStoredMasterKeyResponse. - * @throws Exception exception occurred. + * @throws YarnException exception occurred. */ - RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) throws Exception; + RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) + throws YarnException, IOException; /** * The Router Supports Store new Token. * * @param request DelegationKey. * @return RouterRMTokenResponse. - * @throws Exception exception occurred. + * @throws YarnException exception occurred. */ - RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws Exception; + RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) + throws YarnException, IOException; /** * The Router Supports Update Token. * * @param request DelegationKey. * @return RouterRMTokenResponse. - * @throws Exception exception occurred. + * @throws YarnException exception occurred. */ RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) - throws Exception; + throws YarnException, IOException; /** * The Router Supports Remove Token. * * @param request DelegationKey. * @return RouterRMTokenResponse. - * @throws Exception exception occurred. + * @throws YarnException exception occurred. */ RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) - throws Exception; + throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java index d26e3a654287d..9cfedc2308708 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java @@ -28,6 +28,7 @@ import java.util.TimeZone; import java.util.concurrent.ConcurrentHashMap; +import org.apache.hadoop.classification.VisibleForTesting; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; @@ -382,7 +383,7 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( @Override public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) - throws Exception { + throws YarnException, IOException { // Restore the DelegationKey from the request RouterMasterKey masterKey = request.getRouterMasterKey(); @@ -408,7 +409,7 @@ public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) @Override public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) - throws Exception { + throws YarnException, IOException { // Restore the DelegationKey from the request RouterMasterKey masterKey = request.getRouterMasterKey(); @@ -426,7 +427,7 @@ public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest requ } @Override - public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws Exception { + public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws YarnException, IOException { RouterStoreToken storeToken = request.getRouterStoreToken(); RMDelegationTokenIdentifier tokenIdentifier = (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); @@ -436,7 +437,7 @@ public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws } @Override - public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) throws Exception { + public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) throws YarnException, IOException { RouterStoreToken storeToken = request.getRouterStoreToken(); RMDelegationTokenIdentifier tokenIdentifier = (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); @@ -446,7 +447,7 @@ public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) thr } @Override - public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws Exception { + public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws YarnException, IOException { RouterStoreToken storeToken = request.getRouterStoreToken(); RMDelegationTokenIdentifier tokenIdentifier = (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); @@ -456,7 +457,7 @@ public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) thr } private void storeOrUpdateRouterRMDT(RMDelegationTokenIdentifier rmDTIdentifier, - Long renewDate, boolean isUpdate) throws Exception { + Long renewDate, boolean isUpdate) throws IOException { Map rmDTState = routerRMSecretManagerState.getTokenState(); if (rmDTState.containsKey(rmDTIdentifier)) { LOG.info("Error storing info for RMDelegationToken: {}.", rmDTIdentifier); @@ -469,4 +470,8 @@ private void storeOrUpdateRouterRMDT(RMDelegationTokenIdentifier rmDTIdentifier, LOG.info("Store RM-RMDT with sequence number {}.", rmDTIdentifier.getSequenceNumber()); } + @VisibleForTesting + public RouterRMDTSecretManagerState getRouterRMSecretManagerState() { + return routerRMSecretManagerState; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java index 6725ab74076da..7ff38d3d238b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.federation.store.impl; +import java.io.IOException; import java.nio.ByteBuffer; import java.sql.CallableStatement; import java.sql.Connection; @@ -1034,28 +1035,31 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( @Override public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) - throws Exception { + throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } @Override public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) - throws Exception { + throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } @Override - public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws Exception { + public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) + throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } @Override - public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) throws Exception { + public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) + throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } @Override - public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws Exception { + public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) + throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java index c8a3a99ea5c61..471e5260f79bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java @@ -669,28 +669,31 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( @Override public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) - throws Exception{ + throws YarnException, IOException{ throw new NotImplementedException("Code is not implemented"); } @Override public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) - throws Exception { + throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } @Override - public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws Exception { + public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) + throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } @Override - public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) throws Exception { + public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) + throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } @Override - public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws Exception { + public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) + throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index 844b63bb66297..6b33adac917ff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -449,7 +449,7 @@ public SubClusterId getReservationHomeSubCluster(ReservationId reservationId) * @param newKey DelegationKey * @throws Exception An error occurred */ - public void storeNewMasterKey(DelegationKey newKey) throws Exception { + public void storeNewMasterKey(DelegationKey newKey) throws YarnException, IOException { LOG.info("Storing master key with keyID {}.", newKey.getKeyId()); ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey()); RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(), @@ -464,7 +464,7 @@ public void storeNewMasterKey(DelegationKey newKey) throws Exception { * @param newKey DelegationKey * @throws Exception An error occurred */ - public void removeStoredMasterKey(DelegationKey newKey) throws Exception { + public void removeStoredMasterKey(DelegationKey newKey) throws YarnException, IOException { LOG.info("Removing master key with keyID {}.", newKey.getKeyId()); ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey()); RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(), @@ -481,7 +481,7 @@ public void removeStoredMasterKey(DelegationKey newKey) throws Exception { * @throws IOException IO exception occurred. */ public void storeNewToken(RMDelegationTokenIdentifier identifier, - long renewDate) throws Exception { + long renewDate) throws YarnException, IOException { LOG.info("storing RMDelegation token with sequence number: {}.", identifier.getSequenceNumber()); RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate); @@ -497,7 +497,7 @@ public void storeNewToken(RMDelegationTokenIdentifier identifier, * @throws IOException IO exception occurred. */ public void updateStoredToken(RMDelegationTokenIdentifier identifier, - long renewDate) throws Exception { + long renewDate) throws YarnException, IOException { LOG.info("updating RMDelegation token with sequence number: {}.", identifier.getSequenceNumber()); RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate); @@ -512,7 +512,7 @@ public void updateStoredToken(RMDelegationTokenIdentifier identifier, * @throws IOException IO exception occurred. */ public void removeStoredToken(RMDelegationTokenIdentifier identifier) - throws Exception{ + throws YarnException, IOException{ LOG.info("removing RMDelegation token with sequence number: {}", identifier.getSequenceNumber()); RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, 0L); @@ -730,7 +730,6 @@ protected interface Func { TResult invoke(T input) throws Exception; } - @VisibleForTesting public FederationStateStore getStateStore() { return stateStore; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java index 56fa0524a39dc..c1a5b2a4b5050 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java @@ -18,21 +18,18 @@ package org.apache.hadoop.yarn.server.federation.utils; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Map; +import java.util.*; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; -import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; -import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration; +import org.apache.hadoop.yarn.server.federation.store.records.*; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -206,4 +203,34 @@ public void testAddApplicationHomeSubCluster() throws YarnException { Assert.assertEquals(subClusterId1, result); } + @Test + public void testStoreNewMasterKey() throws YarnException, IOException { + // store delegation key; + DelegationKey key = new DelegationKey(1234, 4321, "keyBytes".getBytes()); + HashSet keySet = new HashSet(); + keySet.add(key); + facade.storeNewMasterKey(key); + + MemoryFederationStateStore federationStateStore = (MemoryFederationStateStore) facade.getStateStore(); + RouterRMDTSecretManagerState secretManagerState = federationStateStore.getRouterRMSecretManagerState(); + Assert.assertEquals(keySet, secretManagerState.getMasterKeyState()); + } + + @Test + public void testStoreNewToken() throws YarnException, IOException { + // store new rm-token + RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( + new Text("owner1"), new Text("renewer1"), new Text("realuser1")); + int sequenceNumber = 1111; + dtId1.setSequenceNumber(sequenceNumber); + Long renewDate1 = new Long(System.currentTimeMillis()); + facade.storeNewToken(dtId1, renewDate1); + + Map token1 = new HashMap(); + token1.put(dtId1, renewDate1); + + MemoryFederationStateStore federationStateStore = (MemoryFederationStateStore) facade.getStateStore(); + RouterRMDTSecretManagerState secretManagerState = federationStateStore.getRouterRMSecretManagerState(); + Assert.assertEquals(token1, secretManagerState.getTokenState()); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java index b29c029c36d03..b15c54d81e836 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/package-info.java @@ -16,5 +16,4 @@ * limitations under the License. */ -/** Router WebApp package. **/ -package org.apache.hadoop.yarn.server.router.security; +package org.apache.hadoop.yarn.server.router.security; \ No newline at end of file From a85d82e26858684522da22f62b3c977145a6c9ab Mon Sep 17 00:00:00 2001 From: slfan1989 Date: Tue, 16 Aug 2022 16:28:36 +0800 Subject: [PATCH 12/12] YARN-9708. Yarn Router Support DelegationToken. --- .../FederationDelegationTokenStateStore.java | 21 ++ .../impl/MemoryFederationStateStore.java | 39 ++++ .../store/impl/SQLFederationStateStore.java | 12 ++ .../impl/ZookeeperFederationStateStore.java | 12 ++ .../utils/FederationStateStoreFacade.java | 43 +++- .../utils/TestFederationStateStoreFacade.java | 91 ++++++++- .../FederationStateStoreService.java | 27 ++- .../clientrm/RouterClientRMService.java | 6 + .../RouterDelegationTokenSecretManager.java | 88 +++++++- .../secure/AbstractSecureRouterTest.java | 8 + ...estRouterDelegationTokenSecretManager.java | 192 ++++++++++++++++++ 11 files changed, 516 insertions(+), 23 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java index 9ab2746222616..45b70aabec4fd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/FederationDelegationTokenStateStore.java @@ -78,6 +78,17 @@ RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) throws YarnException, IOException; + /** + * The Router supports obtaining MasterKey based on KeyId. + * + * @param request + * @return + * @throws YarnException + * @throws IOException + */ + RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request) + throws YarnException, IOException; + /** * The Router Supports Store new Token. * @@ -107,4 +118,14 @@ RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) */ RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws YarnException, IOException; + + /** + * The Router Supports Remove Token. + * + * @param request DelegationKey. + * @return RouterRMTokenResponse. + * @throws YarnException exception occurred. + */ + RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request) + throws YarnException, IOException; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java index 9cfedc2308708..0e1363a66d714 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java @@ -426,6 +426,27 @@ public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest requ return RouterMasterKeyResponse.newInstance(masterKey); } + @Override + public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request) + throws YarnException, IOException { + // Restore the DelegationKey from the request + RouterMasterKey masterKey = request.getRouterMasterKey(); + ByteBuffer keyByteBuf = masterKey.getKeyBytes(); + byte[] keyBytes = new byte[keyByteBuf.remaining()]; + keyByteBuf.get(keyBytes); + DelegationKey delegationKey = + new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes); + + Set rmDTMasterKeyState = routerRMSecretManagerState.getMasterKeyState(); + if (!rmDTMasterKeyState.contains(delegationKey)) { + throw new IOException("GetMasterKey with keyID: " + masterKey.getKeyId() + + " does not exist."); + } + RouterMasterKey resultRouterMasterKey = RouterMasterKey.newInstance(delegationKey.getKeyId(), + ByteBuffer.wrap(delegationKey.getEncodedKey()), delegationKey.getExpiryDate()); + return RouterMasterKeyResponse.newInstance(resultRouterMasterKey); + } + @Override public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws YarnException, IOException { RouterStoreToken storeToken = request.getRouterStoreToken(); @@ -442,6 +463,8 @@ public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) thr RMDelegationTokenIdentifier tokenIdentifier = (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); Long renewDate = storeToken.getRenewDate(); + Map rmDTState = routerRMSecretManagerState.getTokenState(); + rmDTState.remove(tokenIdentifier); storeOrUpdateRouterRMDT(tokenIdentifier, renewDate, true); return RouterRMTokenResponse.newInstance(storeToken); } @@ -470,6 +493,22 @@ private void storeOrUpdateRouterRMDT(RMDelegationTokenIdentifier rmDTIdentifier, LOG.info("Store RM-RMDT with sequence number {}.", rmDTIdentifier.getSequenceNumber()); } + @Override + public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request) + throws YarnException, IOException { + RouterStoreToken storeToken = request.getRouterStoreToken(); + RMDelegationTokenIdentifier tokenIdentifier = + (RMDelegationTokenIdentifier) storeToken.getTokenIdentifier(); + Map rmDTState = routerRMSecretManagerState.getTokenState(); + if (!rmDTState.containsKey(tokenIdentifier)) { + LOG.info("RMDelegationToken: {} does not exist.", tokenIdentifier); + throw new IOException("RMDelegationToken: " + tokenIdentifier + " does not exist."); + } + RouterStoreToken resultToken = + RouterStoreToken.newInstance(tokenIdentifier, rmDTState.get(tokenIdentifier)); + return RouterRMTokenResponse.newInstance(resultToken); + } + @VisibleForTesting public RouterRMDTSecretManagerState getRouterRMSecretManagerState() { return routerRMSecretManagerState; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java index 7ff38d3d238b2..e0e7e610d643d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java @@ -1062,4 +1062,16 @@ public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } + + @Override + public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java index 471e5260f79bf..ea23251815c54 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java @@ -696,4 +696,16 @@ public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws YarnException, IOException { throw new NotImplementedException("Code is not implemented"); } + + @Override + public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java index 6b33adac917ff..f8d994baa1eb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationStateStoreFacade.java @@ -75,10 +75,12 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration; import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; import org.apache.hadoop.yarn.server.federation.store.records.RouterStoreToken; -import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyRequest; import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenRequest; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -449,13 +451,14 @@ public SubClusterId getReservationHomeSubCluster(ReservationId reservationId) * @param newKey DelegationKey * @throws Exception An error occurred */ - public void storeNewMasterKey(DelegationKey newKey) throws YarnException, IOException { + public RouterMasterKeyResponse storeNewMasterKey(DelegationKey newKey) + throws YarnException, IOException { LOG.info("Storing master key with keyID {}.", newKey.getKeyId()); ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey()); RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(), keyBytes, newKey.getExpiryDate()); RouterMasterKeyRequest keyRequest = RouterMasterKeyRequest.newInstance(masterKey); - stateStore.storeNewMasterKey(keyRequest); + return stateStore.storeNewMasterKey(keyRequest); } /** @@ -473,6 +476,22 @@ public void removeStoredMasterKey(DelegationKey newKey) throws YarnException, IO stateStore.removeStoredMasterKey(keyRequest); } + /** + * The Router supports obtaining MasterKey based on KeyId. + * + * @param newKey DelegationKey + * @throws Exception An error occurred + */ + public RouterMasterKeyResponse getMasterKeyByDelegationKey(DelegationKey newKey) + throws YarnException, IOException { + LOG.info("Storing master key with keyID {}.", newKey.getKeyId()); + ByteBuffer keyBytes = ByteBuffer.wrap(newKey.getEncodedKey()); + RouterMasterKey masterKey = RouterMasterKey.newInstance(newKey.getKeyId(), + keyBytes, newKey.getExpiryDate()); + RouterMasterKeyRequest keyRequest = RouterMasterKeyRequest.newInstance(masterKey); + return stateStore.getMasterKeyByDelegationKey(keyRequest); + } + /** * The Router Supports Store new Token. * @@ -513,13 +532,29 @@ public void updateStoredToken(RMDelegationTokenIdentifier identifier, */ public void removeStoredToken(RMDelegationTokenIdentifier identifier) throws YarnException, IOException{ - LOG.info("removing RMDelegation token with sequence number: {}", + LOG.info("removing RMDelegation token with sequence number: {}.", identifier.getSequenceNumber()); RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, 0L); RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken); stateStore.removeStoredToken(request); } + /** + * The Router Supports Remove Token. + * + * @param identifier Delegation Token + * @return RouterRMTokenResponse. + * @throws YarnException exception occurred. + */ + public RouterRMTokenResponse getTokenByRouterStoreToken(RMDelegationTokenIdentifier identifier) + throws YarnException, IOException { + LOG.info("get RouterStoreToken token with sequence number: {}.", + identifier.getSequenceNumber()); + RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, 0L); + RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken); + return stateStore.getTokenByRouterStoreToken(request); + } + /** * Helper method to create instances of Object using the class name defined in * the configuration object. The instances creates {@link RetryProxy} using diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java index c1a5b2a4b5050..056a287a229db 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacade.java @@ -18,18 +18,28 @@ package org.apache.hadoop.yarn.server.federation.utils; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.HashSet; +import java.util.HashMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.util.Time; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.federation.store.FederationStateStore; import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; -import org.apache.hadoop.yarn.server.federation.store.records.*; +import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo; +import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMDTSecretManagerState; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -216,21 +226,88 @@ public void testStoreNewMasterKey() throws YarnException, IOException { Assert.assertEquals(keySet, secretManagerState.getMasterKeyState()); } + @Test + public void testRemoveStoredMasterKey() throws YarnException, IOException { + // store delegation key; + DelegationKey key = new DelegationKey(4567, 7654, "keyBytes".getBytes()); + HashSet keySet = new HashSet(); + keySet.add(key); + facade.storeNewMasterKey(key); + + // check to delete delegationKey + facade.removeStoredMasterKey(key); + keySet.clear(); + + MemoryFederationStateStore federationStateStore = (MemoryFederationStateStore) facade.getStateStore(); + RouterRMDTSecretManagerState secretManagerState = federationStateStore.getRouterRMSecretManagerState(); + Assert.assertEquals(keySet, secretManagerState.getMasterKeyState()); + } + @Test public void testStoreNewToken() throws YarnException, IOException { // store new rm-token RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( new Text("owner1"), new Text("renewer1"), new Text("realuser1")); - int sequenceNumber = 1111; + int sequenceNumber = 1; dtId1.setSequenceNumber(sequenceNumber); - Long renewDate1 = new Long(System.currentTimeMillis()); + Long renewDate1 = Time.now(); facade.storeNewToken(dtId1, renewDate1); Map token1 = new HashMap(); token1.put(dtId1, renewDate1); - MemoryFederationStateStore federationStateStore = (MemoryFederationStateStore) facade.getStateStore(); - RouterRMDTSecretManagerState secretManagerState = federationStateStore.getRouterRMSecretManagerState(); - Assert.assertEquals(token1, secretManagerState.getTokenState()); + MemoryFederationStateStore stateStore = (MemoryFederationStateStore) facade.getStateStore(); + RouterRMDTSecretManagerState storeSecretManagerState = + stateStore.getRouterRMSecretManagerState(); + Assert.assertEquals(token1, storeSecretManagerState.getTokenState()); + } + + @Test + public void testUpdateNewToken() throws YarnException, IOException { + // store new rm-token + RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( + new Text("owner2"), new Text("renewer2"), new Text("realuser2")); + int sequenceNumber = 2; + dtId1.setSequenceNumber(sequenceNumber); + Long renewDate1 = Time.now(); + facade.storeNewToken(dtId1, renewDate1); + + Map token1 = new HashMap(); + token1.put(dtId1, renewDate1); + + renewDate1 = Time.now(); + facade.updateStoredToken(dtId1, renewDate1); + token1.put(dtId1, renewDate1); + + MemoryFederationStateStore stateStore = (MemoryFederationStateStore) facade.getStateStore(); + RouterRMDTSecretManagerState updateSecretManagerState = + stateStore.getRouterRMSecretManagerState(); + Assert.assertEquals(token1, updateSecretManagerState.getTokenState()); + Assert.assertEquals(sequenceNumber, updateSecretManagerState.getDTSequenceNumber()); + } + + @Test + public void testRemoveStoredToken() throws YarnException, IOException { + // store new rm-token + RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( + new Text("owner3"), new Text("renewer3"), new Text("realuser3")); + int sequenceNumber = 3; + dtId1.setSequenceNumber(sequenceNumber); + Long renewDate1 = Time.now(); + facade.storeNewToken(dtId1, renewDate1); + + Map token1 = new HashMap(); + token1.put(dtId1, renewDate1); + + // remove rm-token + facade.removeStoredToken(dtId1); + token1.clear(); + + MemoryFederationStateStore stateStore = (MemoryFederationStateStore) facade.getStateStore(); + RouterRMDTSecretManagerState deleteSecretManagerState = + stateStore.getRouterRMSecretManagerState(); + + Assert.assertEquals(token1, deleteSecretManagerState.getTokenState()); + Assert.assertEquals(sequenceNumber, deleteSecretManagerState.getDTSequenceNumber()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java index b7c8288055abe..f6ff41ab40dcd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/federation/FederationStateStoreService.java @@ -18,10 +18,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.federation; +import java.io.IOException; import java.net.InetSocketAddress; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.NotImplementedException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.retry.RetryPolicy; import org.apache.hadoop.net.NetUtils; @@ -332,28 +334,43 @@ public GetReservationsHomeSubClusterResponse getReservationsHomeSubCluster( @Override public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request) - throws Exception { + throws IOException, YarnException { return stateStoreClient.storeNewMasterKey(request); } @Override public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request) - throws Exception { + throws IOException, YarnException { return stateStoreClient.removeStoredMasterKey(request); } @Override - public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) throws Exception { + public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request) + throws IOException, YarnException { return stateStoreClient.storeNewToken(request); } @Override - public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) throws Exception { + public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request) + throws IOException, YarnException { return stateStoreClient.updateStoredToken(request); } @Override - public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) throws Exception { + public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request) + throws IOException, YarnException { return stateStoreClient.removeStoredToken(request); } + + @Override + public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } + + @Override + public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request) + throws YarnException, IOException { + throw new NotImplementedException("Code is not implemented"); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java index 1c8550ef26a08..e986e05a0bbd9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/RouterClientRMService.java @@ -169,6 +169,7 @@ protected void serviceStart() throws Exception { // Initialize RouterRMDelegationTokenSecretManager. routerDTSecretManager = createRouterRMDelegationTokenSecretManager(conf); + routerDTSecretManager.startThreads(); this.server = rpc.getServer(ApplicationClientProtocol.class, this, listenerEndpoint, serverConf, routerDTSecretManager, numWorkerThreads); @@ -593,4 +594,9 @@ protected RouterDelegationTokenSecretManager createRouterRMDelegationTokenSecret return new RouterDelegationTokenSecretManager(secretKeyInterval, tokenMaxLifetime, tokenRenewInterval, 3600000); } + + @VisibleForTesting + public RouterDelegationTokenSecretManager getRouterDTSecretManager() { + return routerDTSecretManager; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java index 3f934f6cd916b..72ba2f5130fb1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/security/RouterDelegationTokenSecretManager.java @@ -22,12 +22,22 @@ import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager; import org.apache.hadoop.security.token.delegation.DelegationKey; import org.apache.hadoop.util.ExitUtil; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKey; +import org.apache.hadoop.yarn.server.federation.store.records.RouterMasterKeyResponse; +import org.apache.hadoop.yarn.server.federation.store.records.RouterRMTokenResponse; import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; /** * A Router specific delegation token secret manager. @@ -59,6 +69,7 @@ public RouterDelegationTokenSecretManager(long delegationKeyUpdateInterval, long delegationTokenRemoverScanInterval) { super(delegationKeyUpdateInterval, delegationTokenMaxLifetime, delegationTokenRenewInterval, delegationTokenRemoverScanInterval); + this.federationFacade = FederationStateStoreFacade.getInstance(); } @Override @@ -76,7 +87,7 @@ private boolean shouldIgnoreException(Exception e) { * @param newKey DelegationKey */ @Override - protected void storeNewMasterKey(DelegationKey newKey) { + public void storeNewMasterKey(DelegationKey newKey) { try { federationFacade.storeNewMasterKey(newKey); } catch (Exception e) { @@ -93,7 +104,7 @@ protected void storeNewMasterKey(DelegationKey newKey) { * @param delegationKey DelegationKey */ @Override - protected void removeStoredMasterKey(DelegationKey delegationKey) { + public void removeStoredMasterKey(DelegationKey delegationKey) { try { federationFacade.removeStoredMasterKey(delegationKey); } catch (Exception e) { @@ -112,7 +123,7 @@ protected void removeStoredMasterKey(DelegationKey delegationKey) { * @throws IOException IO exception occurred. */ @Override - protected void storeNewToken(RMDelegationTokenIdentifier identifier, + public void storeNewToken(RMDelegationTokenIdentifier identifier, long renewDate) throws IOException { try { federationFacade.storeNewToken(identifier, renewDate); @@ -133,8 +144,7 @@ protected void storeNewToken(RMDelegationTokenIdentifier identifier, * @throws IOException IO exception occurred. */ @Override - protected void updateStoredToken(RMDelegationTokenIdentifier id, long renewDate) - throws IOException { + public void updateStoredToken(RMDelegationTokenIdentifier id, long renewDate) throws IOException { try { federationFacade.updateStoredToken(id, renewDate); } catch (Exception e) { @@ -153,8 +163,7 @@ protected void updateStoredToken(RMDelegationTokenIdentifier id, long renewDate) * @throws IOException IO exception occurred. */ @Override - protected void removeStoredToken(RMDelegationTokenIdentifier - identifier) throws IOException { + public void removeStoredToken(RMDelegationTokenIdentifier identifier) throws IOException { try { federationFacade.removeStoredToken(identifier); } catch (Exception e) { @@ -166,9 +175,74 @@ protected void removeStoredToken(RMDelegationTokenIdentifier } } + /** + * The Router supports obtaining MasterKey based on KeyId. + * + * @param newKey DelegationKey + * @throws Exception An error occurred + * @return DelegationKey + */ + public DelegationKey getMasterKeyByDelegationKey(DelegationKey newKey) + throws YarnException, IOException { + try { + RouterMasterKeyResponse response = federationFacade.getMasterKeyByDelegationKey(newKey); + RouterMasterKey masterKey = response.getRouterMasterKey(); + ByteBuffer keyByteBuf = masterKey.getKeyBytes(); + byte[] keyBytes = new byte[keyByteBuf.remaining()]; + keyByteBuf.get(keyBytes); + DelegationKey delegationKey = + new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes); + return delegationKey; + } catch (Exception ex) { + throw new YarnException(ex); + } + } + + /** + * The Router supports obtaining MasterKey based on KeyId. + * + * @param identifier RMDelegationTokenIdentifier + * @return RMDelegationTokenIdentifier. + * @throws YarnException exception occurred. + */ + public RMDelegationTokenIdentifier getTokenByRouterStoreToken( + RMDelegationTokenIdentifier identifier) throws YarnException, IOException { + try { + RouterRMTokenResponse response = federationFacade.getTokenByRouterStoreToken(identifier); + YARNDelegationTokenIdentifier responseIdentifier = + response.getRouterStoreToken().getTokenIdentifier(); + return (RMDelegationTokenIdentifier) responseIdentifier; + } catch (Exception ex) { + throw new YarnException(ex); + } + } + @InterfaceAudience.Private @VisibleForTesting public int getLatestDTSequenceNumber() { return delegationTokenSequenceNumber; } + + public void setFederationFacade(FederationStateStoreFacade federationFacade) { + this.federationFacade = federationFacade; + } + + @InterfaceAudience.Private + @VisibleForTesting + public synchronized Set getAllMasterKeys() { + HashSet keySet = new HashSet<>(); + keySet.addAll(allKeys.values()); + return keySet; + } + + @InterfaceAudience.Private + @VisibleForTesting + public synchronized Map getAllTokens() { + Map allTokens = new HashMap<>(); + for (Map.Entry entry : currentTokens.entrySet()) { + allTokens.put(entry.getKey(), entry.getValue().getRenewDate()); + } + return allTokens; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java index 61a5e8923b42a..090cd05ce5bb6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/AbstractSecureRouterTest.java @@ -23,7 +23,9 @@ import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore; import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId; +import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade; import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart; import org.apache.hadoop.yarn.server.router.Router; @@ -175,6 +177,9 @@ public static File createKeytab(String principal, String filename) throws Except */ public synchronized void startSecureRouter() { assertNull("Router is already running", router); + MemoryFederationStateStore stateStore = new MemoryFederationStateStore(); + stateStore.init(getConf()); + FederationStateStoreFacade.getInstance().reinitialize(stateStore, getConf()); UserGroupInformation.setConfiguration(conf); router = new Router(); router.init(conf); @@ -228,4 +233,7 @@ public static ConcurrentHashMap getMockRMs() { return mockRMs; } + public static Configuration getConf() { + return conf; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java new file mode 100644 index 0000000000000..e4f3fbf0eaa47 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/secure/TestRouterDelegationTokenSecretManager.java @@ -0,0 +1,192 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.router.secure; + +import org.apache.hadoop.io.Text; +import org.apache.hadoop.security.token.delegation.DelegationKey; +import org.apache.hadoop.test.LambdaTestUtils; +import org.apache.hadoop.util.Time; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; +import org.apache.hadoop.yarn.server.router.clientrm.RouterClientRMService; +import org.apache.hadoop.yarn.server.router.security.RouterDelegationTokenSecretManager; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; + +public class TestRouterDelegationTokenSecretManager extends AbstractSecureRouterTest { + + private static final Logger LOG = LoggerFactory.getLogger(TestRouterDelegationTokenSecretManager.class); + + @Test + public void testRouterStoreNewMasterKey() throws Exception { + LOG.info("Test RouterDelegationTokenSecretManager: StoreNewMasterKey."); + + // Start the Router in Secure Mode + startSecureRouter(); + + // Store NewMasterKey + RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); + RouterDelegationTokenSecretManager secretManager = routerClientRMService.getRouterDTSecretManager(); + DelegationKey storeKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); + secretManager.storeNewMasterKey(storeKey); + + // Get DelegationKey + DelegationKey paramKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); + DelegationKey responseKey = secretManager.getMasterKeyByDelegationKey(paramKey); + + Assert.assertNotNull(paramKey); + Assert.assertEquals(storeKey.getExpiryDate(), responseKey.getExpiryDate()); + Assert.assertEquals(storeKey.getKeyId(), responseKey.getKeyId()); + Assert.assertTrue(Arrays.equals(storeKey.getEncodedKey(), responseKey.getEncodedKey())); + Assert.assertEquals(storeKey, responseKey); + + stopSecureRouter(); + } + + @Test + public void testRouterRemoveStoredMasterKey() throws Exception { + LOG.info("Test RouterDelegationTokenSecretManager: RemoveStoredMasterKey."); + + // Start the Router in Secure Mode + startSecureRouter(); + + // Store NewMasterKey + RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); + RouterDelegationTokenSecretManager secretManager = routerClientRMService.getRouterDTSecretManager(); + DelegationKey storeKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); + secretManager.storeNewMasterKey(storeKey); + + // Remove DelegationKey + secretManager.removeStoredMasterKey(storeKey); + + // Get DelegationKey + DelegationKey paramKey = new DelegationKey(1234, 4321, "keyBytes".getBytes()); + LambdaTestUtils.intercept(YarnException.class, + "GetMasterKey with keyID: " + storeKey.getKeyId() + " does not exist.", + () -> secretManager.getMasterKeyByDelegationKey(paramKey)); + + stopSecureRouter(); + } + + @Test + public void testRouterStoreNewToken() throws Exception { + LOG.info("Test RouterDelegationTokenSecretManager: StoreNewToken."); + + // Start the Router in Secure Mode + startSecureRouter(); + + // Store new rm-token + RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); + RouterDelegationTokenSecretManager secretManager = routerClientRMService.getRouterDTSecretManager(); + RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( + new Text("owner1"), new Text("renewer1"), new Text("realuser1")); + int sequenceNumber = 1; + dtId1.setSequenceNumber(sequenceNumber); + Long renewDate1 = Time.now(); + secretManager.storeNewToken(dtId1, renewDate1); + + // query rm-token + RMDelegationTokenIdentifier dtId2 = new RMDelegationTokenIdentifier( + new Text("owner1"), new Text("renewer1"), new Text("realuser1")); + dtId2.setSequenceNumber(sequenceNumber); + RMDelegationTokenIdentifier dtId3 = secretManager.getTokenByRouterStoreToken(dtId2); + Assert.assertEquals(dtId1, dtId3); + + // query rm-token2 not exists + sequenceNumber++; + dtId2.setSequenceNumber(2); + LambdaTestUtils.intercept(YarnException.class, + "RMDelegationToken: " + dtId2 + " does not exist.", + () -> secretManager.getTokenByRouterStoreToken(dtId2)); + + stopSecureRouter(); + } + + @Test + public void testRouterUpdateNewToken() throws Exception { + LOG.info("Test RouterDelegationTokenSecretManager: UpdateNewToken."); + + // Start the Router in Secure Mode + startSecureRouter(); + + // Store new rm-token + RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); + RouterDelegationTokenSecretManager secretManager = routerClientRMService.getRouterDTSecretManager(); + RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( + new Text("owner1"), new Text("renewer1"), new Text("realuser1")); + int sequenceNumber = 1; + dtId1.setSequenceNumber(sequenceNumber); + Long renewDate1 = Time.now(); + secretManager.storeNewToken(dtId1, renewDate1); + + sequenceNumber++; + dtId1.setSequenceNumber(sequenceNumber); + secretManager.updateStoredToken(dtId1, renewDate1); + + // query rm-token + RMDelegationTokenIdentifier dtId2 = new RMDelegationTokenIdentifier( + new Text("owner1"), new Text("renewer1"), new Text("realuser1")); + dtId2.setSequenceNumber(sequenceNumber); + RMDelegationTokenIdentifier dtId3 = secretManager.getTokenByRouterStoreToken(dtId2); + Assert.assertNotNull(dtId3); + Assert.assertEquals(dtId1.getKind(), dtId3.getKind()); + Assert.assertEquals(dtId1.getOwner(), dtId3.getOwner()); + Assert.assertEquals(dtId1.getRealUser(), dtId3.getRealUser()); + Assert.assertEquals(dtId1.getRenewer(), dtId3.getRenewer()); + Assert.assertEquals(dtId1.getIssueDate(), dtId3.getIssueDate()); + Assert.assertEquals(dtId1.getMasterKeyId(), dtId3.getMasterKeyId()); + Assert.assertEquals(dtId1.getSequenceNumber(), dtId3.getSequenceNumber()); + Assert.assertEquals(sequenceNumber, dtId3.getSequenceNumber()); + Assert.assertEquals(dtId1, dtId3); + + stopSecureRouter(); + } + + @Test + public void testRouterRemoveToken() throws Exception { + LOG.info("Test RouterDelegationTokenSecretManager: RouterRemoveToken."); + + // Start the Router in Secure Mode + startSecureRouter(); + + // Store new rm-token + RouterClientRMService routerClientRMService = this.getRouter().getClientRMProxyService(); + RouterDelegationTokenSecretManager secretManager = routerClientRMService.getRouterDTSecretManager(); + RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier( + new Text("owner1"), new Text("renewer1"), new Text("realuser1")); + int sequenceNumber = 1; + dtId1.setSequenceNumber(sequenceNumber); + Long renewDate1 = Time.now(); + secretManager.storeNewToken(dtId1, renewDate1); + + // Remove rm-token + secretManager.removeStoredToken(dtId1); + + // query rm-token + LambdaTestUtils.intercept(YarnException.class, + "RMDelegationToken: " + dtId1 + " does not exist.", + () -> secretManager.getTokenByRouterStoreToken(dtId1)); + + stopSecureRouter(); + } +}