@@ -24,12 +24,16 @@ def __init__(self, object_name, config, git_staging_repo, s3, bucket, cvmfs_repo
2424 self .config = config
2525 self .git_repo = git_staging_repo
2626 self .metadata_file = object_name + config ['paths' ]['metadata_file_extension' ]
27+ self .metadata_sig_file = self .metadata_file + config ['signatures' ]['signature_file_extension' ]
2728 self .object = object_name
29+ self .object_sig = object_name + config ['signatures' ]['signature_file_extension' ]
2830 self .s3 = s3
2931 self .bucket = bucket
3032 self .cvmfs_repo = cvmfs_repo
3133 self .local_path = os .path .join (config ['paths' ]['download_dir' ], os .path .basename (object_name ))
34+ self .local_sig_path = self .local_path + config ['signatures' ]['signature_file_extension' ]
3235 self .local_metadata_path = self .local_path + config ['paths' ]['metadata_file_extension' ]
36+ self .local_metadata_sig_path = self .local_metadata_path + config ['signatures' ]['signature_file_extension' ]
3337 self .url = f'https://{ bucket } { object_name }
3438
3539 self .states = {
@@ -48,22 +52,42 @@ def download(self, force=False):
4852 """
4953 Download this tarball and its corresponding metadata file, if this hasn't been already done.
5054 """
51- if force or not os .path .exists (self .local_path ):
52- try :
53- self .s3 .download_file (self .bucket , self .object , self .local_path )
54- except :
55- logging .error (
56- f'Failed to download tarball { self .object } { self .bucket } { self .local_path }
57- )
58- self .local_path = None
59- if force or not os .path .exists (self .local_metadata_path ):
60- try :
61- self .s3 .download_file (self .bucket , self .metadata_file , self .local_metadata_path )
62- except :
63- logging .error (
64- f'Failed to download metadata file { self .metadata_file } { self .bucket } { self .local_metadata_path }
65- )
66- self .local_metadata_path = None
55+ files = [
56+ (self .object , self .local_path , self .object_sig , self .local_sig_path ),
57+ (self .metadata_file , self .local_metadata_path , self .metadata_sig_file , self .local_metadata_sig_path ),
58+ ]
59+ skip = False
60+ for (object , local_file , sig_object , local_sig_file ) in files :
61+ if force or not os .path .exists (local_file ):
62+ # First we try to download signature file, which may or may not be available
63+ # and may be optional or required.
64+ try :
65+ self .s3 .download_file (self .bucket , sig_object , local_sig_file )
66+ except :
67+ if self .config ['signatures' ].getboolean ('signatures_required' , True ):
68+ logging .error (
69+ f'Failed to download signature file { sig_object } { object } { self .bucket } { local_sig_file }
70+ )
71+ skip = True
72+ break
73+ else :
74+ logging .warning (
75+ f'Failed to download signature file { sig_object } { object } { self .bucket } { local_sig_file } +
76+ 'Ignoring this, because signatures are not required with the current configuration.'
77+ )
78+ # Now we download the file itself.
79+ try :
80+ self .s3 .download_file (self .bucket , object , local_file )
81+ except :
82+ logging .error (
83+ f'Failed to download { object } { self .bucket } { local_file }
84+ )
85+ skip = True
86+ break
87+ # If any required download failed, make sure to skip this tarball completely.
88+ if skip :
89+ self .local_path = None
90+ self .local_metadata_path = None
6791
6892 def find_state (self ):
6993 """Find the state of this tarball by searching through the state directories in the git repository."""
@@ -156,6 +180,49 @@ def run_handler(self):
156180 handler = self .states [self .state ]['handler' ]
157181 handler ()
158182
183+ def verify_signatures (self ):
184+ """Verify the signatures of the downloaded tarball and metadata file using the corresponding signature files."""
185+
186+ sig_missing_msg = 'Signature file %s is missing.'
187+ sig_missing = False
188+ for sig_file in [self .local_sig_path , self .local_metadata_sig_path ]:
189+ if not os .path .exists (sig_file ):
190+ logging .warning (sig_missing_msg % sig_file )
191+ sig_missing = True
192+
193+ if sig_missing :
194+ # If signature files are missing, we return a failure,
195+ # unless the configuration specifies that signatures are not required.
196+ if self .config ['signatures' ].getboolean ('signatures_required' , True ):
197+ return False
198+ else :
199+ return True
200+
201+ # If signatures are provided, we should always verify them, regardless of the signatures_required.
202+ # In order to do so, we need the verification script and an allowed signers file.
203+ verify_script = self .config ['signatures' ]['signature_verification_script' ]
204+ allowed_signers_file = self .config ['signatures' ]['allowed_signers_file' ]
205+ if not os .path .exists (verify_script ):
206+ logging .error (f'Unable to verify signatures, the specified signature verification script does not exist!' )
207+ return False
208+
209+ if not os .path .exists (allowed_signers_file ):
210+ logging .error (f'Unable to verify signatures, the specified allowed signers file does not exist!' )
211+ return False
212+
213+ for (file , sig_file ) in [(self .local_path , self .local_sig_path ), (self .local_metadata_path , self .local_metadata_sig_path )]:
214+ verify_cmd = subprocess .run (
215+ [verify_script , '--verify' , '--allowed-signers-file' , allowed_signers_file , '--file' , file , '--signature-file' , sig_file ],
216+ stdout = subprocess .PIPE ,
217+ stderr = subprocess .PIPE )
218+ if verify_cmd .returncode == 0 :
219+ logging .debug (f'Signature for { file } )
220+ else :
221+ logging .error (f'Failed to verify signature for { file } )
222+ return False
223+
224+ return True
225+
159226 def verify_checksum (self ):
160227 """Verify the checksum of the downloaded tarball with the one in its metadata file."""
161228 local_sha256 = sha256sum (self .local_path )
@@ -171,13 +238,26 @@ def ingest(self):
171238 #TODO: check if there is an open issue for this tarball, and if there is, skip it.
172239 logging .info (f'Tarball { self .object } )
173240 self .download ()
241+ logging .info ('Verifying its signature...' )
242+ if not self .verify_signatures ():
243+ issue_msg = f'Failed to verify signatures for `{ self .object }
244+ logging .error (issue_msg )
245+ if not self .issue_exists (issue_msg , state = 'open' ):
246+ self .git_repo .create_issue (title = issue_msg , body = issue_msg )
247+ return
248+ else :
249+ logging .debug (f'Signatures of { self .object } )
250+
174251 logging .info ('Verifying its checksum...' )
175252 if not self .verify_checksum ():
176- logging .error ('Checksum of downloaded tarball does not match the one in its metadata file!' )
177- # Open issue?
253+ issue_msg = f'Failed to verify checksum for `{ self .object }
254+ logging .error (issue_msg )
255+ if not self .issue_exists (issue_msg , state = 'open' ):
256+ self .git_repo .create_issue (title = issue_msg , body = issue_msg )
178257 return
179258 else :
180259 logging .debug (f'Checksum of { self .object } )
260+
181261 script = self .config ['paths' ]['ingestion_script' ]
182262 sudo = ['sudo' ] if self .config ['cvmfs' ].getboolean ('ingest_as_root' , True ) else []
183263 logging .info (f'Running the ingestion script for { self .object } )
@@ -222,6 +302,10 @@ def mark_new_tarball_as_staged(self):
222302 logging .warn ('Skipping this tarball...' )
223303 return
224304
305+ # Verify the signatures of the tarball and metadata file.
306+ if not self .verify_signatures ():
307+ logging .warn ('Signature verification of the tarball or its metadata failed, skipping this tarball...' )
308+
225309 contents = ''
226310 with open (self .local_metadata_path , 'r' ) as meta :
227311 contents = meta .read ()
0 commit comments