Issue 4340 - Drop dependency on external xar tool

xarfile.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import re
 import struct
 import time
 import zlib
 
 from Crypto.Hash import SHA
 from Crypto.PublicKey import RSA
 from Crypto.Signature import PKCS1_v1_5
 
 from buildtools.packager import getTemplate
 
+XAR_HEADER = struct.Struct('>IHHQQI')
 XAR_HEADER_MAGIC = 0x78617221
-XAR_HEADER_SIZE = 28
 XAR_VERSION = 1
 XAR_CKSUM_SHA1 = 1
 
-PRIVATE_KEY_REGEXP = r'-+BEGIN PRIVATE KEY-+(.*?)-+END PRIVATE KEY-+'
-CERTIFICATE_REGEXP = r'-+BEGIN CERTIFICATE-+(.*?)-+END CERTIFICATE-+'
 
-
-def read_key(keyfile):
+def read_certificates_and_key(keyfile):
     with open(keyfile, 'r') as file:
         data = file.read()
-        match = re.search(PRIVATE_KEY_REGEXP, data, re.S)

[Sebastian Noack, 2016/08/17 12:53:45]

You can leave the with block after the data has been read.

However, as mentioned on the other review, it seems simpler (less duplication),
if you just combine those two function:

  def read_bio(keyfile):
      with open(keyfile, 'r') as file:
          data = file.read()

      certs = []
      key = None
      for match in re.finditer(r'-+BEGIN (.*?)-+(.*?)-+END \1-+', data, re.S):
          what, content = match.groups()
          if what == 'CERTIFICATE':
              certs.append(content)
          if what == 'PRIVATE KEY':
              key = RSA.importKey(content)

      if not key:
          raise Exception('Cound not find private key in file')

      return certs, key

[Wladimir Palant, 2016/08/17 14:11:40]

Done.

-        if not match:
-            raise Exception('Cound not find private key in file')
-        return RSA.importKey(match.group(0))
 
+    certificates = []
+    key = None
+    for match in re.finditer(r'-+BEGIN (.*?)-+(.*?)-+END \1-+', data, re.S):
+        section = match.group(1)
+        if section == 'CERTIFICATE':
+            certificates.append(re.sub(r'\s+', '', match.group(2)))
+        elif section == 'PRIVATE KEY':
+            key = RSA.importKey(match.group(0))
+    if not key:
+        raise Exception('Could not find private key in file')
 
-def read_certificates(keyfile):
-    certificates = []
-    with open(keyfile, 'r') as file:
-        data = file.read()
-        for match in re.finditer(CERTIFICATE_REGEXP, data, re.S):
-            certificates.append(re.sub(r'\s+', '', match.group(1)))
-    return certificates
+    return certificates, key
 
 
 def get_checksum(data):
     return SHA.new(data).digest()
 
 
 def get_hexchecksum(data):
     return SHA.new(data).hexdigest()
 
 
@@ skipping 42 matching lines @@
             'offset': offset,
         }
         file_id += 1
         offset += len(compressed)
         directory_stack[-1][1].append(file)
         compressed_data.append(compressed)
     return compressed_data
 
 
 def create(archivepath, contents, keyfile):
-    key = read_key(keyfile)
+    certificates, key = read_certificates_and_key(keyfile)
     checksum_length = len(get_checksum(''))

[Sebastian Noack, 2016/08/17 12:53:45]

No need to hash any (empty) data to get the digest size:

  SHA.SHA1Hash.digest_size

[Wladimir Palant, 2016/08/17 14:11:40]

Strictly speaking - no, it's not necessary. However, I'd rather not spread
implementation details throughout the code. It doesn't have to be SHA1 and I
strongly suspect that at some point in the future it longer will be.

     params = {
-        'certificates': read_certificates(keyfile),
+        'certificates': certificates,
 
         # Timestamp epoch starts at 2001-01-01T00:00:00.000Z
         'timestamp_numerical': time.time() - 978307200,
         'timestamp_iso': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
 
         'checksum': {
             'offset': 0,
             'size': checksum_length,
         },
         'signature': {
             'offset': checksum_length,
             'size': len(get_signature(key, '')),
         },
         'files': [],
     }
 
     offset = params['signature']['offset'] + params['signature']['size']
     compressed_data = compress_files(contents, params['files'], offset)
 
     template = getTemplate('xartoc.xml.tmpl', autoEscape=True)
     toc_uncompressed = template.render(params).encode('utf-8')
     toc_compressed = zlib.compress(toc_uncompressed, 9)
 
     with open(archivepath, 'wb') as file:
         # The file starts with a minimalistic header
-        header = struct.pack('>IHHQQI', XAR_HEADER_MAGIC, XAR_HEADER_SIZE,

[Sebastian Noack, 2016/08/17 12:53:45]

Note that you could avoid hard-coding the header size:

  HEADER_STRUCT = struct.Struct('>IHHQQI')
  ...
  header = HEADER_STRUCT.pack(XAR_HEADER_MAGIC, HEADER_STRUCT.size, ...)

[Wladimir Palant, 2016/08/17 14:11:40]

Done.

-                             XAR_VERSION, len(toc_compressed),
-                             len(toc_uncompressed), XAR_CKSUM_SHA1)
-        file.write(header)
+        file.write(XAR_HEADER.pack(XAR_HEADER_MAGIC, XAR_HEADER.size,
+                                   XAR_VERSION, len(toc_compressed),
+                                   len(toc_uncompressed), XAR_CKSUM_SHA1))
 
         # It's followed up with a compressed XML table of contents
         file.write(toc_compressed)
 
         # Now the actual data, all the offsets are in the table of contents
         file.write(get_checksum(toc_compressed))
         file.write(get_signature(key, toc_compressed))
         for blob in compressed_data:
             file.write(blob)