Issue 4340 - Drop dependency on external xar tool

xarfile.py

Index: xarfile.py
===================================================================
new file mode 100644
--- /dev/null
+++ b/xarfile.py
@@ -0,0 +1,144 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import re
+import struct
+import time
+import zlib
+
+from Crypto.Hash import SHA
+from Crypto.PublicKey import RSA
+from Crypto.Signature import PKCS1_v1_5
+
+from buildtools.packager import getTemplate
+
+XAR_HEADER_MAGIC = 0x78617221
+XAR_HEADER_SIZE = 28
+XAR_VERSION = 1
+XAR_CKSUM_SHA1 = 1
+
+PRIVATE_KEY_REGEXP = r'-+BEGIN PRIVATE KEY-+(.*?)-+END PRIVATE KEY-+'
+CERTIFICATE_REGEXP = r'-+BEGIN CERTIFICATE-+(.*?)-+END CERTIFICATE-+'
+
+
+def read_key(keyfile):
+    with open(keyfile, 'r') as file:
+        data = file.read()
+        match = re.search(PRIVATE_KEY_REGEXP, data, re.S)

[Sebastian Noack, 2016/08/17 12:53:45]

You can leave the with block after the data has been read.

However, as mentioned on the other review, it seems simpler (less duplication),
if you just combine those two function:

  def read_bio(keyfile):
      with open(keyfile, 'r') as file:
          data = file.read()

      certs = []
      key = None
      for match in re.finditer(r'-+BEGIN (.*?)-+(.*?)-+END \1-+', data, re.S):
          what, content = match.groups()
          if what == 'CERTIFICATE':
              certs.append(content)
          if what == 'PRIVATE KEY':
              key = RSA.importKey(content)

      if not key:
          raise Exception('Cound not find private key in file')

      return certs, key

[Wladimir Palant, 2016/08/17 14:11:40]

Done.

+        if not match:
+            raise Exception('Cound not find private key in file')
+        return RSA.importKey(match.group(0))
+
+
+def read_certificates(keyfile):
+    certificates = []
+    with open(keyfile, 'r') as file:
+        data = file.read()
+        for match in re.finditer(CERTIFICATE_REGEXP, data, re.S):
+            certificates.append(re.sub(r'\s+', '', match.group(1)))
+    return certificates
+
+
+def get_checksum(data):
+    return SHA.new(data).digest()
+
+
+def get_hexchecksum(data):
+    return SHA.new(data).hexdigest()
+
+
+def get_signature(key, data):
+    return PKCS1_v1_5.new(key).sign(SHA.new(data))
+
+
+def compress_files(filedata, root, offset):
+    compressed_data = []
+    filedata = sorted(filedata.iteritems())
+    directory_stack = [('', root)]
+    file_id = 1
+    for path, data in filedata:
+        # Remove directories that are done
+        while not path.startswith(directory_stack[-1][0]):
+            directory_stack.pop()
+
+        # Add new directories
+        directory_path = directory_stack[-1][0]
+        relpath = path[len(directory_path):]
+        while '/' in relpath:
+            name, relpath = relpath.split('/', 1)
+            directory_path += name + '/'
+            directory = {
+                'id': file_id,
+                'name': name,
+                'type': 'directory',
+                'mode': '0755',
+                'children': [],
+            }
+            file_id += 1
+            directory_stack[-1][1].append(directory)
+            directory_stack.append((directory_path, directory['children']))
+
+        # Add the actual file
+        compressed = zlib.compress(data, 9)
+        file = {
+            'id': file_id,
+            'name': relpath,
+            'type': 'file',
+            'mode': '0644',
+            'checksum_uncompressed': get_hexchecksum(data),
+            'size_uncompressed': len(data),
+            'checksum_compressed': get_hexchecksum(compressed),
+            'size_compressed': len(compressed),
+            'offset': offset,
+        }
+        file_id += 1
+        offset += len(compressed)
+        directory_stack[-1][1].append(file)
+        compressed_data.append(compressed)
+    return compressed_data
+
+
+def create(archivepath, contents, keyfile):
+    key = read_key(keyfile)
+    checksum_length = len(get_checksum(''))

[Sebastian Noack, 2016/08/17 12:53:45]

No need to hash any (empty) data to get the digest size:

  SHA.SHA1Hash.digest_size

[Wladimir Palant, 2016/08/17 14:11:40]

Strictly speaking - no, it's not necessary. However, I'd rather not spread
implementation details throughout the code. It doesn't have to be SHA1 and I
strongly suspect that at some point in the future it longer will be.

+    params = {
+        'certificates': read_certificates(keyfile),
+
+        # Timestamp epoch starts at 2001-01-01T00:00:00.000Z
+        'timestamp_numerical': time.time() - 978307200,
+        'timestamp_iso': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
+
+        'checksum': {
+            'offset': 0,
+            'size': checksum_length,
+        },
+        'signature': {
+            'offset': checksum_length,
+            'size': len(get_signature(key, '')),
+        },
+        'files': [],
+    }
+
+    offset = params['signature']['offset'] + params['signature']['size']
+    compressed_data = compress_files(contents, params['files'], offset)
+
+    template = getTemplate('xartoc.xml.tmpl', autoEscape=True)
+    toc_uncompressed = template.render(params).encode('utf-8')
+    toc_compressed = zlib.compress(toc_uncompressed, 9)
+
+    with open(archivepath, 'wb') as file:
+        # The file starts with a minimalistic header
+        header = struct.pack('>IHHQQI', XAR_HEADER_MAGIC, XAR_HEADER_SIZE,

[Sebastian Noack, 2016/08/17 12:53:45]

Note that you could avoid hard-coding the header size:

  HEADER_STRUCT = struct.Struct('>IHHQQI')
  ...
  header = HEADER_STRUCT.pack(XAR_HEADER_MAGIC, HEADER_STRUCT.size, ...)

[Wladimir Palant, 2016/08/17 14:11:40]

Done.

+                             XAR_VERSION, len(toc_compressed),
+                             len(toc_uncompressed), XAR_CKSUM_SHA1)
+        file.write(header)
+
+        # It's followed up with a compressed XML table of contents
+        file.write(toc_compressed)
+
+        # Now the actual data, all the offsets are in the table of contents
+        file.write(get_checksum(toc_compressed))
+        file.write(get_signature(key, toc_compressed))
+        for blob in compressed_data:
+            file.write(blob)