Issue 4340 - Drop dependency on external xar tool

xarfile.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import base64
 import re
 import struct
 import time
-from xml.etree import ElementTree
 import zlib
 
 from Crypto.Hash import SHA
 from Crypto.PublicKey import RSA
 from Crypto.Signature import PKCS1_v1_5
 
+from buildtools.packager import getTemplate
+
+XAR_HEADER = struct.Struct('>IHHQQI')
 XAR_HEADER_MAGIC = 0x78617221
-XAR_HEADER_SIZE = 28
 XAR_VERSION = 1
 XAR_CKSUM_SHA1 = 1
 
-def read_key(keyfile):
+
+def read_certificates_and_key(keyfile):
     with open(keyfile, 'r') as file:
         data = file.read()
-        data = re.sub(r'(-+END PRIVATE KEY-+).*', r'\1', data, flags=re.S)
-        return RSA.importKey(data)
 
-def read_certificates(keyfile):
-    certs = []
-    with open(keyfile, 'r') as file:
-        data = file.read()
-        for match in re.finditer(r'-+BEGIN CERTIFICATE-+(.*?)-+END CERTIFICATE-+', data, re.S):
-            certs.append(base64.b64decode(match.group(1)))
-    return certs
+    certificates = []
+    key = None
+    for match in re.finditer(r'-+BEGIN (.*?)-+(.*?)-+END \1-+', data, re.S):
+        section = match.group(1)
+        if section == 'CERTIFICATE':
+            certificates.append(re.sub(r'\s+', '', match.group(2)))
+        elif section == 'PRIVATE KEY':
+            key = RSA.importKey(match.group(0))
+    if not key:
+        raise Exception('Could not find private key in file')
+
+    return certificates, key
+
 
 def get_checksum(data):
     return SHA.new(data).digest()
 
+
 def get_hexchecksum(data):
     return SHA.new(data).hexdigest()
+
 
 def get_signature(key, data):
     return PKCS1_v1_5.new(key).sign(SHA.new(data))
 
+
 def compress_files(filedata, root, offset):
-    files = []
-    filedata = sorted(filedata)
-    directory_stack = [{'path': '', 'element': root}]
+    compressed_data = []
+    filedata = sorted(filedata.iteritems())
+    directory_stack = [('', root)]
     file_id = 1
     for path, data in filedata:
         # Remove directories that are done
-        while True:
-            directory = directory_stack[-1]
-            directory_path = directory['path']
-            if path.startswith(directory_path):
-                break
+        while not path.startswith(directory_stack[-1][0]):
             directory_stack.pop()
 
         # Add new directories
+        directory_path = directory_stack[-1][0]
         relpath = path[len(directory_path):]
         while '/' in relpath:
-            directory_name, relpath = relpath.split('/', 1)
-            directory_path += directory_name + '/'
-            element = ElementTree.SubElement(directory['element'], 'file')
+            name, relpath = relpath.split('/', 1)
+            directory_path += name + '/'
             directory = {
-                'path': directory_path,
-                'element': element,
+                'id': file_id,
+                'name': name,
+                'type': 'directory',
+                'mode': '0755',
+                'children': [],
             }
-            element.set('id', str(file_id))
             file_id += 1
-            ElementTree.SubElement(element, 'name').text = directory_name
-            ElementTree.SubElement(element, 'type').text = 'directory'
-            ElementTree.SubElement(element, 'mode').text = '0755'
-            directory_stack.append(directory)
+            directory_stack[-1][1].append(directory)
+            directory_stack.append((directory_path, directory['children']))
 
         # Add the actual file
-        element = ElementTree.SubElement(directory['element'], 'file')
-        element.set('id', str(file_id))
+        compressed = zlib.compress(data, 9)
+        file = {
+            'id': file_id,
+            'name': relpath,
+            'type': 'file',
+            'mode': '0644',
+            'checksum_uncompressed': get_hexchecksum(data),
+            'size_uncompressed': len(data),
+            'checksum_compressed': get_hexchecksum(compressed),
+            'size_compressed': len(compressed),
+            'offset': offset,
+        }
         file_id += 1
-        ElementTree.SubElement(element, 'name').text = relpath
-        ElementTree.SubElement(element, 'type').text = 'file'
-        ElementTree.SubElement(element, 'mode').text = '0644'
+        offset += len(compressed)
+        directory_stack[-1][1].append(file)
+        compressed_data.append(compressed)
+    return compressed_data
 
-        datatag = ElementTree.SubElement(element, 'data')
-        ElementTree.SubElement(datatag, 'extracted-checksum', {'style': 'sha1'}).text = get_hexchecksum(data)
-        ElementTree.SubElement(datatag, 'size').text = str(len(data))
-
-        compressed = zlib.compress(data, 9)
-        ElementTree.SubElement(datatag, 'encoding', {'style': 'application/x-gzip'})
-        ElementTree.SubElement(datatag, 'archived-checksum', {'style': 'sha1'}).text = get_hexchecksum(compressed)
-        ElementTree.SubElement(datatag, 'offset').text = str(offset)
-        ElementTree.SubElement(datatag, 'length').text = str(len(compressed))
-        offset += len(compressed)
-
-        files.append(compressed)
-    return files
 
 def create(archivepath, contents, keyfile):
-    key = read_key(keyfile)
-    certs = read_certificates(keyfile)
+    certificates, key = read_certificates_and_key(keyfile)
+    checksum_length = len(get_checksum(''))
+    params = {
+        'certificates': certificates,
 
-    root = ElementTree.Element('xar')
-    toc = ElementTree.SubElement(root, 'toc')
+        # Timestamp epoch starts at 2001-01-01T00:00:00.000Z
+        'timestamp_numerical': time.time() - 978307200,
+        'timestamp_iso': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
 
-    creation_time = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
-    ElementTree.SubElement(toc, 'creation-time').text = creation_time
+        'checksum': {
+            'offset': 0,
+            'size': checksum_length,
+        },
+        'signature': {
+            'offset': checksum_length,
+            'size': len(get_signature(key, '')),
+        },
+        'files': [],
+    }
 
-    # Timestamp epoch starts at 2001-01-01T00:00:00.000Z
-    sign_time = str(time.time() - 978307200)
-    ElementTree.SubElement(toc, 'signature-creation-time').text = sign_time
+    offset = params['signature']['offset'] + params['signature']['size']
+    compressed_data = compress_files(contents, params['files'], offset)
 
-    offset = 0
-
-    checksum_size = len(get_checksum(''))
-    checksum = ElementTree.SubElement(toc, 'checksum', {'style': 'sha1'})
-    ElementTree.SubElement(checksum, 'offset').text = str(offset)
-    ElementTree.SubElement(checksum, 'size').text = str(checksum_size)
-    offset += checksum_size
-
-    signature_size = len(get_signature(key, ''))
-    signature = ElementTree.SubElement(toc, 'signature', {'style': 'RSA'})
-    ElementTree.SubElement(signature, 'offset').text = str(offset)
-    ElementTree.SubElement(signature, 'size').text = str(signature_size)
-    offset += signature_size
-
-    keyinfo = ElementTree.SubElement(signature, 'KeyInfo')
-    keyinfo.set('xmlns', 'http://www.w3.org/2000/09/xmldsig#')
-    x509data = ElementTree.SubElement(keyinfo, 'X509Data')
-    for cert in certs:
-        ElementTree.SubElement(x509data, 'X509Certificate').text = base64.b64encode(cert)
-
-    files = compress_files(contents.iteritems(), toc, offset)
-
-    toc_uncompressed = ElementTree.tostring(root).encode('utf-8')
+    template = getTemplate('xartoc.xml.tmpl', autoEscape=True)
+    toc_uncompressed = template.render(params).encode('utf-8')
     toc_compressed = zlib.compress(toc_uncompressed, 9)
 
     with open(archivepath, 'wb') as file:
         # The file starts with a minimalistic header
-        header = struct.pack('>IHHQQI', XAR_HEADER_MAGIC, XAR_HEADER_SIZE,
-            XAR_VERSION, len(toc_compressed), len(toc_uncompressed),
-            XAR_CKSUM_SHA1)
-        file.write(header)
+        file.write(XAR_HEADER.pack(XAR_HEADER_MAGIC, XAR_HEADER.size,
+                                   XAR_VERSION, len(toc_compressed),
+                                   len(toc_uncompressed), XAR_CKSUM_SHA1))
 
         # It's followed up with a compressed XML table of contents
         file.write(toc_compressed)
 
         # Now the actual data, all the offsets are in the table of contents
         file.write(get_checksum(toc_compressed))
         file.write(get_signature(key, toc_compressed))
-        for compressed in files:
-            file.write(compressed)
+        for blob in compressed_data:
+            file.write(blob)