Issue 5383 - Add tests for the Chrome and Firefox packagers

packagerChrome.py

 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import errno
+import glob
 import io
 import json
 import os
 import re
-from StringIO import StringIO
 import struct
+import subprocess
 import sys
-import collections
+import random
 
 from packager import (readMetadata, getDefaultFileName, getBuildVersion,
                       getTemplate, Files)
 
 defaultLocale = 'en_US'
 
 
 def getIgnoredFiles(params):
     return {'store.description'}
 
@@ skipping 14 matching lines @@
 
 def processFile(path, data, params):
     # We don't change anything yet, this function currently only exists here so
     # that it can be overridden if necessary.
     return data
 
 
 def makeIcons(files, filenames):
     icons = {}
     for filename in filenames:
-        png_tuple = (137, 'PNG', '\r\n', '\x1a', '\n')
-        header = struct.unpack('>B3s2scc', files[filename][0:8])

[Sebastian Noack, 2017/09/20 00:58:00]

There is no reason to split the magic number into separate parts.

Also if there are less bytes in the file than to be unpacked, this will cause
an exception that is less descriptive than the one you would raise otherwise.

So how about this?

  try:
    magic, width, height = struct.unpack_from('>8s8xii', files[filename])
  except struct.error:
    magic = None
  if magic != '\x89PNG\r\n\x1a\n':
    raise Exception(...)

[tlucas, 2017/09/20 08:52:49]

Done.

-        assert header == png_tuple, '{} is no valid PNG.'.format(filename)

[Vasily Kuznetsov, 2017/09/19 17:52:26]

Python's `assert` is not a reliable method to test for actually possible
erroneous conditions because it's removed if the code is compiled with
optimization [1]. It's only safe to rely on asserts during tests (since you know
that the tests won't run with optimizations) and my understanding is that for
real errors you should use a `raise` statement instead and raise a real
exception (not `AssertionError`).

This check is for a rather rare condition, but it's still an error that might
occur and that we (as developers of this script) have not control over. I think
we should use a `raise` here.

[1]:https://docs.python.org/2/reference/simple_stmts.html#grammar-token-assert_stmt

[Sebastian Noack, 2017/09/20 00:58:00]

I'd like to add that the general idea of programming with assertions (not only
in Python) is to test your code assumptions in order to make debugging easier.
Asserts are meant to detect programming mistakes, not to signal runtime errors
that might occur if the code is behaving as expected. That Python removes
assertions from optimized bytecode, and that you mostly find assertions in test
suites, is just the logical consequence.

[tlucas, 2017/09/20 08:52:49]

Thanks for the insight - restructured this code along with the comment above.

-
-        width, height = struct.unpack('>ii', files[filename][16:24])
+        try:
+            magic, width, height = struct.unpack_from('>8s8xii',
+                                                      files[filename])
+        except struct.error:
+            magic = None
+        if magic != '\x89PNG\r\n\x1a\n':
+            raise Exception(filename + ' is no valid PNG.')
         if(width != height):
             print >>sys.stderr, 'Warning: %s size is %ix%i, icon should be square' % (filename, width, height)
         icons[width] = filename
     return icons
 
 
 def createScriptPage(params, template_name, script_option):
     template = getTemplate(template_name, autoEscape=True)
     return template.render(
         basename=params['metadata'].get('general', 'basename'),
@@ skipping 69 matching lines @@
     # Normalize JSON structure
     licenseComment = re.compile(r'/\*.*?\*/', re.S)
     data = json.loads(re.sub(licenseComment, '', manifest, 1))
     if '_dummy' in data:
         del data['_dummy']
     manifest = json.dumps(data, sort_keys=True, indent=2)
 
     return manifest.encode('utf-8')
 
 
-def convertJS(params, files):
-    output_files = collections.OrderedDict()
-    args = {}
-
-    for item in params['metadata'].items('convert_js'):
-        name, value = item
-        filename, arg = re.search(r'^(.*?)(?:\[(.*)\])?$', name).groups()
-        if arg is None:
-            output_files[filename] = (value.split(), item.source)
-        else:
-            args.setdefault(filename, {})[arg] = value
-
-    template = getTemplate('modules.js.tmpl')
-
-    for filename, (input_files, origin) in output_files.iteritems():
-        if '/' in filename and not files.isIncluded(filename):
-            continue
-
-        current_args = args.get(filename, {})
-        current_args['autoload'] = [module for module in
-                                    current_args.get('autoload', '').split(',')
-                                    if module != '']
-
-        base_dir = os.path.dirname(origin)
-        modules = []
-
-        for input_filename in input_files:
-            module_name = os.path.splitext(os.path.basename(input_filename))[0]
-            prefix = os.path.basename(os.path.dirname(input_filename))
-            if prefix != 'lib':
-                module_name = '{}_{}'.format(prefix, module_name)
-            with open(os.path.join(base_dir, input_filename), 'r') as file:
-                modules.append((module_name, file.read().decode('utf-8')))
-            files.pop(input_filename, None)
-
-        files[filename] = template.render(
-            args=current_args,
-            basename=params['metadata'].get('general', 'basename'),
-            modules=modules,
-            type=params['type'],
-            version=params['metadata'].get('general', 'version')
-        ).encode('utf-8')
-
-
 def toJson(data):
     return json.dumps(
         data, ensure_ascii=False, sort_keys=True,
         indent=2, separators=(',', ': ')
     ).encode('utf-8') + '\n'
 
 
-def import_string_webext(data, key, source):
-    """Import a single translation from the source dictionary into data"""
-    data[key] = source
-
-
-def import_string_gecko(data, key, value):
-    """Import Gecko-style locales into data.
-
-    Only sets {'message': value} in the data-dictionary, after stripping
-    undesired Gecko-style access keys.
-    """
-    match = re.search(r'^(.*?)\s*\(&.\)$', value)
-    if match:
-        value = match.group(1)
-    else:
-        index = value.find('&')
-        if index >= 0:
-            value = value[0:index] + value[index + 1:]
-
-    data[key] = {'message': value}
+def create_bundles(params, files):
+    base_extension_path = params['baseDir']
+    info_templates = {
+        'chrome': 'chromeInfo.js.tmpl',
+        'edge': 'edgeInfo.js.tmpl',
+        'gecko': 'geckoInfo.js.tmpl'
+    }
+
+    # Historically we didn't use relative paths when requiring modules, so in
+    # order for webpack to know where to find them we need to pass in a list of
+    # resolve paths. Going forward we should always use relative paths, once we
+    # do that consistently this can be removed. See issues 5760, 5761 and 5762.
+    resolve_paths = [os.path.join(base_extension_path, dir, 'lib')
+                     for dir in ['', 'adblockpluscore', 'adblockplusui']]
+
+    info_template = getTemplate(info_templates[params['type']])
+    info_module = info_template.render(
+        basename=params['metadata'].get('general', 'basename'),
+        version=params['metadata'].get('general', 'version')
+    ).encode('utf-8')
+
+    configuration = {
+        'bundles': [],
+        'extension_path': base_extension_path,
+        'info_module': info_module,
+        'resolve_paths': resolve_paths,
+    }
+
+    for item in params['metadata'].items('bundles'):
+        name, value = item
+        base_item_path = os.path.dirname(item.source)
+
+        bundle_file = os.path.relpath(os.path.join(base_item_path, name),
+                                      base_extension_path)
+        entry_files = [os.path.join(base_item_path, module_path)
+                       for module_path in value.split()]
+        configuration['bundles'].append({
+            'bundle_name': bundle_file,
+            'entry_points': entry_files,
+        })
+
+    cmd = ['node', os.path.join(os.path.dirname(__file__), 'webpack_runner.js')]
+    process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                               stdin=subprocess.PIPE)
+    output = process.communicate(input=toJson(configuration))[0]
+    if process.returncode != 0:
+        raise subprocess.CalledProcessError(process.returncode, cmd=cmd)
+    output = json.loads(output)
+
+    # Clear the mapping for any files included in a bundle, to avoid them being
+    # duplicated in the build.
+    for to_ignore in output['included']:
+        files.pop(to_ignore, None)
+
+    for bundle in output['files']:
+        files[bundle] = output['files'][bundle].encode('utf-8')
 
 
 def import_locales(params, files):
-    import localeTools
-
-    # FIXME: localeTools doesn't use real Chrome locales, it uses dash as
-    # separator instead.
-    convert_locale_code = lambda code: code.replace('-', '_')
-
-    # We need to map Chrome locales to Gecko locales. Start by mapping Chrome
-    # locales to themselves, merely with the dash as separator.
-    locale_mapping = {convert_locale_code(l): l for l in localeTools.chromeLocales}
-
-    # Convert values to Crowdin locales first (use Chrome => Crowdin mapping).
-    for chrome_locale, crowdin_locale in localeTools.langMappingChrome.iteritems():
-        locale_mapping[convert_locale_code(chrome_locale)] = crowdin_locale
-
-    # Now convert values to Gecko locales (use Gecko => Crowdin mapping).
-    reverse_mapping = {v: k for k, v in locale_mapping.iteritems()}
-    for gecko_locale, crowdin_locale in localeTools.langMappingGecko.iteritems():
-        if crowdin_locale in reverse_mapping:
-            locale_mapping[reverse_mapping[crowdin_locale]] = gecko_locale
-
-    for target, source in locale_mapping.iteritems():
-        targetFile = '_locales/%s/messages.json' % target
-        if not targetFile in files:
-            continue
-
-        for item in params['metadata'].items('import_locales'):
-            fileName, keys = item
-            parts = map(lambda n: source if n == '*' else n, fileName.split('/'))
-            sourceFile = os.path.join(os.path.dirname(item.source), *parts)
-            incompleteMarker = os.path.join(os.path.dirname(sourceFile), '.incomplete')
-            if not os.path.exists(sourceFile) or os.path.exists(incompleteMarker):
-                continue
-
-            data = json.loads(files[targetFile].decode('utf-8'))
+    for item in params['metadata'].items('import_locales'):
+        filename, keys = item
+        for sourceFile in glob.glob(os.path.join(os.path.dirname(item.source),
+                                                 *filename.split('/'))):
+            locale = sourceFile.split(os.path.sep)[-2]
+            targetFile = os.path.join('_locales', locale, 'messages.json')
+            data = json.loads(files.get(targetFile, '{}').decode('utf-8'))
 
             try:
-                # The WebExtensions (.json) and Gecko format provide
-                # translations differently and/or provide additional
-                # information like e.g. "placeholders". We want to adhere to
-                # that and preserve the addtional info.
-                if sourceFile.endswith('.json'):
-                    with io.open(sourceFile, 'r', encoding='utf-8') as handle:
-                        sourceData = json.load(handle)
-                    import_string = import_string_webext
-                else:
-                    sourceData = localeTools.readFile(sourceFile)
-                    import_string = import_string_gecko
+                with io.open(sourceFile, 'r', encoding='utf-8') as handle:
+                    sourceData = json.load(handle)
 
                 # Resolve wildcard imports
-                if keys == '*' or keys == '=*':
+                if keys == '*':
                     importList = sourceData.keys()
                     importList = filter(lambda k: not k.startswith('_'), importList)
-                    if keys == '=*':
-                        importList = map(lambda k: '=' + k, importList)
                     keys = ' '.join(importList)
 
                 for stringID in keys.split():
-                    noMangling = False
-                    if stringID.startswith('='):
-                        stringID = stringID[1:]
-                        noMangling = True
-
                     if stringID in sourceData:
-                        if noMangling:
-                            key = re.sub(r'\W', '_', stringID)
-                        else:
-                            key = re.sub(r'\..*', '', parts[-1]) + '_' + re.sub(r'\W', '_', stringID)
-                        if key in data:
-                            print 'Warning: locale string %s defined multiple times' % key
-
-                        import_string(data, key, sourceData[stringID])
+                        if stringID in data:
+                            print ('Warning: locale string {} defined multiple'
+                                   ' times').format(stringID)
+
+                        data[stringID] = sourceData[stringID]
             except Exception as e:
                 print 'Warning: error importing locale data from %s: %s' % (sourceFile, e)
 
             files[targetFile] = toJson(data)
 
 
 def truncate(text, length_limit):
     if len(text) <= length_limit:
         return text
     return text[:length_limit - 1].rstrip() + u'\u2026'
 
 
-def fixTranslationsForCWS(files):
-    # Chrome Web Store requires messages used in manifest.json to be present in
-    # all languages. It also enforces length limits for extension names and
-    # descriptions.
+def fix_translations_for_chrome(files):
     defaults = {}
     data = json.loads(files['_locales/%s/messages.json' % defaultLocale])
     for match in re.finditer(r'__MSG_(\S+)__', files['manifest.json']):
         name = match.group(1)
         defaults[name] = data[name]
 
     limits = {}
     manifest = json.loads(files['manifest.json'])
     for key, limit in (('name', 45), ('description', 132), ('short_name', 12)):
         match = re.search(r'__MSG_(\S+)__', manifest.get(key, ''))
         if match:
             limits[match.group(1)] = limit
 
-    for filename in files:
-        if not filename.startswith('_locales/') or not filename.endswith('/messages.json'):
+    for path in list(files):
+        match = re.search(r'^_locales/(?:es_(AR|CL|(MX))|[^/]+)/(.*)', path)
+        if not match:
             continue
 
-        data = json.loads(files[filename])
-        for name, info in defaults.iteritems():
-            data.setdefault(name, info)
-        for name, limit in limits.iteritems():
-            if name in data:
-                data[name]['message'] = truncate(data[name]['message'], limit)
-        files[filename] = toJson(data)
+        # The Chrome Web Store requires messages used in manifest.json to
+        # be present in all languages, and enforces length limits on
+        # extension name and description.
+        is_latam, is_mexican, filename = match.groups()
+        if filename == 'messages.json':
+            data = json.loads(files[path])
+            for name, info in defaults.iteritems():
+                data.setdefault(name, info)
+            for name, limit in limits.iteritems():
+                info = data.get(name)
+                if info:
+                    info['message'] = truncate(info['message'], limit)
+            files[path] = toJson(data)
+
+        # Chrome combines Latin American dialects of Spanish into es-419.
+        if is_latam:
+            data = files.pop(path)
+            if is_mexican:
+                files['_locales/es_419/' + filename] = data
 
 
 def signBinary(zipdata, keyFile):
     from Crypto.Hash import SHA
     from Crypto.PublicKey import RSA
     from Crypto.Signature import PKCS1_v1_5
 
     try:
         with open(keyFile, 'rb') as file:
             key = RSA.importKey(file.read())
@@ skipping 18 matching lines @@
         file = open(outputFile, 'wb')
     else:
         file = outputFile
     if pubkey != None and signature != None:
         file.write(struct.pack('<4sIII', 'Cr24', 2, len(pubkey), len(signature)))
         file.write(pubkey)
         file.write(signature)
     file.write(zipdata)
 
 
+def add_devenv_requirements(files, metadata, params):
+    files.read(
+        os.path.join(os.path.dirname(__file__), 'chromeDevenvPoller__.js'),
+        relpath='devenvPoller__.js',
+    )
+    files['devenvVersion__'] = str(random.random())
+
+    if metadata.has_option('general', 'testScripts'):
+        files['qunit/index.html'] = createScriptPage(
+            params, 'testIndex.html.tmpl', ('general', 'testScripts')
+        )
+
+
 def createBuild(baseDir, type='chrome', outFile=None, buildNum=None, releaseBuild=False, keyFile=None, devenv=False):
     metadata = readMetadata(baseDir, type)
     version = getBuildVersion(baseDir, metadata, releaseBuild, buildNum)
 
     if outFile == None:
-        if type == 'gecko-webext':
+        if type == 'gecko':
             file_extension = 'xpi'
         else:
             file_extension = 'crx' if keyFile else 'zip'
         outFile = getDefaultFileName(metadata, version, file_extension)
 
     params = {
         'type': type,
         'baseDir': baseDir,
         'releaseBuild': releaseBuild,
         'version': version,
         'devenv': devenv,
         'metadata': metadata,
     }
 
     mapped = metadata.items('mapping') if metadata.has_section('mapping') else []
     files = Files(getPackageFiles(params), getIgnoredFiles(params),
                   process=lambda path, data: processFile(path, data, params))
 
     files.readMappedFiles(mapped)
     files.read(baseDir, skip=[opt for opt, _ in mapped])
 
-    if metadata.has_section('convert_js'):
-        convertJS(params, files)
+    if metadata.has_section('bundles'):
+        create_bundles(params, files)
 
     if metadata.has_section('preprocess'):
         files.preprocess(
             [f for f, _ in metadata.items('preprocess')],
             {'needsExt': True}
         )
 
     if metadata.has_section('import_locales'):
         import_locales(params, files)
 
     files['manifest.json'] = createManifest(params, files)
     if type == 'chrome':
-        fixTranslationsForCWS(files)
+        fix_translations_for_chrome(files)
 
     if devenv:
-        import buildtools
-        import random
-        files.read(os.path.join(buildtools.__path__[0], 'chromeDevenvPoller__.js'), relpath='devenvPoller__.js')
-        files['devenvVersion__'] = str(random.random())
-
-        if metadata.has_option('general', 'testScripts'):
-            files['qunit/index.html'] = createScriptPage(
-                params, 'testIndex.html.tmpl', ('general', 'testScripts')
-            )
+        add_devenv_requirements(files, metadata, params)
 
     zipdata = files.zipToString()
     signature = None
     pubkey = None
     if keyFile != None:
         signature = signBinary(zipdata, keyFile)
         pubkey = getPublicKey(keyFile)
     writePackage(outFile, pubkey, signature, zipdata)