Issue 5383 - Add tests for the Chrome and Firefox packagers

tests/test_packagerChrome.py

+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import re
+import json
+import zipfile
+
+from xml.etree import ElementTree
+
+import pytest
+
+from buildtools import packager, packagerChrome
+from buildtools.tests.tools import copy_metadata
+
+ICON_SIZES = [32, 48, 64, 256, 53]
+
+MINIMUM_MANIFEST = [
+    ('manifest_version', 2),
+    ('author', 'Eyeo GmbH'),
+    ('permissions', [
+        'tabs',
+        '<all_urls>',
+        'contextMenus',
+        'webRequest',
+        'webRequestBlocking',
+        'webNavigation',
+        'storage',
+        'unlimitedStorage',
+        'notifications',
+    ]),
+    ('minimum_chrome_version', '49.0'),
+    ('minimum_opera_version', '36.0'),
+    ('devtools_page', 'devtools.html'),
+    ('description', '__MSG_description__'),
+    ('default_locale', 'en_US'),
+    ('short_name', '__MSG_name__'),
+    ('storage', {'managed_schema': 'managed-storage-schema.json'}),
+    ('options_ui', {'open_in_tab': True, 'page': 'options.html'}),
+]
+
+
+@pytest.fixture
+def chrome_metadata(tmpdir):
+    filename = 'metadata.chrome'
+    copy_metadata(filename, tmpdir)
+
+    return packager.readMetadata(str(tmpdir), 'chrome')
+
+
+@pytest.fixture
+def gecko_webext_metadata(tmpdir, chrome_metadata):
+    filename = 'metadata.gecko-webext'
+    copy_metadata(filename, tmpdir)
+
+    # gecko-webext metadata inherits from chrome metadata, assure existence
+    assert chrome_metadata is not None
+
+    return packager.readMetadata(str(tmpdir), 'gecko-webext')
+
+
+@pytest.fixture
+def icons(tmpdir):
+    """Valid .png files for testing make_icons"""
+    from PIL import Image
+
+    paths = []
+    icon_dir = tmpdir.mkdir('tmp_icons')
+
+    for size in ICON_SIZES:
+        img_path = str(icon_dir.join('abp-{}.png'.format(size)))
+        img = Image.new('1', (size, size if size != 53 else size + 1), 0)
+        img.save(img_path)
+        paths.append(img_path)
+
+    return paths
+
+
+@pytest.fixture
+def keyfile(tmpdir):
+    """Test-privatekey for signing files"""
+
+    content = """-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA0P06DVxOWZ97cvBZWz7DV0DYTFF7Cdrxnu/sCxnHp8IEq+sM
+rqodp2i8eTsddnthTnA93wiFiFXz6/9liQKrSBtogEGG2tmrT73jy5mdOehm7h6m
+Ujy6wLuZQiuY/RWJ6453/0H3bKCA8IlIdU8Pkkn5DJWH+YiiSUD01U5ZMLn8m/+O
+D/r9CLsgWoJV4VCXyzMItcc8hlRbiSe/3DQmZOPbF0Bd1Jv6lNA4RG0U8oA6q5s1
+5w3n//FjNOreVocyxgWWWGk/awlkJme1xn3yB3yh74b51FHsr9vhiTjcs+QNO1BS
+9fA/oavzDOs1Q4fLnTbKn6Jlg5OAjWtiq4vjSQIDAQABAoH/fojTnUNGLP1iwTTE
+5Xoay7l3PL4YwN7PbGvXfuEdAXV3Xp/yDc7yJWpEsyIXtKT/RX6v91oxf1qLVVhN
+Iad8DSyLGRyTie5Aywct5RgdGfKcX5AvI5uhdxAeuvGqr5Fa8ERSYzqNlDeZ2glE
+1cIIq4oeQIBI08zmdXPeyUemuNjK8TEWKOTMOjDLhav/DMviI1cvqp5VIh3RseMr
+1dwprXe0/LWuCh0BrCZZNH/0smd2etXEq06RhgnE2sZkowktNhSgc2pqls92FT88
+ObjI/jRXf6OwK7HxIJ1tMKlk/VCvAI2PNWi6E7IZbqN7Rg7859uWybGZkP/h1Qf5
+fRitAoGBAOJnP3kgavnrUJ482R8MNjOwtqbLf4obS5Y/olOHUi+mu2x+EwxgDtaE
+sXeMjoIAuZvDoAIG0Hn7+J5ZSYkXT9aXQy/odVWDIpFNFFXNFnS73sAWay/FsMJc
+NvgbbWJk9fCq23O7ihuSkfM+zh8dAZoHzV4aw3DsZj37yzuTmTJbAoGBAOxPM6Wy
+fENT8ECGl9d/j8QBV4LvjrfIbz/RcCxgZHG50ELnxCo7FiYRg3d2DjR6Q+EpjclX
+CC+IwCGGf+RBz53WzJPOBkz+plPZzrhhhtpBuWgegeF+TmYT9pPd9XRHJs5qFCak
+J3xxgBur3NDtejNLWBc4bgFfrv8hdYeuIKorAoGBAJplWNbssadvv1G6I0NWG5yS
+lW0X+Akh5iE3kiaucPDIHqa1L55P366RXUku1Hx5rBo6hWL8bK3TlM/ACjLwb+Ti
+0NHaEDJZtHgsfYKp0veWqyiJ4Vz5zzosktwOMEFaopIWooPBUETPZrLgkMaNDGuT
+iIz1aXUX8f2xOf6OAHpjAoGBAMNqq9c2xrQW0fNKWn8HKih4w5mv6WHqCrXHyO+p
+tualNqhdaUdTHXnVudYsdorHISMubeY2ZrqIZ/pRc6mbEsoAO6VvFp7NZ1aoI98u
+J4qOF3kW4WlMPiEEGUEmqDjELj88UfWygkTSx5Iaibzs5cVNZUeujqnsKpcpYDwX
+r0OJAoGBAIWgnx72WjcFmL1kgOu2kH8tH5+cIAb/THX8lnCulQIlxss8n26NMMDa
+Kmh5gC2SOAe92cXG6jCYHZqZtl5gM6xGky1B9j7TDrXVPj+QDcItCuNN4zrdtiah
+US0KNMZKEMjxsaaMDPHj/bP9ULLbMHvtfvt4EnBQoVCDSZdfeBvZ
+-----END RSA PRIVATE KEY-----"""
+
+    keyfile_path = str(tmpdir.mkdir('rsakey').join('keyfile'))
+
+    with open(keyfile_path, 'wb') as key_fp:
+        key_fp.write(content)
+
+    return keyfile_path
+
+
+@pytest.mark.parametrize('release', [True, False])
+@pytest.mark.parametrize('devenv', [True, False])
+@pytest.mark.parametrize('ext_type', ['chrome', 'gecko-webext'])
+def test_create_manifest(srcdir, files, release, devenv, ext_type,
+                         chrome_metadata, gecko_webext_metadata):
+
+    if ext_type == 'chrome':

[Vasily Kuznetsov, 2017/08/10 19:48:28]

Instead of this you could do:

@pytest.mark.parametrize('ext_type,metadata', [
    ('chrome', chrome_metadata),
    ('gecko-webext', gecko_webext_metadata),
])

this would be a bit cleaner.

[tlucas, 2017/08/11 12:17:09]

I thought about that too and i definitely agree with you, but according to ->
https://github.com/pytest-dev/pytest/issues/349
parametrizing with a fixture is a not yet implemented proposal. I could call the
xxx_metadata()-functions in the decorator, but iirc that would call them for
every permutation of the parameters, which we don't want.

if you don't mind i'll leave this as is.

[Vasily Kuznetsov, 2017/08/11 16:46:00]

Yes, you're right.

+        metadata = chrome_metadata
+    else:
+        metadata = gecko_webext_metadata
+
+    version = packager.getBuildVersion(str(srcdir), metadata, release)
+    params = {
+        'type': ext_type,
+        'baseDir': str(srcdir),
+        'releaseBuild': release,
+        'version': version,
+        'devenv': devenv,
+        'metadata': metadata,
+    }
+    manifest = json.loads(packagerChrome.createManifest(params, files))
+
+    expected_values = MINIMUM_MANIFEST + [
+        ('name', '__MSG_name_' + ('devbuild__' if not release else '_')),
+        ('version', '1.2.3' + ('.0' if not release else '')),
+    ]
+
+    if ext_type == 'gecko-webext':
+        expected_values += [
+            ('applications', {
+                'gecko': {
+                    'id': '{d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d}',
+                    'strict_min_version': '50.0',
+                    }
+                })
+        ]
+        if not release:
+            expected_values[-1][1]['gecko'].update({
+                    'update_url': ''.join((
+                        'https://downloads.adblockplus.org/devbuilds/',
+                        'adblockplusfirefox/updates.json'))
+                })
+
+    for key, value in expected_values:
+        assert manifest.get(key, None) == value
+
+
+def test_sign_binary(files, keyfile):
+    from Crypto.Hash import SHA
+    from Crypto.PublicKey import RSA
+    from Crypto.Signature import PKCS1_v1_5
+
+    digest = SHA.new()
+
+    signature = packagerChrome.signBinary(files.zipToString(), keyfile)
+
+    with open(keyfile, 'r') as fp:
+        rsa_key = RSA.importKey(fp.read())
+
+    signer = PKCS1_v1_5.new(rsa_key)
+
+    digest.update(files.zipToString())
+    assert signer.verify(digest, signature)
+
+
+def test_get_public_key(keyfile):
+    expected = (
+        '30820122300d06092a864886f70d01010105000382010f003082010a0282010100d0'
+        'fd3a0d5c4e599f7b72f0595b3ec35740d84c517b09daf19eefec0b19c7a7c204abeb'
+        '0caeaa1da768bc793b1d767b614e703ddf08858855f3ebff658902ab481b68804186'
+        'dad9ab4fbde3cb999d39e866ee1ea6523cbac0bb99422b98fd1589eb8e77ff41f76c'
+        'a080f08948754f0f9249f90c9587f988a24940f4d54e5930b9fc9bff8e0ffafd08bb'
+        '205a8255e15097cb3308b5c73c86545b8927bfdc342664e3db17405dd49bfa94d038'
+        '446d14f2803aab9b35e70de7fff16334eade568732c6059658693f6b09642667b5c6'
+        '7df2077ca1ef86f9d451ecafdbe18938dcb3e40d3b5052f5f03fa1abf30ceb354387'
+        'cb9d36ca9fa2658393808d6b62ab8be3490203010001')
+
+    publickey = packagerChrome.getPublicKey(keyfile)
+    assert publickey.encode('hex') == expected
+
+
+@pytest.mark.usefixtures('chrome_metadata')
+@pytest.mark.parametrize('devenv', [True, False])
+def test_get_package_files(srcdir, devenv):
+    filenames = [
+        'foo.html',
+        'foo.js',
+        'foo.json',
+        'foo.xml',
+    ]
+
+    for filename in filenames:
+        srcdir.join(filename).write('', ensure=True)
+
+    params = {
+        'baseDir': str(srcdir),
+        'devenv': devenv,
+    }
+
+    files = packagerChrome.getPackageFiles(params)
+
+    assert ('qunit' in files) == devenv
+
+    for expected in ['icons', 'ui', 'skin', 'ext', '_locales', 'lib',
+                     'jquery-ui'] + [name for name in filenames]:
+        assert expected in files
+
+
+def test_make_icons(icons, capsys):
+    files = packager.Files(set(), set())
+    for path in icons:
+        files[path] = open(path, 'r').read()
+    result = packagerChrome.makeIcons(files, icons)
+
+    out, err = capsys.readouterr()
+
+    assert 'should be square' in err
+    assert set(result) == set(ICON_SIZES)
+
+
+def test_create_script_page(srcdir, chrome_metadata):
+    params = {'metadata': chrome_metadata}
+
+    template = packagerChrome.createScriptPage(
+        params, 'testIndex.html.tmpl', ('general', 'testScripts')
+    )
+
+    template_scripts = {elem.attrib['src'] for elem in
+                        ElementTree.fromstring(template).iter('script')}
+
+    for src in chrome_metadata.get('general', 'testScripts').split():
+        assert src in template_scripts
+
+
+def test_convert_js(srcdir, files, chrome_metadata):
+    params = {
+        'type': 'chrome',
+        'metadata': chrome_metadata,
+    }
+
+    packagerChrome.convertJS(params, files)
+    source = files['lib/foo.js']
+
+    assert 'var foo;' in source
+    assert 'var bar;' in source
+
+
+def test_to_json():
+    data = {'bar': ['foo', 'foobar', 1, True]}
+    data_json = packagerChrome.toJson(data)
+
+    assert data_json.endswith('\n')
+    json.loads(data_json)
+
+
+def test_import_gecko_locales(tmpdir, srcdir, files, chrome_metadata):
+    version = packager.getBuildVersion(str(srcdir), chrome_metadata, False)
+    params = {
+        'type': 'chrome',
+        'baseDir': str(srcdir),
+        'releaseBuild': False,
+        'version': version,
+        'devenv': False,
+        'metadata': chrome_metadata,
+    }
+
+    data = {'name': 'foo', 'name_devbuild': 'bar'}
+
+    t_dir = tmpdir.mkdir('_trans').mkdir('en-US')
+    t_dir.join('test.properties').write(
+        '\n'.join(

[Vasily Kuznetsov, 2017/08/10 19:48:28]

The following line would fit on this line too. Not a big deal, just would be a
bit easier to read.

[tlucas, 2017/08/11 12:17:09]

Done.

+            '{}={}'.format(k, v) for k, v in data.items()))
+
+    packagerChrome.importGeckoLocales(params, files)
+    trans_data = json.loads(files['_locales/en_US/messages.json'])
+
+    for key, value in data.items():
+        assert trans_data.get('test_' + key, {}).get('message', '') == value
+
+
+def test_fix_translations_for_cws(files):
+    files['_locales/de/messages.json'] = packagerChrome.toJson({})
+    files['manifest.json'] = packagerChrome.toJson(

[Vasily Kuznetsov, 2017/08/10 19:48:28]

Here the following line would also fit into this line, I think.

[tlucas, 2017/08/11 12:17:08]

Done.

+        dict(MINIMUM_MANIFEST))
+    packagerChrome.fixTranslationsForCWS(files)
+
+    ger_messages = json.loads(files['_locales/de/messages.json'])
+    eng_messages = json.loads(files['_locales/en_US/messages.json'])
+
+    # Check for inserted translations, none were present in 'de' beforehand
+    for match in re.finditer(r'__MSG_(\S+)__', files['manifest.json']):
+        name = match.group(1)
+        assert ger_messages[name]['message'] == eng_messages[name]['message']
+
+    # CWS enforces max-lengths on string
+    assert len(ger_messages['description']['message']) <= 132
+    assert len(ger_messages['name']['message']) <= 12
+
+
+@pytest.mark.usefixtures('files', 'gecko_webext_metadata', 'chrome_metadata')
+@pytest.mark.parametrize('release', [True, False])
+@pytest.mark.parametrize('devenv', [True, False])
+@pytest.mark.parametrize('ext_type', ['chrome', 'gecko-webext'])
+@pytest.mark.parametrize('key', [True, False])
+def test_create_build(srcdir, tmpdir, release, devenv, ext_type,
+                      key, keyfile):
+
+    basefiles = [
+        'manifest.json',
+        'lib/foo.js',
+        'foo/logo_50.png',
+        '_locales/en_US/messages.json',
+        '_locales/en-US/test.properties',
+        'logo_44.png',
+        'icons/logo_150.png',
+    ]
+
+    out_file = str(tmpdir.join('out'))
+
+    packagerChrome.createBuild(
+        outFile=out_file,
+        baseDir=str(srcdir),
+        type=ext_type,
+        releaseBuild=release,
+        keyFile=keyfile if key else None,
+        devenv=devenv)
+
+    with zipfile.ZipFile(out_file, 'r') as zipfp:
+        zipfp.testzip()
+
+        filenames = [zipinfo.filename for zipinfo in zipfp.infolist()]
+        for filename in basefiles if not devenv else basefiles + [
+                    'devenvPoller__.js',
+                    'devenvVersion__',
+                    'qunit/index.html',
+                ]:
+            assert filename in filenames