Issue 5383 - Add tests for the Chrome and Firefox packagers

tests/test_packagerWebExt.py

Index: tests/test_packagerWebExt.py
diff --git a/tests/test_packagerWebExt.py b/tests/test_packagerWebExt.py
new file mode 100644
index 0000000000000000000000000000000000000000..985f09e7f4775b7c2d951b8d19e4ecead63baa96
--- /dev/null
+++ b/tests/test_packagerWebExt.py
@@ -0,0 +1,294 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import shutil
+import json
+import re
+
+import pytest
+
+from buildtools import packager
+from buildtools.tests.tools import DirContent

[Vasily Kuznetsov, 2017/09/11 12:50:07]

These imports also can be combined.

[tlucas, 2017/09/12 11:32:10]

Acknowledged.

[tlucas, 2017/09/13 13:43:24]

Done.

+from buildtools.tests.tools import ZipContent
+from buildtools.tests.tools import copy_metadata
+from buildtools.tests.tools import run_webext_build
+from buildtools.tests.tools import assert_all_locales_present
+from buildtools.tests.tools import assert_manifest_content
+from buildtools.tests.tools import locale_files
+from buildtools.tests.conftest import ALL_LANGUAGES
+
+
+LOCALES_MODULE = {
+    'test.Foobar': {
+        'message': 'Ensuring dict-copy from modules for $domain$',
+        'description': 'test description',
+        'placeholders': {'content': '$1', 'example': 'www.adblockplus.org'}
+    }
+}
+
+DTD_TEST = ('<!ENTITY access.key "access key(&amp;a)">'
+            '<!ENTITY ampersand "foo &amp;-bar">')
+
+PROPERTIES_TEST = 'description=very descriptive!'
+
+
+@pytest.fixture
+def gecko_import(tmpdir):
+    tmpdir.mkdir('_imp').mkdir('en-US').join('gecko.dtd').write(DTD_TEST)
+
+
+@pytest.fixture
+def locale_modules(tmpdir):
+    mod_dir = tmpdir.mkdir('_modules')
+    lang_dir = mod_dir.mkdir('en-US')
+    lang_dir.join('module.json').write(json.dumps(LOCALES_MODULE))
+    lang_dir.join('unit.properties').write(json.dumps(PROPERTIES_TEST))
+
+
+@pytest.fixture
+def icons(srcdir):
+    icons_dir = srcdir.mkdir('icons')
+    for filename in ['abp-16.png', 'abp-19.png', 'abp-53.png']:
+        shutil.copy(
+            os.path.join(os.path.dirname(__file__), filename),
+            os.path.join(str(icons_dir), filename),
+        )
+
+
+@pytest.fixture
+def all_lang_locales(tmpdir):
+    return locale_files(ALL_LANGUAGES, '_locales', tmpdir)
+
+
+@pytest.fixture
+def chrome_metadata(tmpdir):
+    filename = 'metadata.chrome'
+    copy_metadata(filename, tmpdir)
+
+
+@pytest.fixture
+def gecko_webext_metadata(tmpdir, chrome_metadata):
+    filename = 'metadata.gecko-webext'
+    copy_metadata(filename, tmpdir)
+
+
+@pytest.fixture
+def keyfile(tmpdir):
+    """Test-privatekey for signing chrome release-package"""
+    return os.path.join(os.path.dirname(__file__), 'chrome_rsa.pem')
+
+
+@pytest.fixture
+def lib_files(tmpdir):
+    files = packager.Files(['lib'], set())
+    files['ext/a.js'] = 'var bar;'
+    files['lib/b.js'] = 'var foo;'
+
+    tmpdir.mkdir('lib').join('b.js').write(files['lib/b.js'])
+    tmpdir.mkdir('ext').join('a.js').write(files['ext/a.js'])
+
+    return files
+
+
+def assert_gecko_locale_conversion(package):
+    locale = json.loads(package.read('_locales/en_US/messages.json'))
+
+    assert locale.get('test_Foobar', {}) == LOCALES_MODULE['test.Foobar']
+    assert locale.get('access_key', {}) == {'message': 'access key'}
+    assert locale.get('ampersand', {}) == {'message': 'foo -bar'}
+    assert locale.get('_description', {}) == {'message': 'very descriptive!"'}
+
+
+def assert_convert_js(package, excluded=False):
+    libfoo = package.read('lib/foo.js')
+
+    assert 'var bar;' in libfoo
+    assert 'require.modules["ext_a"]' in libfoo
+
+    assert ('var foo;' in libfoo) != excluded
+    assert ('require.modules["b"]' in libfoo) != excluded
+
+
+def assert_devenv_scripts(package, devenv):
+    manifest = json.loads(package.read('manifest.json'))
+    filenames = package.namelist()
+    scripts = [
+            'ext/common.js',
+            'ext/background.js',
+    ]
+
+    if devenv:
+        assert 'qunit/index.html' in filenames
+        assert 'devenvPoller__.js' in filenames
+        assert 'devenvVersion__' in filenames
+
+        assert '../ext/common.js' in package.read('qunit/index.html')
+        assert '../ext/background.js' in package.read('qunit/index.html')
+
+        assert set(manifest['background']['scripts']) == set(
+            scripts + ['devenvPoller__.js']
+        )
+    else:
+        assert 'qunit/index.html' not in filenames
+
+        assert set(manifest['background']['scripts']) == set(scripts)
+
+
+def assert_base_files(package):
+    filenames = set(package.namelist())
+
+    assert 'bar.json' in filenames
+    assert 'manifest.json' in filenames
+    assert 'lib/foo.js' in filenames
+    assert 'foo/logo_50.png' in filenames
+    assert 'icons/logo_150.png' in filenames
+
+
+def assert_chrome_signature(filename, keyfile):
+    from struct import unpack

[Vasily Kuznetsov, 2017/09/11 12:50:07]

Is there any specific reason to have these imports are inside of the function?

[tlucas, 2017/09/12 11:32:11]

Not at all - merely a result of a speedy adjustment - will change :)

[tlucas, 2017/09/13 13:43:25]

Done.

+    from Crypto.Hash import SHA
+    from Crypto.PublicKey import RSA
+    from Crypto.Signature import PKCS1_v1_5
+
+    with open(filename, 'r') as fp:
+        content = fp.read()
+
+    _, _, l_pubkey, l_signature = unpack('<4sIII', content[:16])
+    signature = content[16 + l_pubkey: 16 + l_pubkey + l_signature]
+
+    digest = SHA.new()
+    with open(keyfile, 'r') as fp:
+        rsa_key = RSA.importKey(fp.read())
+
+    signer = PKCS1_v1_5.new(rsa_key)
+
+    digest.update(content[16 + l_pubkey + l_signature:])
+    assert signer.verify(digest, signature)
+
+
+def assert_locale_upfix(package):
+    translations = [
+        json.loads(package.read('_locales/{}/messages.json'.format(lang)))
+        for lang in ALL_LANGUAGES
+    ]
+
+    manifest = package.read('manifest.json')
+
+    # Chrome Web Store requires descriptive translations to be present in
+    # every language.
+    for match in re.finditer(r'__MSG_(\S+)__', manifest):
+        name = match.group(1)
+
+        for other in translations[1:]:
+            assert translations[0][name]['message'] == other[name]['message']
+
+
+@pytest.mark.usefixtures(
+    'all_lang_locales',
+    'gecko_import',
+    'locale_modules',
+    'icons',
+    'lib_files',
+    'chrome_metadata',
+)
+@pytest.mark.parametrize('dev_build_release', ['build', 'devenv', 'release'])
+def test_build_chrome(dev_build_release, keyfile, tmpdir, srcdir, capsys):
+    from buildtools import packagerChrome
+    release = dev_build_release == 'release'
+    devenv = dev_build_release == 'devenv'
+
+    run_webext_build('chrome', dev_build_release, srcdir, packagerChrome,
+                     keyfile if release else None)
+
+    # The makeIcons() in packagerChrome.py should warn about non-square
+    # icons via stderr.
+    out, err = capsys.readouterr()
+    assert 'icon should be square' in err
+
+    if devenv:
+        content_class = DirContent
+        out_file_path = os.path.join(str(srcdir), 'devenv.chrome')
+    else:
+        content_class = ZipContent
+        out_file = 'adblockpluschrome-1.2.3'
+        if not release:
+            out_file += '.0'
+
+        if release:
+            out_file += '.crx'
+        else:
+            out_file += '.zip'
+
+        out_file_path = os.path.abspath(os.path.join(
+            os.path.dirname(__file__), os.pardir, out_file))
+
+        assert os.path.exists(out_file_path)
+
+    if release:
+        assert_chrome_signature(out_file_path, keyfile)
+
+    with content_class(out_file_path) as package:
+        assert_base_files(package)
+        assert_devenv_scripts(package, devenv)
+        assert_all_locales_present(package, '_locales')
+        assert_locale_upfix(package)
+        assert_gecko_locale_conversion(package)
+        assert_convert_js(package)
+        expected = os.path.join(
+            os.path.dirname(__file__),
+            'expecteddata',
+            'manifest_chrome_{}.json'.format(dev_build_release),
+        )
+        assert_manifest_content(package.read('manifest.json'), expected)
+
+
+@pytest.mark.usefixtures(
+    'all_lang_locales',
+    'locale_modules',
+    'gecko_import',
+    'icons',
+    'lib_files',
+    'gecko_webext_metadata',
+)
+@pytest.mark.parametrize('dev_build_release', ['build', 'devenv', 'release'])
+def test_build_gecko_webext(dev_build_release, tmpdir, srcdir, capsys):

[Vasily Kuznetsov, 2017/09/11 12:50:07]

This test seems very similar to the previous one apart from some file naming
differences. What do you think about merging them and using parametrization to
switch between the two versions of the file names (and other minor difference)?
Seems like this wouldn't make things much more complicated and we would
eliminate a lot of code duplication.

[tlucas, 2017/09/12 11:32:11]

I agree - will do (also i fear the complexity of the resulting test function
might be a bit over the top)

[tlucas, 2017/09/13 13:43:25]

Done.

+    from buildtools import packagerChrome
+    release = dev_build_release == 'release'
+    devenv = dev_build_release == 'devenv'
+
+    run_webext_build('gecko-webext', dev_build_release, srcdir, packagerChrome)
+
+    # The makeIcons() in packagerChrome.py should warn about non-square
+    # icons via stderr.
+    out, err = capsys.readouterr()
+    assert 'icon should be square' in err
+
+    if devenv:
+        content_class = DirContent
+        out_file_path = os.path.join(str(srcdir), 'devenv.gecko-webext')
+    else:
+        content_class = ZipContent
+        out_file = 'adblockplusfirefox-1.2.3{}.xpi'.format(
+            '.0' if not release else ''
+        )
+
+        out_file_path = os.path.abspath(os.path.join(
+            os.path.dirname(__file__), os.pardir, out_file))
+
+        assert os.path.exists(out_file_path)
+
+    with content_class(out_file_path) as package:
+        assert_base_files(package)
+        assert_devenv_scripts(package, devenv)
+        assert_all_locales_present(package, '_locales')
+        assert_gecko_locale_conversion(package)
+        assert_convert_js(package, True)
+
+        expected = os.path.join(
+            os.path.dirname(__file__),
+            'expecteddata',
+            'manifest_gecko-webext_{}.json'.format(dev_build_release),
+        )
+        assert_manifest_content(package.read('manifest.json'), expected)