Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: releaseAutomation.py

Issue 29611593: Issue 5996 - Release consistent versions across WebExtensions (Closed) Base URL: https://codereview.adblockplus.org/29609559/
Patch Set: Rebase against https://hg.adblockplus.org/buildtools/rev/f92050874f05 Created Nov. 20, 2017, 2:43 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « packagerChrome.py ('k') | tox.ini » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # This Source Code Form is subject to the terms of the Mozilla Public 1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this 2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/. 3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
4 4
5 from __future__ import print_function 5 from __future__ import print_function
6 6
7 import os 7 import os
8 import re 8 import re
9 import codecs 9 import codecs
10 import logging
10 import subprocess 11 import subprocess
12 import sys
11 import tarfile 13 import tarfile
12 import json 14 import json
13 15
14 from packager import readMetadata, getDefaultFileName 16 from packager import readMetadata, getDefaultFileName, get_extension
17 from localeTools import read_locale_config
18
19 WEBEXT_PLATFORMS = {'chrome', 'gecko'}
20 WEBEXT_SOURCE_ARCHIVE = 'adblockpluswebext-{}-source.tgz'
15 21
16 22
17 def get_dependencies(prefix, repos): 23 def get_dependencies(prefix, repos):
18 from ensure_dependencies import read_deps, safe_join 24 from ensure_dependencies import read_deps, safe_join
19 repo = repos[prefix] 25 repo = repos[prefix]
20 deps = read_deps(repo) 26 deps = read_deps(repo)
21 if deps: 27 if deps:
22 for subpath in deps: 28 for subpath in deps:
23 if subpath.startswith('_'): 29 if subpath.startswith('_'):
24 continue 30 continue
(...skipping 81 matching lines...) Expand 10 before | Expand all | Expand 10 after
106 """Run repository-checks in order to bail out early if necessary""" 112 """Run repository-checks in order to bail out early if necessary"""
107 if repo_has_uncommitted(): 113 if repo_has_uncommitted():
108 return False 114 return False
109 if repo_has_incoming(*repo_paths): 115 if repo_has_incoming(*repo_paths):
110 return False 116 return False
111 if repo_has_outgoing(): 117 if repo_has_outgoing():
112 return continue_with_outgoing() 118 return continue_with_outgoing()
113 return True 119 return True
114 120
115 121
116 def run(baseDir, type, version, keyFile, downloadsRepo): 122 def previously_released(version, downloads_repo):
117 if not can_safely_release(baseDir, downloadsRepo): 123 regex = re.compile(r'^' + WEBEXT_SOURCE_ARCHIVE.format('(.*)'),
118 print('Aborting release.') 124 re.MULTILINE)
119 return 1
120 125
121 if type == 'edge': 126 existing = regex.findall(os.linesep.join(os.listdir(downloads_repo)))
122 import buildtools.packagerEdge as packager 127 return version in existing
Wladimir Palant 2017/11/20 15:33:17 I consider that quite a hack, why use regular expr
tlucas 2017/11/23 12:05:52 This is replaced completely, according to our disc
123 elif type == 'chrome':
124 import buildtools.packagerChrome as packager
125 128
126 # Replace version number in metadata file "manually", ConfigParser will mess 129
127 # up the order of lines. 130 def update_metadata(metadata, version):
128 metadata = readMetadata(baseDir, type) 131 # Replace version number in metadata file "manually", ConfigParser will
129 with open(metadata.option_source('general', 'version'), 'r+b') as file: 132 # mess up the order of lines.
130 rawMetadata = file.read() 133 with open(metadata.option_source('general', 'version'), 'r+b') as fp:
134 rawMetadata = fp.read()
131 rawMetadata = re.sub( 135 rawMetadata = re.sub(
132 r'^(\s*version\s*=\s*).*', r'\g<1>%s' % version, 136 r'^(\s*version\s*=\s*).*', r'\g<1>%s' % version,
133 rawMetadata, flags=re.I | re.M 137 rawMetadata, flags=re.I | re.M
134 ) 138 )
135 139
136 file.seek(0) 140 fp.seek(0)
137 file.write(rawMetadata) 141 fp.write(rawMetadata)
138 file.truncate() 142 fp.truncate()
139 143
140 # Read extension name from locale data 144
141 default_locale_path = os.path.join('_locales', packager.defaultLocale, 145 def create_build(platform, base_dir, target_path, version, key_file=None):
146 if platform in WEBEXT_PLATFORMS:
147 import buildtools.packagerChrome as packager
148 else:
149 import buildtools.packagerEdge as packager
150 metadata = readMetadata(base_dir, platform)
151 update_metadata(metadata, version)
152
153 build_path = os.path.join(
154 target_path,
155 getDefaultFileName(metadata, version,
156 get_extension(platform, key_file is not None))
Wladimir Palant 2017/11/20 15:33:16 I suggest calling these as packager.readMetadata()
tlucas 2017/11/23 12:05:51 I agree this would be cleaner, but unfortunately t
157 )
158
159 packager.createBuild(base_dir, type=platform, outFile=build_path,
160 releaseBuild=True, keyFile=key_file)
161
162 return build_path
163
164
165 def release_commit(base_dir, extension_name, version, platforms):
166 subprocess.check_call([
167 'hg', 'commit', '-R', base_dir, '-m',
168 'Releasing {} {} for {}'.format(extension_name, version,
Wladimir Palant 2017/11/20 15:33:16 Since you are touching this, how about prefixing t
tlucas 2017/11/23 12:05:51 Done.
169 ', '.join(platforms))])
Wladimir Palant 2017/11/20 15:33:16 If we want to have it extra nice, maybe map(str.ca
tlucas 2017/11/23 12:05:51 Done.
170
171
172 def release_tag(base_dir, tag_name):
173 subprocess.check_call(['hg', 'tag', '-R', base_dir, '-f', tag_name])
Wladimir Palant 2017/11/20 15:33:17 Since you are touching this, how about passing '-m
tlucas 2017/11/23 12:05:52 Done.
174
175
176 def run(baseDir, platforms, version, keyFile, downloads_repo):
177 if not can_safely_release(baseDir, downloads_repo):
178 print('Aborting release.')
179 return 1
180
181 if (WEBEXT_PLATFORMS.intersection(platforms) and
182 previously_released(version, downloads_repo)):
183 logging.error(('A release of version {} for WebExtension platforms '
184 'was already done. Aborting.').format(version))
Wladimir Palant 2017/11/20 15:33:17 I wonder why we are singling out webext here. Shou
tlucas 2017/11/23 12:05:51 As discussed in IRC, this is now handled different
185 return 2
186
187 downloads = []
188 # Read extension name from first provided platform
189 locale_config = read_locale_config(baseDir, platforms[0],
190 readMetadata(baseDir, platforms[0]))
191 default_locale_path = os.path.join(locale_config['base_path'],
192 locale_config['default_locale'],
142 'messages.json') 193 'messages.json')
143 with open(default_locale_path, 'r') as fp: 194 with open(default_locale_path, 'r') as fp:
144 extensionName = json.load(fp)['name']['message'] 195 extension_name = json.load(fp)['name']['message']
145 196
146 # Now commit the change and tag it 197 webext_builds = WEBEXT_PLATFORMS.intersection(platforms)
147 subprocess.check_call(['hg', 'commit', '-R', baseDir, '-m', 'Releasing %s %s ' % (extensionName, version)])
148 tag_name = version
149 if type == 'edge':
150 tag_name = '{}-{}'.format(tag_name, type)
151 subprocess.check_call(['hg', 'tag', '-R', baseDir, '-f', tag_name])
152 198
153 # Create a release build 199 for platform in webext_builds:
154 downloads = [] 200 used_key_file = None
155 if type == 'chrome': 201 if platform == 'chrome':
156 # Create both signed and unsigned Chrome builds (the latter for Chrome W eb Store). 202 # Currently, only chrome builds are provided by us as signed
157 buildPath = os.path.join(downloadsRepo, getDefaultFileName(metadata, ver sion, 'crx')) 203 # packages. Create an unsigned package in base_dir which should be
158 packager.createBuild(baseDir, type=type, outFile=buildPath, releaseBuild =True, keyFile=keyFile) 204 # uploaded to the Chrome Web Store
159 downloads.append(buildPath) 205 create_build(platform, baseDir, baseDir, version)
206 used_key_file = keyFile
160 207
161 buildPathUnsigned = os.path.join(baseDir, getDefaultFileName(metadata, v ersion, 'zip')) 208 downloads.append(
162 packager.createBuild(baseDir, type=type, outFile=buildPathUnsigned, rele aseBuild=True, keyFile=None) 209 create_build(platform, baseDir, downloads_repo, version,
163 elif type == 'edge': 210 used_key_file)
164 # We only offer the Edge extension for use through the Windows Store 211 )
165 buildPath = os.path.join(downloadsRepo, getDefaultFileName(metadata, ver sion, 'appx'))
166 packager.createBuild(baseDir, type=type, outFile=buildPath, releaseBuild =True)
167 downloads.append(buildPath)
168 212
169 # Create source archive 213 if webext_builds:
170 archivePath = os.path.splitext(buildPath)[0] + '-source.tgz' 214 # Only create one commit, one tag and one source archive for all
171 create_sourcearchive(baseDir, archivePath) 215 # WebExtension platforms
172 downloads.append(archivePath) 216 archive_path = os.path.join(downloads_repo,
217 WEBEXT_SOURCE_ARCHIVE.format(version))
218 create_sourcearchive(baseDir, archive_path)
219 downloads.append(archive_path)
220
221 release_commit(baseDir, extension_name, version, webext_builds)
222 release_tag(baseDir, version)
223
224 for platform in set(platforms).difference(WEBEXT_PLATFORMS):
225 # Create build for platforms not yet using WebExtension
226 downloads.append(
227 create_build(platform, baseDir, downloads_repo, version,
228 keyFile)
229 )
230
231 archivePath = os.path.splitext(downloads[-1])[0] + '-source.tgz'
232 create_sourcearchive(baseDir, archivePath)
233 downloads.append(archivePath)
234
235 release_commit(baseDir, extension_name, version, {platform})
236 release_tag(baseDir, '{}-{}'.format(version, platform))
Wladimir Palant 2017/11/20 15:33:17 What if we release for Firefox, Chrome and Edge at
tlucas 2017/11/23 12:05:51 As discussed in IRC, this is now handled different
173 237
174 # Now add the downloads and commit 238 # Now add the downloads and commit
175 subprocess.check_call(['hg', 'add', '-R', downloadsRepo] + downloads) 239 subprocess.check_call(['hg', 'add', '-R', downloads_repo] + downloads)
176 subprocess.check_call(['hg', 'commit', '-R', downloadsRepo, '-m', 'Releasing %s %s' % (extensionName, version)]) 240 subprocess.check_call([
241 'hg', 'commit', '-R', downloads_repo, '-m',
242 'Releasing {} {} for {}'.format(extension_name, version,
243 ', '.join(platforms))])
177 244
178 # Push all changes 245 # Push all changes
179 subprocess.check_call(['hg', 'push', '-R', baseDir]) 246 subprocess.check_call(['hg', 'push', '-R', baseDir])
180 subprocess.check_call(['hg', 'push', '-R', downloadsRepo]) 247 subprocess.check_call(['hg', 'push', '-R', downloads_repo])
OLDNEW
« no previous file with comments | « packagerChrome.py ('k') | tox.ini » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld