Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code

Side by Side Diff: sitescripts/extensions/bin/updateDownloadLinks.py

Issue 5723465818570752: Issue 520 - Generate PAD files for download portals when updating download links (Closed)
Patch Set: Added missing license info Created May 27, 2014, 12:04 p.m.
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « sitescripts/extensions/android.py ('k') | sitescripts/extensions/pad/__init__.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # coding: utf-8 1 # coding: utf-8
2 2
3 # This file is part of the Adblock Plus web scripts, 3 # This file is part of the Adblock Plus web scripts,
4 # Copyright (C) 2006-2014 Eyeo GmbH 4 # Copyright (C) 2006-2014 Eyeo GmbH
5 # 5 #
6 # Adblock Plus is free software: you can redistribute it and/or modify 6 # Adblock Plus is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License version 3 as 7 # it under the terms of the GNU General Public License version 3 as
8 # published by the Free Software Foundation. 8 # published by the Free Software Foundation.
9 # 9 #
10 # Adblock Plus is distributed in the hope that it will be useful, 10 # Adblock Plus is distributed in the hope that it will be useful,
(...skipping 11 matching lines...) Expand all
22 This script generates a list of extensions and saves these with download links 22 This script generates a list of extensions and saves these with download links
23 and version information 23 and version information
24 """ 24 """
25 25
26 import sys, os, re, urllib, urllib2, urlparse, subprocess, time 26 import sys, os, re, urllib, urllib2, urlparse, subprocess, time
27 import xml.dom.minidom as dom 27 import xml.dom.minidom as dom
28 from ConfigParser import SafeConfigParser 28 from ConfigParser import SafeConfigParser
29 from StringIO import StringIO 29 from StringIO import StringIO
30 from sitescripts.utils import get_config, get_template 30 from sitescripts.utils import get_config, get_template
31 from sitescripts.extensions.utils import compareVersions, Configuration, getSafa riCertificateID 31 from sitescripts.extensions.utils import compareVersions, Configuration, getSafa riCertificateID
32 from sitescripts.extensions.android import get_min_sdk_version
33 from sitescripts.extensions.pad import PadFile
32 from buildtools.packagerGecko import KNOWN_APPS 34 from buildtools.packagerGecko import KNOWN_APPS
33 35
34 def urlencode(value): 36 def urlencode(value):
35 return urllib.quote(value.encode('utf-8'), '') 37 return urllib.quote(value.encode('utf-8'), '')
36 38
37 def urlopen(url, attempts=3): 39 def urlopen(url, attempts=3):
38 """ 40 """
39 Tries to open a particular URL, retries on failure. 41 Tries to open a particular URL, retries on failure.
40 """ 42 """
41 for i in range(attempts): 43 for i in range(attempts):
(...skipping 17 matching lines...) Expand all
59 versionTag = versionTags[0] if len(versionTags) > 0 else None 61 versionTag = versionTags[0] if len(versionTags) > 0 else None
60 if linkTag and versionTag and linkTag.firstChild and versionTag.firstChild: 62 if linkTag and versionTag and linkTag.firstChild and versionTag.firstChild:
61 return (linkTag.firstChild.data, versionTag.firstChild.data) 63 return (linkTag.firstChild.data, versionTag.firstChild.data)
62 else: 64 else:
63 return (None, None) 65 return (None, None)
64 66
65 def getGoogleDownloadLink(galleryID): 67 def getGoogleDownloadLink(galleryID):
66 """ 68 """
67 gets download link for a Chrome add-on from the Chrome Gallery site 69 gets download link for a Chrome add-on from the Chrome Gallery site
68 """ 70 """
69 param = 'id=%s&uc' % urlencode(galleryID) 71 galleryID = urlencode(galleryID)
70 url = 'https://clients2.google.com/service/update2/crx?x=%s' % urlencode(param ) 72
71 contents = urlopen(url).read() 73 url = 'https://clients2.google.com/service/update2/crx?x=%s' % urlencode('id=% s&uc' % galleryID)
72 document = dom.parseString(contents) 74 document = dom.parse(urlopen(url))
73 updateTags = document.getElementsByTagName('updatecheck') 75 updateTags = document.getElementsByTagName('updatecheck')
74 updateTag = updateTags[0] if len(updateTags) > 0 else None 76 version = updateTags and updateTags[0].getAttribute('version')
75 if updateTag and updateTag.hasAttribute('codebase') and updateTag.hasAttribute ('version'): 77
76 return (updateTag.getAttribute('codebase'), updateTag.getAttribute('version' )) 78 if not version:
77 else:
78 return (None, None) 79 return (None, None)
79 80
81 request = urllib2.Request('https://chrome.google.com/webstore/detail/_/' + gal leryID)
82 request.get_method = lambda : 'HEAD'
83 url = urllib2.urlopen(request).geturl()
84
85 return (url, version)
86
80 def getOperaDownloadLink(galleryID): 87 def getOperaDownloadLink(galleryID):
81 """ 88 """
82 gets download link for an Opera add-on from the Opera Addons site 89 gets download link for an Opera add-on from the Opera Addons site
83 """ 90 """
84 class HeadRequest(urllib2.Request): 91 galleryID = urlencode(galleryID)
85 def get_method(self):
86 return "HEAD"
87 92
88 url = 'https://addons.opera.com/extensions/download/%s/' % urlencode(galleryID ) 93 request = urllib2.Request('https://addons.opera.com/extensions/download/%s/' % galleryID)
89 response = urllib2.urlopen(HeadRequest(url)) 94 request.get_method = lambda : 'HEAD'
90 content_disposition = response.info().dict.get('content-disposition', None) 95 response = urllib2.urlopen(request)
91 if content_disposition != None: 96
92 match = re.search(r'filename=\S+-([\d.]+)-\d+\.oex$', content_disposition) 97 content_disposition = response.info().getheader('Content-Disposition')
93 else: 98 if content_disposition:
94 match = None; 99 match = re.search(r'filename=\S+-([\d.]+)-\d+\.crx$', content_disposition)
95 if match: 100 if match:
96 return (url, match.group(1)) 101 return ('https://addons.opera.com/extensions/details/%s/' % galleryID , ma tch.group(1))
97 else: 102
98 return (None, None) 103 return (None, None)
99 104
100 def getLocalLink(repo): 105 def getLocalLink(repo):
101 """ 106 """
102 gets the link for the newest download of an add-on in the local downloads 107 gets the link for the newest download of an add-on in the local downloads
103 repository 108 repository
104 """ 109 """
105 url = repo.downloadsURL
106
107 highestURL = None 110 highestURL = None
108 highestVersion = None 111 highestVersion = None
109 112
110 if repo.type in ('gecko', 'chrome', 'opera', 'safari'): 113 for filename, version in repo.getDownloads():
111 prefix = readRawMetadata(repo).get('general', 'basename') 114 if not highestVersion or compareVersions(version, highestVersion) > 0:
112 else: 115 highestURL = urlparse.urljoin(repo.downloadsURL, filename)
113 prefix = os.path.basename(repo.repository) 116 highestVersion = version
114 prefix += '-'
115 suffix = repo.packageSuffix
116 117
117 # go through the downloads repository looking for downloads matching this exte nsion
118 command = ['hg', 'locate', '-R', repo.downloadsRepo, '-r', 'default']
119 result = subprocess.check_output(command)
120 for fileName in result.splitlines():
121 if fileName.startswith(prefix) and fileName.endswith(suffix):
122 version = fileName[len(prefix):len(fileName) - len(suffix)]
123 if highestVersion == None or compareVersions(version, highestVersion) > 0:
124 highestURL = urlparse.urljoin(url, fileName)
125 highestVersion = version
126 return (highestURL, highestVersion) 118 return (highestURL, highestVersion)
127 119
128 def getDownloadLink(repo): 120 def getDownloadLink(repo):
129 """ 121 """
130 gets the download link to the most current version of an extension 122 gets the download link to the most current version of an extension
131 """ 123 """
132 galleryURL = None 124 # you can't easily install extensions from third-party sources on Chrome
133 galleryVersion = None 125 # and Opera. So always get the link for the version on the Web Store.
134 if repo.type == "gecko" and repo.galleryID: 126 if repo.galleryID:
127 if repo.type == "chrome":
128 return getGoogleDownloadLink(repo.galleryID)
129 if repo.type == "opera":
130 return getOperaDownloadLink(repo.galleryID)
131
132 (localURL, localVersion) = getLocalLink(repo)
133
134 # get a link to Firefox Add-Ons, if the latest version has been published ther e
135 if repo.type == 'gecko' and repo.galleryID:
135 (galleryURL, galleryVersion) = getMozillaDownloadLink(repo.galleryID) 136 (galleryURL, galleryVersion) = getMozillaDownloadLink(repo.galleryID)
136 elif repo.type == "chrome" and repo.galleryID: 137 if not localVersion or (galleryVersion and
137 (galleryURL, galleryVersion) = getGoogleDownloadLink(repo.galleryID) 138 compareVersions(galleryVersion, localVersion) >= 0):
138 elif repo.type == "opera" and repo.galleryID: 139 return (galleryURL, galleryVersion)
139 (galleryURL, galleryVersion) = getOperaDownloadLink(repo.galleryID)
140 140
141 (downloadsURL, downloadsVersion) = getLocalLink(repo) 141 return (localURL, localVersion)
142 if galleryVersion == None or (downloadsVersion != None and
143 compareVersions(galleryVersion, downloadsVersion ) < 0):
144 return (downloadsURL, downloadsVersion)
145 else:
146 return (galleryURL, galleryVersion)
147 142
148 def getQRCode(text): 143 def getQRCode(text):
149 try: 144 try:
150 import qrcode 145 import qrcode
151 import base64 146 import base64
152 import Image # required by qrcode but not formally a dependency 147 import Image # required by qrcode but not formally a dependency
153 except: 148 except:
154 return None 149 return None
155 150
156 data = StringIO() 151 data = StringIO()
(...skipping 11 matching lines...) Expand all
168 continue 163 continue
169 if not result.has_section(repo.repositoryName): 164 if not result.has_section(repo.repositoryName):
170 result.add_section(repo.repositoryName) 165 result.add_section(repo.repositoryName)
171 result.set(repo.repositoryName, "downloadURL", downloadURL) 166 result.set(repo.repositoryName, "downloadURL", downloadURL)
172 result.set(repo.repositoryName, "version", version) 167 result.set(repo.repositoryName, "version", version)
173 168
174 qrcode = getQRCode(downloadURL) 169 qrcode = getQRCode(downloadURL)
175 if qrcode != None: 170 if qrcode != None:
176 result.set(repo.repositoryName, "qrcode", qrcode) 171 result.set(repo.repositoryName, "qrcode", qrcode)
177 172
178 def readRawMetadata(repo, version='tip'):
179 files = subprocess.check_output(['hg', '-R', repo.repository, 'locate', '-r', version]).splitlines()
180 genericFilename = 'metadata'
181 filename = '%s.%s' % (genericFilename, repo.type)
182
183 # Fall back to platform-independent metadata file
184 if filename not in files:
185 filename = genericFilename
186
187 command = ['hg', '-R', repo.repository, 'cat', '-r', version, os.path.join(rep o.repository, filename)]
188 result = subprocess.check_output(command)
189
190 parser = SafeConfigParser()
191 parser.readfp(StringIO(result))
192
193 return parser
194
195 def readMetadata(repo, version): 173 def readMetadata(repo, version):
196 """ 174 """
197 reads extension ID and compatibility information from metadata file in the 175 reads extension ID and compatibility information from metadata file in the
198 extension's repository 176 extension's repository
199 """ 177 """
200 if repo.type == 'android': 178 if repo.type == 'android':
201 command = ['hg', '-R', repo.repository, 'id', '-r', version, '-n'] 179 command = ['hg', '-R', repo.repository, 'id', '-r', version, '-n']
202 result = subprocess.check_output(command) 180 result = subprocess.check_output(command)
203 revision = re.sub(r'\D', '', result) 181 revision = re.sub(r'\D', '', result)
204 182
205 command = ['hg', '-R', repo.repository, 'cat', '-r', version, os.path.join(r epo.repository, 'AndroidManifest.xml')]
206 result = subprocess.check_output(command)
207 manifest = dom.parseString(result)
208 usesSdk = manifest.getElementsByTagName('uses-sdk')[0]
209
210 return { 183 return {
211 'revision': revision, 184 'revision': revision,
212 'minSdkVersion': usesSdk.attributes["android:minSdkVersion"].value, 185 'minSdkVersion': get_min_sdk_version(repo, version),
213 } 186 }
214 elif repo.type == 'safari': 187 elif repo.type == 'safari':
215 metadata = readRawMetadata(repo, version) 188 metadata = repo.readMetadata(version)
216 return { 189 return {
217 'certificateID': getSafariCertificateID(repo.keyFile), 190 'certificateID': getSafariCertificateID(repo.keyFile),
218 'version': version, 191 'version': version,
219 'shortVersion': version, 192 'shortVersion': version,
220 'basename': metadata.get('general', 'basename'), 193 'basename': metadata.get('general', 'basename'),
221 } 194 }
222 elif repo.type == 'gecko': 195 elif repo.type == 'gecko':
223 metadata = readRawMetadata(repo, version) 196 metadata = repo.readMetadata(version)
224 result = { 197 result = {
225 'extensionID': metadata.get('general', 'id'), 198 'extensionID': metadata.get('general', 'id'),
226 'version': version, 199 'version': version,
227 'compat': [] 200 'compat': []
228 } 201 }
229 for key, value in KNOWN_APPS.iteritems(): 202 for key, value in KNOWN_APPS.iteritems():
230 if metadata.has_option('compat', key): 203 if metadata.has_option('compat', key):
231 minVersion, maxVersion = metadata.get('compat', key).split('/') 204 minVersion, maxVersion = metadata.get('compat', key).split('/')
232 result['compat'].append({'id': value, 'minVersion': minVersion, 'maxVers ion': maxVersion}) 205 result['compat'].append({'id': value, 'minVersion': minVersion, 'maxVers ion': maxVersion})
233 return result 206 return result
(...skipping 16 matching lines...) Expand all
250 extensions[repo.type].append(data) 223 extensions[repo.type].append(data)
251 224
252 if len(extensions['android']) > 1: 225 if len(extensions['android']) > 1:
253 print >>sys.stderr, 'Warning: more than one Android app defined, update mani fest only works for one' 226 print >>sys.stderr, 'Warning: more than one Android app defined, update mani fest only works for one'
254 227
255 for repoType in extensions.iterkeys(): 228 for repoType in extensions.iterkeys():
256 manifestPath = get_config().get('extensions', '%sUpdateManifestPath' % repoT ype) 229 manifestPath = get_config().get('extensions', '%sUpdateManifestPath' % repoT ype)
257 template = get_template(get_config().get('extensions', '%sUpdateManifest' % repoType)) 230 template = get_template(get_config().get('extensions', '%sUpdateManifest' % repoType))
258 template.stream({'extensions': extensions[repoType]}).dump(manifestPath) 231 template.stream({'extensions': extensions[repoType]}).dump(manifestPath)
259 232
233 def writePadFile(links):
234 for repo in Configuration.getRepositoryConfigurations():
235 if repo.pad and links.has_section(repo.repositoryName):
236 PadFile.forRepository(repo, links.get(repo.repositoryName, 'version'),
237 links.get(repo.repositoryName, 'downloadURL')) .write()
238
260 def updateLinks(): 239 def updateLinks():
261 """ 240 """
262 writes the current extension download links to a file 241 writes the current extension download links to a file
263 """ 242 """
264 243
265 # Now get download links and save them to file 244 # Now get download links and save them to file
266 result = SafeConfigParser() 245 result = SafeConfigParser()
267 getDownloadLinks(result) 246 getDownloadLinks(result)
268 file = open(get_config().get('extensions', 'downloadLinksFile'), 'wb') 247 file = open(get_config().get('extensions', 'downloadLinksFile'), 'wb')
269 result.write(file) 248 result.write(file)
270 file.close() 249 file.close()
271 250
272 writeUpdateManifest(result) 251 writeUpdateManifest(result)
252 writePadFile(result)
273 253
274 if __name__ == "__main__": 254 if __name__ == "__main__":
275 updateLinks() 255 updateLinks()
OLDNEW
« no previous file with comments | « sitescripts/extensions/android.py ('k') | sitescripts/extensions/pad/__init__.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld