OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/env python |
| 2 # coding: utf-8 |
| 3 |
| 4 # This Source Code Form is subject to the terms of the Mozilla Public |
| 5 # License, v. 2.0. If a copy of the MPL was not distributed with this |
| 6 # file, You can obtain one at http://mozilla.org/MPL/2.0/. |
| 7 |
| 8 import sys |
| 9 import os |
| 10 import posixpath |
| 11 import re |
| 12 import io |
| 13 import errno |
| 14 import logging |
| 15 import subprocess |
| 16 import urlparse |
| 17 import argparse |
| 18 |
| 19 from collections import OrderedDict |
| 20 from ConfigParser import RawConfigParser |
| 21 |
| 22 USAGE = """ |
| 23 A dependencies file should look like this: |
| 24 |
| 25 # VCS-specific root URLs for the repositories |
| 26 _root = hg:https://hg.adblockplus.org/ git:https://github.com/adblockplus/ |
| 27 # File to update this script from (optional) |
| 28 _self = buildtools/ensure_dependencies.py |
| 29 # Check out elemhidehelper repository into extensions/elemhidehelper directory |
| 30 # at tag "1.2". |
| 31 extensions/elemhidehelper = elemhidehelper 1.2 |
| 32 # Check out buildtools repository into buildtools directory at VCS-specific |
| 33 # revision IDs. |
| 34 buildtools = buildtools hg:016d16f7137b git:f3f8692f82e5 |
| 35 """ |
| 36 |
| 37 class Mercurial(): |
| 38 def istype(self, repodir): |
| 39 return os.path.exists(os.path.join(repodir, ".hg")) |
| 40 |
| 41 def clone(self, source, target): |
| 42 if not source.endswith("/"): |
| 43 source += "/" |
| 44 subprocess.check_call(["hg", "clone", "--quiet", "--noupdate", source, targe
t]) |
| 45 |
| 46 def get_revision_id(self, repo, rev=None): |
| 47 command = ["hg", "id", "--repository", repo, "--id"] |
| 48 if rev: |
| 49 command.extend(["--rev", rev]) |
| 50 |
| 51 # Ignore stderr output and return code here: if revision lookup failed we |
| 52 # should simply return an empty string. |
| 53 result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess
.PIPE).communicate()[0] |
| 54 return result.strip() |
| 55 |
| 56 def pull(self, repo): |
| 57 subprocess.check_call(["hg", "pull", "--repository", repo, "--quiet"]) |
| 58 |
| 59 def update(self, repo, rev): |
| 60 subprocess.check_call(["hg", "update", "--repository", repo, "--quiet", "--c
heck", "--rev", rev]) |
| 61 |
| 62 def ignore(self, target, repo): |
| 63 |
| 64 if not self.istype(target): |
| 65 |
| 66 config_path = os.path.join(repo, ".hg", "hgrc") |
| 67 ignore_path = os.path.abspath(os.path.join(repo, ".hg", "dependencies")) |
| 68 |
| 69 config = RawConfigParser() |
| 70 config.read(config_path) |
| 71 |
| 72 if not config.has_section("ui"): |
| 73 config.add_section("ui") |
| 74 |
| 75 config.set("ui", "ignore.dependencies", ignore_path) |
| 76 with open(config_path, "w") as stream: |
| 77 config.write(stream) |
| 78 |
| 79 module = os.path.relpath(target, repo) |
| 80 _ensure_line_exists(ignore_path, module) |
| 81 |
| 82 class Git(): |
| 83 def istype(self, repodir): |
| 84 return os.path.exists(os.path.join(repodir, ".git")) |
| 85 |
| 86 def clone(self, source, target): |
| 87 source = source.rstrip("/") |
| 88 if not source.endswith(".git"): |
| 89 source += ".git" |
| 90 subprocess.check_call(["git", "clone", "--quiet", source, target]) |
| 91 |
| 92 def get_revision_id(self, repo, rev="HEAD"): |
| 93 command = ["git", "rev-parse", "--revs-only", rev + '^{commit}'] |
| 94 return subprocess.check_output(command, cwd=repo).strip() |
| 95 |
| 96 def pull(self, repo): |
| 97 subprocess.check_call(["git", "fetch", "--quiet", "--all", "--tags"], cwd=re
po) |
| 98 |
| 99 def update(self, repo, rev): |
| 100 subprocess.check_call(["git", "checkout", "--quiet", rev], cwd=repo) |
| 101 |
| 102 def ignore(self, target, repo): |
| 103 module = os.path.relpath(target, repo) |
| 104 exclude_file = os.path.join(repo, ".git", "info", "exclude") |
| 105 _ensure_line_exists(exclude_file, module) |
| 106 |
| 107 repo_types = OrderedDict(( |
| 108 ("hg", Mercurial()), |
| 109 ("git", Git()), |
| 110 )) |
| 111 |
| 112 def parse_spec(path, line): |
| 113 if "=" not in line: |
| 114 logging.warning("Invalid line in file %s: %s" % (path, line)) |
| 115 return None, None |
| 116 |
| 117 key, value = line.split("=", 1) |
| 118 key = key.strip() |
| 119 items = value.split() |
| 120 if not len(items): |
| 121 logging.warning("No value specified for key %s in file %s" % (key, path)) |
| 122 return key, None |
| 123 |
| 124 result = OrderedDict() |
| 125 if not key.startswith("_"): |
| 126 result["_source"] = items.pop(0) |
| 127 |
| 128 for item in items: |
| 129 if ":" in item: |
| 130 type, value = item.split(":", 1) |
| 131 else: |
| 132 type, value = ("*", item) |
| 133 if type in result: |
| 134 logging.warning("Ignoring duplicate value for type %s (key %s in file %s)"
% (type, key, path)) |
| 135 else: |
| 136 result[type] = value |
| 137 return key, result |
| 138 |
| 139 def read_deps(repodir): |
| 140 result = {} |
| 141 deps_path = os.path.join(repodir, "dependencies") |
| 142 try: |
| 143 with io.open(deps_path, "rt", encoding="utf-8") as handle: |
| 144 for line in handle: |
| 145 # Remove comments and whitespace |
| 146 line = re.sub(r"#.*", "", line).strip() |
| 147 if not line: |
| 148 continue |
| 149 |
| 150 key, spec = parse_spec(deps_path, line) |
| 151 if spec: |
| 152 result[key] = spec |
| 153 return result |
| 154 except IOError, e: |
| 155 if e.errno != errno.ENOENT: |
| 156 raise |
| 157 return None |
| 158 |
| 159 def safe_join(path, subpath): |
| 160 # This has been inspired by Flask's safe_join() function |
| 161 forbidden = set([os.sep, os.altsep]) - set([posixpath.sep, None]) |
| 162 if any(sep in subpath for sep in forbidden): |
| 163 raise Exception("Illegal directory separator in dependency path %s" % subpat
h) |
| 164 |
| 165 normpath = posixpath.normpath(subpath) |
| 166 if posixpath.isabs(normpath): |
| 167 raise Exception("Dependency path %s cannot be absolute" % subpath) |
| 168 if normpath == posixpath.pardir or normpath.startswith(posixpath.pardir + posi
xpath.sep): |
| 169 raise Exception("Dependency path %s has to be inside the repository" % subpa
th) |
| 170 return os.path.join(path, *normpath.split(posixpath.sep)) |
| 171 |
| 172 def get_repo_type(repo): |
| 173 for name, repotype in repo_types.iteritems(): |
| 174 if repotype.istype(repo): |
| 175 return name |
| 176 return None |
| 177 |
| 178 def ensure_repo(parentrepo, target, roots, sourcename): |
| 179 if os.path.exists(target): |
| 180 return |
| 181 |
| 182 parenttype = get_repo_type(parentrepo) |
| 183 type = None |
| 184 for key in roots: |
| 185 if key == parenttype or (key in repo_types and type is None): |
| 186 type = key |
| 187 if type is None: |
| 188 raise Exception("No valid source found to create %s" % target) |
| 189 |
| 190 if os.path.exists(roots[type]): |
| 191 url = os.path.join(roots[type], sourcename) |
| 192 else: |
| 193 url = urlparse.urljoin(roots[type], sourcename) |
| 194 |
| 195 logging.info("Cloning repository %s into %s" % (url, target)) |
| 196 repo_types[type].clone(url, target) |
| 197 |
| 198 for repo in repo_types.itervalues(): |
| 199 if repo.istype(parentrepo): |
| 200 repo.ignore(target, parentrepo) |
| 201 |
| 202 def update_repo(target, revisions): |
| 203 type = get_repo_type(target) |
| 204 if type is None: |
| 205 logging.warning("Type of repository %s unknown, skipping update" % target) |
| 206 return |
| 207 |
| 208 if type in revisions: |
| 209 revision = revisions[type] |
| 210 elif "*" in revisions: |
| 211 revision = revisions["*"] |
| 212 else: |
| 213 logging.warning("No revision specified for repository %s (type %s), skipping
update" % (target, type)) |
| 214 return |
| 215 |
| 216 resolved_revision = repo_types[type].get_revision_id(target, revision) |
| 217 if not resolved_revision: |
| 218 logging.info("Revision %s is unknown, downloading remote changes" % revision
) |
| 219 repo_types[type].pull(target) |
| 220 resolved_revision = repo_types[type].get_revision_id(target, revision) |
| 221 if not resolved_revision: |
| 222 raise Exception("Failed to resolve revision %s" % revision) |
| 223 |
| 224 current_revision = repo_types[type].get_revision_id(target) |
| 225 if resolved_revision != current_revision: |
| 226 logging.info("Updating repository %s to revision %s" % (target, resolved_rev
ision)) |
| 227 repo_types[type].update(target, resolved_revision) |
| 228 |
| 229 def resolve_deps(repodir, level=0, self_update=True, overrideroots=None, skipdep
endencies=set()): |
| 230 config = read_deps(repodir) |
| 231 if config is None: |
| 232 if level == 0: |
| 233 logging.warning("No dependencies file in directory %s, nothing to do...\n%
s" % (repodir, USAGE)) |
| 234 return |
| 235 if level >= 10: |
| 236 logging.warning("Too much subrepository nesting, ignoring %s" % repo) |
| 237 |
| 238 if overrideroots is not None: |
| 239 config["_root"] = overrideroots |
| 240 |
| 241 for dir, revisions in config.iteritems(): |
| 242 if dir.startswith("_") or revisions["_source"] in skipdependencies: |
| 243 continue |
| 244 target = safe_join(repodir, dir) |
| 245 ensure_repo(repodir, target, config.get("_root", {}), revisions["_source"]) |
| 246 update_repo(target, revisions) |
| 247 resolve_deps(target, level + 1, self_update=False, overrideroots=overrideroo
ts, skipdependencies=skipdependencies) |
| 248 |
| 249 if self_update and "_self" in config and "*" in config["_self"]: |
| 250 source = safe_join(repodir, config["_self"]["*"]) |
| 251 try: |
| 252 with io.open(source, "rb") as handle: |
| 253 sourcedata = handle.read() |
| 254 except IOError, e: |
| 255 if e.errno != errno.ENOENT: |
| 256 raise |
| 257 logging.warning("File %s doesn't exist, skipping self-update" % source) |
| 258 return |
| 259 |
| 260 target = __file__ |
| 261 with io.open(target, "rb") as handle: |
| 262 targetdata = handle.read() |
| 263 |
| 264 if sourcedata != targetdata: |
| 265 logging.info("Updating %s from %s, don't forget to commit" % (source, targ
et)) |
| 266 with io.open(target, "wb") as handle: |
| 267 handle.write(sourcedata) |
| 268 if __name__ == "__main__": |
| 269 logging.info("Restarting %s" % target) |
| 270 os.execv(sys.executable, [sys.executable, target] + sys.argv[1:]) |
| 271 else: |
| 272 logging.warning("Cannot restart %s automatically, please rerun" % target
) |
| 273 |
| 274 def _ensure_line_exists(path, pattern): |
| 275 with open(path, 'a+') as f: |
| 276 file_content = [l.strip() for l in f.readlines()] |
| 277 if not pattern in file_content: |
| 278 file_content.append(pattern) |
| 279 f.seek(0, os.SEEK_SET) |
| 280 f.truncate() |
| 281 for l in file_content: |
| 282 print >>f, l |
| 283 |
| 284 if __name__ == "__main__": |
| 285 logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) |
| 286 |
| 287 parser = argparse.ArgumentParser(description="Verify dependencies for a set of
repositories, by default the repository of this script.") |
| 288 parser.add_argument("repos", metavar="repository", type=str, nargs="*", help="
Repository path") |
| 289 parser.add_argument("-q", "--quiet", action="store_true", help="Suppress infor
mational output") |
| 290 args = parser.parse_args() |
| 291 |
| 292 if args.quiet: |
| 293 logging.disable(logging.INFO) |
| 294 |
| 295 repos = args.repos |
| 296 if not len(repos): |
| 297 repos = [os.path.dirname(__file__)] |
| 298 for repo in repos: |
| 299 resolve_deps(repo) |
OLD | NEW |