Issue 170 - Replacing Mercurial subrepositories

ensure_dependencies.py

+#!/usr/bin/env python
+# coding: utf-8
+
+# This file is part of the Adblock Plus build tools,
+# Copyright (C) 2006-2014 Eyeo GmbH
+#
+# Adblock Plus is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Adblock Plus is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import os
+import re
+import io
+import errno
+import logging
+import subprocess
+import urlparse
+from collections import OrderedDict
+
+class Mercurial():
+  def istype(self, repodir):
+    return os.path.exists(os.path.join(repodir, ".hg"))
+
+  def clone(self, source, target):
+    if not source.endswith("/"):
+      source += "/"
+    subprocess.check_call(["hg", "clone", "--quiet", "--noupdate", source, target])
+
+  def get_revision_id(self, repo, rev=None):
+    command = ["hg", "id", "--repository", repo, "--id"]
+    if rev:
+      command.extend(["--rev", rev])
+
+    # Ignore stderr output and return code here: if revision lookup failed we
+    # should simply return an empty string.
+    result = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()[0]
+    return result.strip()
+
+  def pull(self, repo):
+    subprocess.check_call(["hg", "pull", "--repository", repo, "--quiet"])
+
+  def update(self, repo, rev):
+    subprocess.check_call(["hg", "update", "--repository", repo, "--quiet", "--check", "--rev", rev])
+
+class Git():
+  def istype(self, repodir):
+    return os.path.exists(os.path.join(repodir, ".git"))
+
+  def clone(self, source, target):
+    source = source.rstrip("/")
+    if not source.endswith(".git"):
+      source += ".git"
+    subprocess.check_call(["git", "clone", "--quiet", "--no-checkout", source, target])
+
+  def get_revision_id(self, repo, rev="HEAD"):
+    command = ["git", "-C", repo, "rev-parse", "--revs-only", rev]
+    return subprocess.check_output(command).strip()
+
+  def pull(self, repo):
+    subprocess.check_call(["git", "-C", repo, "fetch", "--quiet", "--all", "--tags"])
+
+  def update(self, repo, rev):
+    subprocess.check_call(["git", "-C", repo, "checkout", "--quiet", rev])
+
+repo_types = {
+  "hg": Mercurial(),
+  "git": Git(),
+}
+
+def parse_spec(path, line):
+  if "=" not in line:
+    logging.warning("Invalid line in file %s: %s" % (path, line))
+    return None, None
+
+  key, value = line.split("=", 1)
+  key = key.strip()
+  items = value.split()
+  if not len(items):
+    logging.warning("No value specified for key %s in file %s" % (key, path))
+    return key, None
+
+  result = OrderedDict()
+  if not key.startswith("_"):
+    result["_source"] = items.pop(0)
+
+  for item in items:
+    if ":" in item:
+      type, value = item.split(":", 1)
+    else:
+      type, value = ("*", item)
+    if type in result:
+      logging.warning("Ignoring duplicate value for type %s (key %s in file %s)" % (type, key, path))
+    else:
+      result[type] = value
+  return key, result
+
+def read_deps(repodir):
+  result = {}
+  deps_path = os.path.join(repodir, "dependencies")
+  try:
+    with io.open(deps_path, "rt", encoding="utf-8") as handle:
+      for line in handle:
+        # Remove comments and whitespace
+        line = re.sub(r"#.*", "", line).strip()
+        if not line:
+          continue
+
+        key, spec = parse_spec(deps_path, line)
+        if spec:
+          result[key] = spec
+    return result
+  except IOError, e:
+    if e.errno != errno.ENOENT:
+      raise
+    return None
+
+def safe_join(path, subpath):
+  return os.path.join(path, *[f for f in subpath.split("/") if f not in (os.curdir, os.pardir)])

[Felix Dahlke, 2014/09/05 08:55:53]

So we're ignoring . and .. in paths? Shouldn't we rather ignore just . and
complain when we find a ..?

[Sebastian Noack, 2014/09/05 09:29:31]

It's common practice to just ignore '..' when sanitizing paths, in most FTP and
HTTP file servers, as well as on UNIX. However, if you prefer to raise an
exception instead, I won't mind.

[Felix Dahlke, 2014/09/05 09:55:01]

We're using this for local paths, don't we? Otherwise I wouldn't really mind.

[Wladimir Palant, 2014/09/05 18:37:39]

This implementation was indeed overly simplistic, e.g. a backslash in the path
would trick it on Windows. I changed it to consider various invalid conditions
and to raise an exception.

+
+def get_repo_type(repo):
+  for name, repotype in repo_types.iteritems():
+    if repotype.istype(repo):
+      return name
+  return None
+
+def ensure_repo(parentrepo, target, roots, sourcename):
+  if os.path.exists(target):
+    return
+
+  parenttype = get_repo_type(parentrepo)
+  type = None
+  for key in roots:
+    if key == parenttype or (key in repo_types and type is None):
+      type = key
+  if type is None:
+    raise Exception("No valid source found to create %s" % target)
+
+  url = urlparse.urljoin(roots[type], sourcename)
+  logging.info("Cloning repository %s into %s" % (url, target))
+  repo_types[type].clone(url, target)
+
+def update_repo(target, revisions):
+  type = get_repo_type(target)
+  if type is None:
+    logging.warning("Type of repository %s unknown, skipping update" % target)
+    return
+
+  if type in revisions:
+    revision = revisions[type]
+  elif "*" in revisions:
+    revision = revisions["*"]
+  else:
+    logging.warning("No revision specified for repository %s (type %s), skipping update" % (target, type))
+    return
+
+  resolved_revision = repo_types[type].get_revision_id(target, revision)
+  if not resolved_revision:
+    logging.info("Revision %s is unknown, downloading remote changes" % revision)
+    repo_types[type].pull(target)
+    resolved_revision = repo_types[type].get_revision_id(target, revision)
+    if not resolved_revision:
+      raise Exception("Failed to resolve revision %s" % revision)
+
+  current_revision = repo_types[type].get_revision_id(target)
+  if resolved_revision != current_revision:
+    logging.info("Updating repository %s to revision %s" % (target, resolved_revision))
+    repo_types[type].update(target, resolved_revision)
+
+def resolve_deps(repodir, level=0):
+  config = read_deps(repodir)
+  if config is None:
+    if level == 0:
+      logging.warning("No dependencies file in directory %s, nothing to do..." % repodir)
+    return
+  if level >= 10:
+    logging.warning("Too much subrepository nesting, ignoring %s" % repo)
+
+  for dir in config:

[Felix Dahlke, 2014/09/05 08:55:53]

Why not do something like |for dir, dep in config| here? You're using
|config[dir]| a few times below.

[Wladimir Palant, 2014/09/05 18:37:39]

Done.

+    if dir.startswith("_"):
+      continue
+    target = safe_join(repodir, dir)
+    ensure_repo(repodir, target, config.get("_root", {}), config[dir]["_source"])
+    update_repo(target, config[dir])
+    resolve_deps(target, level + 1)
+
+  if level == 0 and "_self" in config and "*" in config["_self"]:
+    source = safe_join(repodir, config["_self"]["*"])
+    try:
+      with io.open(source, "rb") as handle:
+        sourcedata = handle.read()
+    except IOError, e:
+      if e.errno != errno.ENOENT:
+        raise
+      logging.warning("File %s doesn't exist, skipping self-update" % source)
+      return
+
+    target = __file__
+    with io.open(target, "rb") as handle:
+      targetdata = handle.read()
+
+    if sourcedata != targetdata:
+      logging.info("Updating %s from %s, don't forget to commit" % (source, target))
+      with io.open(target, "wb") as handle:
+        handle.write(sourcedata)
+      if __name__ == "__main__":
+        logging.info("Restarting %s" % target)
+        os.execv(sys.executable, [sys.executable, target] + sys.argv[1:])
+      else:
+        logging.warning("Cannot restart %s automatically, please rerun" % target)
+
+if __name__ == "__main__":
+  logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
+  repos = sys.argv[1:]
+  if not len(repos):
+    repos = [os.getcwd()]
+  for repo in repos:
+    resolve_deps(repo)