Issue 170 - Replacing Mercurial subrepositories

ensure_dependencies.py

 #!/usr/bin/env python
 # coding: utf-8
 
 # This file is part of the Adblock Plus build tools,
 # Copyright (C) 2006-2014 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with Adblock Plus.  If not, see <http://www.gnu.org/licenses/>.
 
 import sys
 import os
+import posixpath
 import re
 import io
 import errno
 import logging
 import subprocess
 import urlparse
 from collections import OrderedDict
+
+USAGE = """
+A dependencies file should look like this:
+
+  # VCS-specific root URLs for the repositories
+  _root = hg:https://hg.adblockplus.org/ git:https://github.com/adblockplus/
+  # File to update this script from (optional)
+  _self = buildtools/ensure_dependencies.py
+  # Check out elemhidehelper repository into extensions/elemhidehelper directory
+  # at tag "1.2".
+  extensions/elemhidehelper = elemhidehelper 1.2
+  # Check out buildtools repository into buildtools directory at VCS-specific
+  # revision IDs.
+  buildtools = buildtools hg:016d16f7137b git:f3f8692f82e5
+"""
 
 class Mercurial():
   def istype(self, repodir):
     return os.path.exists(os.path.join(repodir, ".hg"))
 
   def clone(self, source, target):
     if not source.endswith("/"):
       source += "/"
     subprocess.check_call(["hg", "clone", "--quiet", "--noupdate", source, target])
 
@@ skipping 79 matching lines @@
         key, spec = parse_spec(deps_path, line)
         if spec:
           result[key] = spec
     return result
   except IOError, e:
     if e.errno != errno.ENOENT:
       raise
     return None
 
 def safe_join(path, subpath):
-  return os.path.join(path, *[f for f in subpath.split("/") if f not in (os.curdir, os.pardir)])

[Felix Dahlke, 2014/09/05 08:55:53]

So we're ignoring . and .. in paths? Shouldn't we rather ignore just . and
complain when we find a ..?

[Sebastian Noack, 2014/09/05 09:29:31]

It's common practice to just ignore '..' when sanitizing paths, in most FTP and
HTTP file servers, as well as on UNIX. However, if you prefer to raise an
exception instead, I won't mind.

[Felix Dahlke, 2014/09/05 09:55:01]

We're using this for local paths, don't we? Otherwise I wouldn't really mind.

[Wladimir Palant, 2014/09/05 18:37:39]

This implementation was indeed overly simplistic, e.g. a backslash in the path
would trick it on Windows. I changed it to consider various invalid conditions
and to raise an exception.

+  # This has been inspired by Flask's safe_join() function
+  forbidden = set([os.sep, os.altsep]) - set([posixpath.sep, None])
+  if any(sep in subpath for sep in forbidden):
+    raise Exception("Illegal directory separator in dependency path %s" % subpath)
+
+  normpath = posixpath.normpath(subpath)
+  if posixpath.isabs(normpath):
+    raise Exception("Dependency path %s cannot be absolute" % subpath)
+  if normpath == posixpath.pardir or normpath.startswith(posixpath.pardir + posixpath.sep):
+    raise Exception("Dependency path %s has to be inside the repository" % subpath)
+  return os.path.join(path, *normpath.split(posixpath.sep))
 
 def get_repo_type(repo):
   for name, repotype in repo_types.iteritems():
     if repotype.istype(repo):
       return name
   return None
 
 def ensure_repo(parentrepo, target, roots, sourcename):
   if os.path.exists(target):
     return
@@ skipping 30 matching lines @@
     repo_types[type].pull(target)
     resolved_revision = repo_types[type].get_revision_id(target, revision)
     if not resolved_revision:
       raise Exception("Failed to resolve revision %s" % revision)
 
   current_revision = repo_types[type].get_revision_id(target)
   if resolved_revision != current_revision:
     logging.info("Updating repository %s to revision %s" % (target, resolved_revision))
     repo_types[type].update(target, resolved_revision)
 
-def resolve_deps(repodir, level=0):
+def resolve_deps(repodir, level=0, self_update=True, overrideroots=None, skipdependencies=set()):
   config = read_deps(repodir)
   if config is None:
     if level == 0:
-      logging.warning("No dependencies file in directory %s, nothing to do..." % repodir)
+      logging.warning("No dependencies file in directory %s, nothing to do...\n%s" % (repodir, USAGE))
     return
   if level >= 10:
     logging.warning("Too much subrepository nesting, ignoring %s" % repo)
 
-  for dir in config:

[Felix Dahlke, 2014/09/05 08:55:53]

Why not do something like |for dir, dep in config| here? You're using
|config[dir]| a few times below.

[Wladimir Palant, 2014/09/05 18:37:39]

Done.

-    if dir.startswith("_"):
+  if overrideroots is not None:
+    config["_root"] = overrideroots
+
+  for dir, revisions in config.iteritems():
+    if dir.startswith("_") or revisions["_source"] in skipdependencies:
       continue
     target = safe_join(repodir, dir)
-    ensure_repo(repodir, target, config.get("_root", {}), config[dir]["_source"])
-    update_repo(target, config[dir])
-    resolve_deps(target, level + 1)
-
-  if level == 0 and "_self" in config and "*" in config["_self"]:
+    ensure_repo(repodir, target, config.get("_root", {}), revisions["_source"])
+    update_repo(target, revisions)
+    resolve_deps(target, level + 1, self_update=False, overrideroots=overrideroots, skipdependencies=skipdependencies)
+
+  if self_update and "_self" in config and "*" in config["_self"]:
     source = safe_join(repodir, config["_self"]["*"])
     try:
       with io.open(source, "rb") as handle:
         sourcedata = handle.read()
     except IOError, e:
       if e.errno != errno.ENOENT:
         raise
       logging.warning("File %s doesn't exist, skipping self-update" % source)
       return
 
@@ skipping 11 matching lines @@
       else:
         logging.warning("Cannot restart %s automatically, please rerun" % target)
 
 if __name__ == "__main__":
   logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
   repos = sys.argv[1:]
   if not len(repos):
     repos = [os.getcwd()]
   for repo in repos:
     resolve_deps(repo)