Issue 1299 - Generate docs outside the devbuild build process

sitescripts/docs/bin/generate_docs.py

 #!/usr/bin/env python
 
 # This file is part of Adblock Plus <https://adblockplus.org/>,
 # Copyright (C) 2006-2016 Eyeo GmbH
 #
 # Adblock Plus is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
 # published by the Free Software Foundation.
 #
 # Adblock Plus is distributed in the hope that it will be useful,
@@ skipping 16 matching lines @@
     key_parts = key.split("_", 1)
     if len(key_parts) < 2:
       continue
     project_name, field_name = key_parts
     if field_name not in {"repository", "target_directory", "command"}:
       continue
     projects.setdefault(project_name, {})[field_name] = value
   return projects
 
 def sync_sources(sources_dir, repository_url):
-  remote_id = subprocess.check_output(["hg", "id", "--id", repository_url])
-  id_path = sources_dir.rstrip(os.path.sep) + ".id"
-  try:
-    with open(id_path, "rb") as id_file:
-      local_id = id_file.read()
-    if local_id == remote_id:
-      return
-  except IOError:
-    pass
-
-  shutil.rmtree(sources_dir, ignore_errors=True)
-  subprocess.check_call(["hg", "archive",
-                         "--repository", repository_url,
-                         "--rev", "master",
-                         sources_dir])
-
-  # In theory, it is possible that some changesets are pushed after we fetch
-  # the ID above, but before we run `hg archive`, which will lead to an
-  # unnecessary `hg archive` operation the next time this runs.
-  with open(id_path, "wb") as id_file:
-    id_file.write(remote_id)
-
-  # This is a workaround for https://issues.adblockplus.org/ticket/3635.
-  os.makedirs(os.path.join(sources_dir, ".hg"))
+  if os.path.exists(sources_dir):
+    subprocess.check_call(["hg", "pull", "--quiet", "--update"
+                           "--repository", sources_dir])

[Wladimir Palant, 2016/02/09 10:44:28]

You don't really want to call |hg pull --update|, it will update to repository
tip and only if it is a linear update. You should rather call |hg update --rev
master| after pulling explicitly. You can also pull with --rev master parameter,
this might avoid pulling unnecessary branches.

[Felix Dahlke, 2016/02/09 12:40:22]

Done.

+  else:
+    subprocess.check_call(["hg", "clone", "--quiet",
+                           "--updaterev", "master",
+                           repository_url, sources_dir])
 
 def replace_dir(source_dir, target_dir):
   if not os.path.exists(target_dir):
     parent_dir = os.path.dirname(target_dir)
     try:
       os.makedirs(parent_dir)
     except OSError:
       pass
     os.rename(source_dir, target_dir)
   else:
@@ skipping 19 matching lines @@
     sources_dir = os.path.join(temp_directory, name)
     sync_sources(sources_dir, data["repository"])
     output_dir = sources_dir.rstrip(os.path.sep) + ".docs"
     run_generation_command(data["command"], sources_dir, output_dir)
     replace_dir(output_dir, data["target_directory"])
 
 if __name__ == "__main__":
   config = get_config()
   projects = read_projects(config)
   generate_docs(projects, config)