Moved Chrome extension scripts to buildtools repository

localeTools.py

Index: localeTools.py
===================================================================
--- a/localeTools.py
+++ b/localeTools.py
@@ -5,21 +5,85 @@
 # http://mozilla.org/MPL/2.0/.
 
 import re, os, sys, codecs, json, urllib, urllib2
 from StringIO import StringIO
 from ConfigParser import SafeConfigParser
 from zipfile import ZipFile
 from xml.parsers.expat import ParserCreate, XML_PARAM_ENTITY_PARSING_ALWAYS
 
-langMapping = {
+langMappingGecko = {
   'dsb': 'dsb-DE',
   'hsb': 'hsb-DE',
 }
 
+langMappingChrome = {
+  'es-419': 'es-AR',
+  'es': 'es-ES',
+  'sv': 'sv-SE',
+  'ml': 'ml-IN',
+  'nb': 'no',
+}
+
+chromeLocales = [
+  "am",
+  "ar",
+  "bg",
+  "bn",
+  "ca",
+  "cs",
+  "da",
+  "de",
+  "el",
+  "en-GB",
+  "en-US",
+  "es-419",
+  "es",
+  "et",
+  "fa",
+  "fi",
+  "fil",
+  "fr",
+  "gu",
+  "he",
+  "hi",
+  "hr",
+  "hu",
+  "id",
+  "it",
+  "ja",
+  "kn",
+  "ko",
+  "lt",
+  "lv",
+  "ml",
+  "mr",
+  "ms",
+  "nb",
+  "nl",
+  "pl",
+  "pt-BR",
+  "pt-PT",
+  "ro",
+  "ru",
+  "sk",
+  "sl",
+  "sr",
+  "sv",
+  "sw",
+  "ta",
+  "te",
+  "th",
+  "tr",
+  "uk",
+  "vi",
+  "zh-CN",
+  "zh-TW",
+]
+
 class OrderedDict(dict):
   def __init__(self):
     self.__order = []
   def __setitem__(self, key, value):
     self.__order.append(key)
     dict.__setitem__(self, key, value)
   def iteritems(self):
     done = set()
@@ -29,16 +93,20 @@ class OrderedDict(dict):
         done.add(key)
 
 def escapeEntity(value):
   return value.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;').replace('"', '&quot;')
 
 def unescapeEntity(value):
   return value.replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>').replace('&quot;', '"')
 
+def mapLocale(type, locale):
+  mapping = langMappingChrome if type == 'chrome' else langMappingGecko
+  return mapping.get(locale, locale)
+
 def parseDTDString(data, path):
   result = []
   currentComment = [None]
 
   parser = ParserCreate()
   parser.UseForeignDTD(True)
   parser.SetParamEntityParsing(XML_PARAM_ENTITY_PARSING_ALWAYS)
 
@@ -140,104 +208,168 @@ def toJSON(path):
   result = OrderedDict()
   for name, comment, value in it:
     obj = {'message': value}
     if comment == None:
       obj['description'] = name
     else:
       obj['description'] = '%s: %s' % (name, comment)
     result[name] = obj
-  return json.dumps(result, indent=2)
+  return json.dumps(result, ensure_ascii=False, indent=2)
 
 def fromJSON(path, data):
   data = json.loads(data)
   if not data:
     if os.path.exists(path):
       os.remove(path)
     return
 
   dir = os.path.dirname(path)
   if not os.path.exists(dir):
     os.makedirs(dir)
   file = codecs.open(path, 'wb', encoding='utf-8')
   for key, value in data.iteritems():
     file.write(generateStringEntry(key, value['message'], path))
   file.close()
 
-def setupTranslations(locales, projectName, key):
+def preprocessChromeLocale(path, metadata, isMaster):
+  fileHandle = codecs.open(path, 'rb', encoding='utf-8')
+  data = json.load(fileHandle)
+  fileHandle.close()
+
+  # Remove synced keys, these don't need to be translated
+  if metadata.has_section('locale_sync'):
+    for file, stringIDs in metadata.items('locale_sync'):
+      for stringID in re.split(r'\s+', stringIDs):
+        if file == 'remove':
+          key = stringID
+        else:
+          key = re.sub(r'\..*', '', file) + '_' + re.sub(r'\W', '_', stringID)
+        if key in data:
+          del data[key]
+
+  for key, value in data.iteritems():
+    if isMaster:
+      # Make sure the key name is listed in the description
+      if "description" in value:
+        value["description"] = "%s: %s" % (key, value["description"])
+      else:
+        value["description"] = key
+    else:
+      # Delete description from translations
+      if "description" in value:
+        del value["description"]
+
+  return json.dumps(data, ensure_ascii=False, sort_keys=True, indent=2)
+
+def setupTranslations(type, locales, projectName, key):
+  # Copy locales list, we don't want to change the parameter
   locales = set(locales)
-  firefoxLocales = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.html').read()
-  for match in re.finditer(r'&amp;lang=([\w\-]+)"', firefoxLocales):
-    locales.add(langMapping.get(match.group(1), match.group(1)))
-  langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language-tools/').read()
-  for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S):
-    if match.group(0).find('Install Language Pack') >= 0:
-      match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
-      if match2:
-        locales.add(langMapping.get(match2.group(1), match2.group(1)))
+
+  # Fill up with locales that we don't have but the browser supports
+  if type == 'chrome':
+    for locale in chromeLocales:
+      locales.add(locale)
+  else:
+    firefoxLocales = urllib2.urlopen('http://www.mozilla.org/en-US/firefox/all.html').read()
+    for match in re.finditer(r'&amp;lang=([\w\-]+)"', firefoxLocales):
+      locales.add(mapLocale(type, match.group(1)))
+    langPacks = urllib2.urlopen('https://addons.mozilla.org/en-US/firefox/language-tools/').read()
+    for match in re.finditer(r'<tr>.*?</tr>', langPacks, re.S):
+      if match.group(0).find('Install Language Pack') >= 0:
+        match2 = re.search(r'lang="([\w\-]+)"', match.group(0))
+        if match2:
+          locales.add(mapLocale(type, match2.group(1)))
+
+  # Convert locale codes to the ones that Crowdin will understand
+  locales = set(map(lambda locale: mapLocale(type, locale), locales))
 
   allowed = set()
   allowedLocales = urllib2.urlopen('http://crowdin.net/page/language-codes').read()
   for match in re.finditer(r'<tr>\s*<td>([\w\-]+)</td>', allowedLocales, re.S):
     allowed.add(match.group(1))
   if not allowed.issuperset(locales):
     print 'Warning, following locales aren\'t allowed by server: ' + ', '.join(locales - allowed)
 
   locales = list(locales & allowed)
   locales.sort()
   params = urllib.urlencode([('languages[]', locale) for locale in locales])
   result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/edit-project?key=%s&%s' % (projectName, key, params)).read()
   if result.find('<success') < 0:
     raise Exception('Server indicated that the operation was not successful\n' + result)
 
-def updateTranslationMaster(dir, locale, projectName, key):
+def postFiles(files, url):
+  boundary = '----------ThIs_Is_tHe_bouNdaRY_$'
+  body = ''
+  for file, data in files:
+    body += '--%s\r\n' % boundary
+    body += 'Content-Disposition: form-data; name="files[%s]"; filename="%s"\r\n' % (file, file)
+    body += 'Content-Type: application/octet-stream\r\n'
+    body += 'Content-Transfer-Encoding: binary\r\n'
+    body += '\r\n' + data + '\r\n'
+    body += '--%s--\r\n' % boundary
+
+  body = body.encode('utf-8')
+  request = urllib2.Request(url, StringIO(body))
+  request.add_header('Content-Type', 'multipart/form-data; boundary=%s' % boundary)
+  request.add_header('Content-Length', len(body))
+  result = urllib2.urlopen(request).read()
+  if result.find('<success') < 0:
+    raise Exception('Server indicated that the operation was not successful\n' + result)
+
+def updateTranslationMaster(type, metadata, dir, projectName, key):
   result = json.load(urllib2.urlopen('http://api.crowdin.net/api/project/%s/info?key=%s&json=1' % (projectName, key)))
 
   existing = set(map(lambda f: f['name'], result['files']))
   add = []
   update = []
   for file in os.listdir(dir):
     path = os.path.join(dir, file)
     if os.path.isfile(path):
-      data = toJSON(path)
+      if type == 'chrome':
+        data = preprocessChromeLocale(path, metadata, True)
+        newName = file
+      else:
+        data = toJSON(path)
+        newName = file + '.json'
+
       if data:
-        newName = file + '.json'
         if newName in existing:
           update.append((newName, data))
           existing.remove(newName)
         else:
           add.append((newName, data))
 
-  def postFiles(files, url):
-    boundary = '----------ThIs_Is_tHe_bouNdaRY_$'
-    body = ''
-    for file, data in files:
-      body +=  '--%s\r\n' % boundary
-      body += 'Content-Disposition: form-data; name="files[%s]"; filename="%s"\r\n' % (file, file)
-      body += 'Content-Type: application/octet-stream\r\n'
-      body += '\r\n' + data.encode('utf-8') + '\r\n'
-      body += '--%s--\r\n' % boundary
-
-    request = urllib2.Request(url, body)
-    request.add_header('Content-Type', 'multipart/form-data; boundary=%s' % boundary)
-    request.add_header('Content-Length', len(body))
-    result = urllib2.urlopen(request).read()
-    if result.find('<success') < 0:
-      raise Exception('Server indicated that the operation was not successful\n' + result)
-
   if len(add):
     titles = urllib.urlencode([('titles[%s]' % name, re.sub(r'\.json', '', name)) for name, data in add])
     postFiles(add, 'http://api.crowdin.net/api/project/%s/add-file?key=%s&type=chrome&%s' % (projectName, key, titles))
   if len(update):
     postFiles(update, 'http://api.crowdin.net/api/project/%s/update-file?key=%s' % (projectName, key))
   for file in existing:
     result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/delete-file?key=%s&file=%s' % (projectName, key, file)).read()
     if result.find('<success') < 0:
       raise Exception('Server indicated that the operation was not successful\n' + result)
 
+def uploadTranslations(type, metadata, dir, locale, projectName, key):
+  files = []
+  for file in os.listdir(dir):
+    path = os.path.join(dir, file)
+    if os.path.isfile(path):
+      if type == 'chrome':
+        data = preprocessChromeLocale(path, metadata, False)
+        newName = file
+      else:
+        data = toJSON(path)
+        newName = file + '.json'
+
+      if data:
+        files.append((newName, data))
+  if len(files):
+    postFiles(files, 'http://api.crowdin.net/api/project/%s/upload-translation?key=%s&language=%s' % (projectName, key, mapLocale(type, locale)))
+
 def getTranslations(localesDir, defaultLocale, projectName, key):
   result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/export?key=%s' % (projectName, key)).read()
   if result.find('<success') < 0:
     raise Exception('Server indicated that the operation was not successful\n' + result)
 
   result = urllib2.urlopen('http://api.crowdin.net/api/project/%s/download/all.zip?key=%s' % (projectName, key)).read()
   zip = ZipFile(StringIO(result))
   dirs = {}
@@ -245,17 +377,17 @@ def getTranslations(localesDir, defaultL
     if not info.filename.endswith('.dtd.json') and not info.filename.endswith('.properties.json'):
       continue
 
     dir, file = os.path.split(info.filename)
     origFile = re.sub(r'\.json$', '', file)
     if not re.match(r'^[\w\-]+$', dir) or dir == defaultLocale:
       continue
 
-    for key, value in langMapping.iteritems():
+    for key, value in langMappingGecko.iteritems():
       if value == dir:
         dir = key
 
     if not dir in dirs:
       dirs[dir] = set()
     dirs[dir].add(origFile)
 
     data = zip.open(info.filename).read()