bug 395396, move exceptions from compare-locales to app/locales, r=robcee for compare-locales, r=bsmedberg for exceptions, l10n testing only
authoraxel@pike.org
Thu, 20 Sep 2007 02:34:33 -0700
changeset 6134 50100721f4bbc4375c37ed5a17636431cb3942ac
parent 6133 c85e085b37a731cdb7419ce683234c4338ec25a8
child 6135 e83e38c02f4727d426c9f4fa2f91cd2c9311e1dc
push idunknown
push userunknown
push dateunknown
reviewersrobcee, bsmedberg
bugs395396
milestone1.9a8pre
bug 395396, move exceptions from compare-locales to app/locales, r=robcee for compare-locales, r=bsmedberg for exceptions, l10n testing only
browser/locales/filter.py
testing/tests/l10n/lib/Mozilla/CompareLocales.py
new file mode 100755
--- /dev/null
+++ b/browser/locales/filter.py
@@ -0,0 +1,28 @@
+def test(mod, path, entity = None):
+  import re
+  # ignore anyhting but Firefox
+  if mod not in ("netwerk", "dom", "toolkit", "security/manager",
+                 "browser", "extensions/reporter", "extensions/spellcheck",
+                 "other-licenses/branding/firefox"):
+    return False
+  if mod != "browser" and mod != "extensions/spellcheck":
+    # we only have exceptions for browser and extensions/spellcheck
+    return True
+  if not entity:
+    if mod == "extensions/spellcheck":
+      return False
+    # browser
+    return not (re.match(r"searchplugins\/.+\.xml", path) or
+                re.match(r"chrome\/help\/images\/[A-Za-z-_]+\.png", path))
+  if mod == "extensions/spellcheck":
+    # l10n ships en-US dictionary or something, do compare
+    return True
+  if path == "defines.inc":
+    return entity != "MOZ_LANGPACK_CONTRIBUTORS"
+
+  if path != "chrome/browser-region/region.properties":
+    # only region.properties exceptions remain, compare all others
+    return True
+  
+  return not (re.match(r"browser\.search\.order.[1-9]", entity) or
+              re.match(r"browser\.contentHandlers\.types.[0-5]", entity))
--- a/testing/tests/l10n/lib/Mozilla/CompareLocales.py
+++ b/testing/tests/l10n/lib/Mozilla/CompareLocales.py
@@ -33,52 +33,22 @@
 # the provisions above, a recipient may use your version of this file under
 # the terms of any one of the MPL, the GPL or the LGPL.
 #
 # ***** END LICENSE BLOCK *****
 
 'Mozilla l10n compare locales tool'
 
 import os
+import os.path
 import re
 import logging
 import Parser
 import Paths
 
-def __regify(tpl):
-  return tuple(map(re.compile, tpl))
-
-exceptions = [
-  # ignore langpack contributor section for mail and brownser
-  __regify(('mail|browser', 'defines.inc', 'MOZ_LANGPACK_CONTRIBUTORS')),
-  # ignore search engine order for browser
-  __regify(('browser', 'chrome\\/browser-region\\/region\\.properties',
-   'browser\\.search\\.order\.[1-9]')),
-  # ignore feed engine order for browser
-  __regify(('browser', 'chrome\\/browser-region\\/region\\.properties',
-   'browser\\.contentHandlers\\.types\.[0-5]'))]
-
-def __dont_ignore(tpl):
-  for mod, path, key in exceptions:
-    if mod.match(tpl[0]) and path.match(tpl[1]) and key.match(tpl[2]):
-      return False
-  return True
-
-fl_exceptions = [
-  # ignore search plugins
-  __regify(('browser', 'searchplugins\\/.+\\.xml')),
-  # ignore help images
-  __regify(('browser', 'chrome\\/help\\/images\\/[A-Za-z-_]+\\.png'))]
-
-def do_ignore_fl(tpl):
-  for mod, path in fl_exceptions:
-    if mod.match(tpl[0]) and path.match(tpl[1]):
-      return True
-  return False
-
 class FileCollector:
   class Iter:
     def __init__(self, path):
       self.__base = path
     def __iter__(self):
       self.__w = os.walk(self.__base)
       try:
         self.__nextDir()
@@ -136,80 +106,111 @@ def collectFiles(aComparer, apps = None,
     for locs in locales.values():
       all.update(locs)
     locales['toolkit'] = list(all)
   else:
     locales = Paths.allLocales(apps)
   modules = Paths.Modules(apps)
   en = FileCollector()
   l10n = FileCollector()
+  # load filter functions for each app
+  fltrs = []
+  for app in apps:
+    filterpath = 'mozilla/%s/locales/filter.py' % app
+    if not os.path.exists(filterpath):
+      continue
+    l = {}
+    execfile(filterpath, {}, l)
+    if 'test' not in l or not callable(l['test']):
+      logging.debug('%s does not define function "test"' % filterpath)
+      continue
+    fltrs.append(l['test'])
+  
+  # define fltr function to be used, calling into the app specific ones
+  # if one of our apps wants us to know about a triple, make it so
+  def fltr(mod, lpath, entity = None):
+    for f in fltrs:
+      keep  = True
+      try:
+        keep = f(mod, lpath, entity)
+      except Exception, e:
+        logging.error(str(e))
+      if not keep:
+        return False
+    return True
+  
   for cat in modules.keys():
     logging.debug(" testing " + cat+ " on " + str(modules))
     aComparer.notifyLocales(cat, locales[cat])
     for mod in modules[cat]:
       en_fls = en.getFiles(mod, 'en-US')
       for loc in locales[cat]:
         fls = dict(en_fls) # create copy for modification
         for l_fl, l_path in l10n.iterateFiles(mod, loc):
           if fls.has_key(l_fl):
             # file in both en-US and locale, compare
             aComparer.compareFile(mod, loc, l_fl)
             del fls[l_fl]
           else:
-            # file in locale, but not in en-US, remove?
-            aComparer.removeFile(mod, loc, l_fl)
+            if fltr(mod, l_fl):
+              # file in locale, but not in en-US, remove
+              aComparer.removeFile(mod, loc, l_fl)
+            else:
+              logging.debug(" ignoring %s from %s in %s" %
+                            (l_fl, loc, mod))
         # all locale files dealt with, remaining fls need to be added?
         for lf in fls.keys():
-          aComparer.addFile(mod,loc,lf)
+          if fltr(mod, lf):
+            aComparer.addFile(mod,loc,lf)
+          else:
+            logging.debug(" ignoring %s from %s in %s" %
+                          (lf, loc, mod))
+
+  return fltr
 
 class CompareCollector:
   'collects files to be compared, added, removed'
   def __init__(self):
     self.cl = {}
     self.files = {}
     self.modules = {}
   def notifyLocales(self, aModule, aLocaleList):
     for loc in aLocaleList:
       if self.modules.has_key(loc):
         self.modules[loc].append(aModule)
       else:
         self.modules[loc] = [aModule]
   def addFile(self, aModule, aLocale, aLeaf):
-    if do_ignore_fl((aModule, aLeaf)):
-      logging.debug(" ignoring %s from %s in %s" % (aLeaf, aLocale, aModule))
-      return
     logging.debug(" add %s for %s in %s" % (aLeaf, aLocale, aModule))
     if not self.files.has_key(aLocale):
       self.files[aLocale] = {'missingFiles': [(aModule, aLeaf)],
                              'obsoleteFiles': []}
     else:
       self.files[aLocale]['missingFiles'].append((aModule, aLeaf))
     pass
   def compareFile(self, aModule, aLocale, aLeaf):
     if not self.cl.has_key((aModule, aLeaf)):
       self.cl[(aModule, aLeaf)] = [aLocale]
     else:
       self.cl[(aModule, aLeaf)].append(aLocale)
     pass
   def removeFile(self, aModule, aLocale, aLeaf):
-    if do_ignore_fl((aModule, aLeaf)):
-      logging.debug(" ignoring %s from %s in %s" % (aLeaf, aLocale, aModule))
-      return
     logging.debug(" remove %s from %s in %s" % (aLeaf, aLocale, aModule))
     if not self.files.has_key(aLocale):
       self.files[aLocale] = {'obsoleteFiles': [(aModule, aLeaf)],
                              'missingFiles':[]}
     else:
       self.files[aLocale]['obsoleteFiles'].append((aModule, aLeaf))
     pass
 
 def compare(apps=None, testLocales=None):
   result = {}
   c = CompareCollector()
-  collectFiles(c, apps=apps, locales=testLocales)
+  fltr = collectFiles(c, apps=apps, locales=testLocales)
+  
   key = re.compile('[kK]ey')
   for fl, locales in c.cl.iteritems():
     (mod,path) = fl
     try:
       parser = Parser.getParser(path)
     except UserWarning:
       logging.warning(" Can't compare " + path + " in " + mod)
       continue
@@ -217,17 +218,17 @@ def compare(apps=None, testLocales=None)
     enMap = parser.mapping()
     for loc in locales:
       if not result.has_key(loc):
         result[loc] = {'missing':[],'obsolete':[],
                        'changed':0,'unchanged':0,'keys':0}
       enTmp = dict(enMap)
       parser.read(Paths.get_path(mod, loc, path))
       for k,v in parser:
-        if not __dont_ignore((mod, path, k)):
+        if not fltr(mod, path, k):
           if enTmp.has_key(k):
             del enTmp[k]
           continue
         if not enTmp.has_key(k):
           result[loc]['obsolete'].append((mod,path,k))
           continue
         enVal = enTmp[k]
         del enTmp[k]
@@ -235,17 +236,18 @@ def compare(apps=None, testLocales=None)
           result[loc]['keys'] += 1
         else:
           if enVal == v:
             result[loc]['unchanged'] +=1
             logging.info('%s in %s unchanged' %
                          (k, Paths.get_path(mod, loc, path)))
           else:
             result[loc]['changed'] +=1
-      result[loc]['missing'].extend(filter(__dont_ignore, [(mod,path,k) for k in enTmp.keys()]))
+      result[loc]['missing'].extend(filter(lambda t: fltr(*t),
+                                           [(mod,path,k) for k in enTmp.keys()]))
   for loc,dics in c.files.iteritems():
     if not result.has_key(loc):
       result[loc] = dics
     else:
       for key, list in dics.iteritems():
         result[loc][key] = list
   for loc, mods in c.modules.iteritems():
     result[loc]['tested'] = mods