bug 409956, #filter emptyLines is brittle on newlines, r=ted.mielczarek, a=mossop
authorAxel Hecht <l10n@mozilla.com>
Tue, 03 Jun 2008 12:18:02 +0200
changeset 15217 4157b3484b8ecb5908c479b3523359a1c846f63b
parent 15216 ec130a95b7bcada2f262801de666c6ebb43f2099
child 15218 79967bfbed1b7f97e1804afacc6902146bb7aaca
push idunknown
push userunknown
push dateunknown
reviewersted, mossop
bugs409956
milestone1.9.1a1pre
bug 409956, #filter emptyLines is brittle on newlines, r=ted.mielczarek, a=mossop
config/Preprocessor.py
config/tests/unit-LineEndings.py
config/tests/unit-Preprocessor.py
--- a/config/Preprocessor.py
+++ b/config/Preprocessor.py
@@ -142,17 +142,19 @@ class Preprocessor:
       ln = self.context['LINE']
       if self.writtenLines != ln:
         self.out.write('//@line %(line)d "%(file)s"%(le)s'%{'line': ln,
                                                             'file': self.context['FILE'],
                                                             'le': self.LE})
         self.writtenLines = ln
     for f in self.filters:
       aLine = f[1](aLine)
-    aLine = aLine.rstrip('\r\n') + self.LE
+    # ensure our line ending. Only need to handle \n, as we're reading
+    # with universal line ending support, at least for files.
+    aLine = re.sub('\n', self.LE, aLine)
     self.out.write(aLine)
   
   def handleCommandLine(self, args, defaultToStdin = False):
     """
     Parse a commandline into this parser.
     Uses OptionParser internally, no args mean sys.argv[1:].
     """
     includes = []
@@ -342,17 +344,17 @@ class Preprocessor:
       if v in self.context:
         return str(self.context[v])
       return ''
     for i in range(1, len(lst), 2):
       lst[i] = vsubst(lst[i])
     lst.append('\n') # add back the newline
     self.write(reduce(lambda x, y: x+y, lst, ''))
   def do_literal(self, args):
-    self.write(args)
+    self.write(args + self.LE)
   def do_filter(self, args):
     filters = [f for f in args.split(' ') if hasattr(self, 'filter_' + f)]
     if len(filters) == 0:
       return
     current = dict(self.filters)
     for f in filters:
       current[f] = getattr(self, 'filter_' + f)
     filterNames = current.keys()
@@ -412,17 +414,17 @@ class Preprocessor:
     oldWrittenLines = self.writtenLines
     oldCheckLineNumbers = self.checkLineNumbers
     self.checkLineNumbers = False
     if isName:
       try:
         args = str(args)
         if not os.path.isabs(args):
           args = os.path.join(self.context['DIRECTORY'], args)
-        args = open(args)
+        args = open(args, 'rU')
       except:
         raise Preprocessor.Error(self, 'FILE_NOT_FOUND', str(args))
     self.checkLineNumbers = bool(re.search('\.js(?:\.in)?$', args.name))
     oldFile = self.context['FILE']
     oldLine = self.context['LINE']
     oldDir = self.context['DIRECTORY']
     if args.isatty():
       # we're stdin, use '-' and '' for file and dir
new file mode 100644
--- /dev/null
+++ b/config/tests/unit-LineEndings.py
@@ -0,0 +1,46 @@
+import unittest
+
+from StringIO import StringIO
+import os
+import sys
+import os.path
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+
+from Preprocessor import Preprocessor
+
+class TestLineEndings(unittest.TestCase):
+  """
+  Unit tests for the Context class
+  """
+
+  def setUp(self):
+    self.pp = Preprocessor()
+    self.pp.out = StringIO()
+    self.tempnam = os.tempnam('.')
+
+  def tearDown(self):
+    os.remove(self.tempnam)
+
+  def createFile(self, lineendings):
+    f = open(self.tempnam, 'wb')
+    for line, ending in zip(['a', '#literal b', 'c'], lineendings):
+      f.write(line+ending)
+    f.close()
+
+  def testMac(self):
+    self.createFile(['\x0D']*3)
+    self.pp.do_include(self.tempnam)
+    self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
+
+  def testUnix(self):
+    self.createFile(['\x0A']*3)
+    self.pp.do_include(self.tempnam)
+    self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
+
+  def testWindows(self):
+    self.createFile(['\x0D\x0A']*3)
+    self.pp.do_include(self.tempnam)
+    self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
+
+if __name__ == '__main__':
+  unittest.main()
--- a/config/tests/unit-Preprocessor.py
+++ b/config/tests/unit-Preprocessor.py
@@ -172,16 +172,40 @@ BAR
   def test_filter_attemptSubstitution(self):
     f = NamedIO('filter_attemptSubstitution.in', '''#filter attemptSubstitution
 P@VAR@ASS
 #unfilter attemptSubstitution
 ''')
     self.pp.do_include(f)
     self.assertEqual(self.pp.out.getvalue(), "PASS\n")
   
+  def test_filter_emptyLines(self):
+    f = NamedIO('filter_emptyLines.in', '''lines with a
+
+blank line
+#filter emptyLines
+lines with
+
+no blank lines
+#unfilter emptyLines
+yet more lines with
+
+blank lines
+''')
+    self.pp.do_include(f)
+    self.assertEqual(self.pp.out.getvalue(), '''lines with a
+
+blank line
+lines with
+no blank lines
+yet more lines with
+
+blank lines
+''')
+  
   def test_filter_slashslash(self):
     f = NamedIO('filter_slashslash.in', '''#filter slashslash
 PASS//FAIL  // FAIL
 #unfilter slashslash
 PASS // PASS
 ''')
     self.pp.do_include(f)
     self.assertEqual(self.pp.out.getvalue(), "PASS\nPASS // PASS\n")
@@ -367,10 +391,18 @@ FAIL
 PASS
 #else
 FAIL
 #endif
 ''')
     self.pp.do_include(f)
     self.assertEqual(self.pp.out.getvalue(), "PASS\n")
 
+  def test_lineEndings(self):
+    f = NamedIO('lineEndings.in', '''first
+#literal second
+''')
+    self.pp.setLineEndings('cr')
+    self.pp.do_include(f)
+    self.assertEqual(self.pp.out.getvalue(), "first\rsecond\r")
+
 if __name__ == '__main__':
   unittest.main()