Bug 584758: Add --verifyonly switch to acceptance/runtests.py; add verifyonly check in smokes and random 300s runs in deep (r=brbaker)
authorChris Peyer <cpeyer@adobe.com>
Tue, 10 Aug 2010 08:13:19 -0700
changeset 5004 ce701c85ec59fecd9c4dfeba1d24508084a55696
parent 5003 48c6af0f6000e5a645de0b363b3d87031ed95d29
child 5005 bb18dcbaeb684de89c025e1d5d9d3aa867d4eaf6
push id2676
push usercpeyer@adobe.com
push dateTue, 10 Aug 2010 15:13:24 +0000
reviewersbrbaker
bugs584758
Bug 584758: Add --verifyonly switch to acceptance/runtests.py; add verifyonly check in smokes and random 300s runs in deep (r=brbaker)
build/buildbot/master/tamarinredux.py
test/acceptance/runtests.py
test/acceptance/testconfig.txt
test/runsmokes.txt
--- a/build/buildbot/master/tamarinredux.py
+++ b/build/buildbot/master/tamarinredux.py
@@ -1791,16 +1791,17 @@ class tamarinredux:
     windows_deep_factory.addStep(test_misc)
     windows_deep_factory.addStep(test_selftest(name="Debug", shellname="avmshell_d"))
     windows_deep_factory.addStep(test_selftest(name="ReleaseDebugger", shellname="avmshell_s"))
     windows_deep_factory.addStep(test_selftest(name="DebugDebugger", shellname="avmshell_sd"))
     windows_deep_factory.addStep(compile_generic(name="ReleaseDebugger-air", shellname="avmshell_air", args="--enable-shell --enable-override-global-new --enable-use-system-malloc --enable-debugger", upload="true"))
     windows_deep_factory.addStep(test_generic(name="ReleaseDebugger-air", shellname="avmshell_air", vmargs="", config="", scriptargs=""))
     windows_deep_factory.addStep(test_generic(name="Release-Dgreedy", shellname="avmshell", vmargs="-Dgreedy", config="", scriptargs="--timeout=180 --random"))
     windows_deep_factory.addStep(test_generic(name="DebugDebugger_VerifyAll", shellname="avmshell_sd", vmargs="", config="", scriptargs="--verify --timeout=300 --random"))
+    windows_deep_factory.addStep(test_generic(name="DebugDebugger_VerifyOnly", shellname="avmshell_sd", vmargs="", config="", scriptargs="--verifyonly --timeout=300 --random"))
     windows_deep_factory.addStep(util_process_clean)
     windows_deep_factory.addStep(util_clean_buildsdir)
 
     windows_deep_builder = {
                 'name': "windows-deep",
                 'slavename': "windows-deep",
                 'factory': windows_deep_factory,
                 'builddir': './windows-deep',
@@ -1824,18 +1825,20 @@ class tamarinredux:
     mac_deep_factory.addStep(test_selftest(name="DebugDebugger", shellname="avmshell_sd"))
     mac_deep_factory.addStep(test_selftest(name="DebugDebugger", shellname="avmshell_sd_64"))
     mac_deep_factory.addStep(compile_generic(name="ReleaseDebugger-air", shellname="avmshell_air", args="--enable-shell --enable-override-global-new --enable-use-system-malloc --enable-debugger", upload="true"))
     mac_deep_factory.addStep(test_generic(name="ReleaseDebugger-air", shellname="avmshell_air", vmargs="", config="", scriptargs=""))
     mac_deep_factory.addStep(compile_generic(name="ReleaseDebugger64-air", shellname="avmshell_air_64", args="--enable-shell --target=x86_64-darwin --enable-override-global-new --enable-use-system-malloc --enable-debugger", upload="true"))
     mac_deep_factory.addStep(test_generic(name="ReleaseDebugger64-air", shellname="avmshell_air_64", vmargs="", config="", scriptargs=""))
     mac_deep_factory.addStep(test_generic(name="Release-Dgreedy", shellname="avmshell", vmargs="-Dgreedy", config="", scriptargs="--timeout=180 --random"))
     mac_deep_factory.addStep(test_generic(name="DebugDebugger_VerifyAll", shellname="avmshell_sd", vmargs="", config="", scriptargs="--verify --timeout=300 --random"))
+    mac_deep_factory.addStep(test_generic(name="DebugDebugger_VerifyOnly", shellname="avmshell_sd", vmargs="", config="", scriptargs="--verifyonly --timeout=300 --random"))
     mac_deep_factory.addStep(test_generic(name="Release64-Dgreedy", shellname="avmshell_64", vmargs="-Dgreedy", config="", scriptargs="--timeout=180 --random"))
     mac_deep_factory.addStep(test_generic(name="DebugDebugger64_VerifyAll", shellname="avmshell_sd_64", vmargs="", config="", scriptargs="--verify --timeout=300 --random"))
+    mac_deep_factory.addStep(test_generic(name="DebugDebugger64_VerifyOnly", shellname="avmshell_sd_64", vmargs="", config="", scriptargs="--verifyonly --timeout=300 --random"))
     mac_deep_factory.addStep(TestSuiteShellCommand(
                      command=['./run-brightspot.sh', WithProperties('%s','revision')],
                      env={'branch': WithProperties('%s','branch'), 'silent':WithProperties('%s','silent')},
                      description='running brightspot tests...',
                      descriptionDone='finished running brightspot tests.',
                      name="RunBrightspot",
                      workdir="../repo/build/buildbot/slaves/scripts",
                      timeout=3600)
@@ -1935,16 +1938,17 @@ class tamarinredux:
     mac_ppc_deep_factory.addStep(test_generic(name="Debug64", shellname="avmshell_d_64_ppc", vmargs="", config="ppc-mac-tvm-debug-deep", scriptargs=""))
     mac_ppc_deep_factory.addStep(test_generic(name="DebugDebugger64", shellname="avmshell_sd_64_ppc", vmargs="", config="ppc-mac-tvm-debugdebugger-deep", scriptargs=""))
     mac_ppc_deep_factory.addStep(deep_release_esc)
     mac_ppc_deep_factory.addStep(test_misc)
     mac_ppc_deep_factory.addStep(compile_generic(name="ReleaseDebugger-air", shellname="avmshell_air_ppc", args="--enable-shell --enable-override-global-new --enable-use-system-malloc --enable-debugger --mac-sdk=104u", upload="true"))
     mac_ppc_deep_factory.addStep(test_generic(name="ReleasDebugger-air", shellname="avmshell_air_ppc", vmargs="", config="", scriptargs=""))
     mac_ppc_deep_factory.addStep(test_generic(name="Release-Dgreedy", shellname="avmshell_ppc", vmargs="-Dgreedy", config="", scriptargs="--timeout=180 --random"))
     mac_ppc_deep_factory.addStep(test_generic(name="DebugDebugger_VerifyAll", shellname="avmshell_sd_ppc", vmargs="", config="", scriptargs="--verify --timeout=300 --random"))
+    mac_ppc_deep_factory.addStep(test_generic(name="DebugDebugger_VerifyOnly", shellname="avmshell_sd_ppc", vmargs="", config="", scriptargs="--verifyonly --timeout=300 --random"))
     mac_ppc_deep_factory.addStep(util_process_clean)
     mac_ppc_deep_factory.addStep(util_clean_buildsdir)
 
     mac_ppc_deep_builder = {
                 'name': "mac-ppc-deep",
                 'slavename': "mac-ppc-deep",
                 'factory': mac_ppc_deep_factory,
                 'builddir': './mac-ppc-deep',
@@ -1963,16 +1967,17 @@ class tamarinredux:
     solaris_sparc_deep_factory.addStep(test_selftest(name="ReleaseDebugger", shellname="avmshell_s"))
     solaris_sparc_deep_factory.addStep(test_selftest(name="DebugDebugger", shellname="avmshell_sd"))
     solaris_sparc_deep_factory.addStep(test_generic(name="Debug", shellname="avmshell_d", vmargs="", config="sparc-sol-tvm-debug-deep", scriptargs="--showtimes", _timeout=2400))
     solaris_sparc_deep_factory.addStep(test_generic(name="DebugDebugger", shellname="avmshell_sd", vmargs="", config="sparc-sol-tvm-debugdebugger-deep", scriptargs="--showtimes", _timeout=2400))
     solaris_sparc_deep_factory.addStep(deep_release_esc)
     solaris_sparc_deep_factory.addStep(test_misc)
     solaris_sparc_deep_factory.addStep(test_generic(name="Release-Dgreedy", shellname="avmshell", vmargs="-Dgreedy", config="", scriptargs="--timeout=180 --random"))
     solaris_sparc_deep_factory.addStep(test_generic(name="DebugDebugger_VerifyAll", shellname="avmshell_sd", vmargs="", config="", scriptargs="--verify --timeout=300 --random"))
+    solaris_sparc_deep_factory.addStep(test_generic(name="DebugDebugger_VerifyOnly", shellname="avmshell_sd", vmargs="", config="", scriptargs="--verifyonly --timeout=300 --random"))
     solaris_sparc_deep_factory.addStep(util_process_clean)
     solaris_sparc_deep_factory.addStep(util_clean_buildsdir)
 
     solaris_sparc_deep_builder = {
                 'name': "solaris-sparc-deep",
                 'slavename': "solaris-sparc-deep",
                 'factory': solaris_sparc_deep_factory,
                 'builddir': './solaris-sparc-deep',
@@ -2004,16 +2009,17 @@ class tamarinredux:
                 descriptionDone='finished 32-64 differential vmtests.',
                 name="Testsuite_Differential3264",
                 workdir="../repo/build/buildbot/slaves/scripts"
                 )
     )
     windows_64_deep_factory.addStep(test_misc)
     windows_64_deep_factory.addStep(test_generic(name="Release-Dgreedy", shellname="avmshell_64", vmargs="-Dgreedy", config="", scriptargs="--timeout=180 --random"))
     windows_64_deep_factory.addStep(test_generic(name="DebugDebugger_VerifyAll", shellname="avmshell_sd_64", vmargs="", config="", scriptargs="--verify --timeout=300 --random"))
+    windows_64_deep_factory.addStep(test_generic(name="DebugDebugger_VerifyOnly", shellname="avmshell_sd_64", vmargs="", config="", scriptargs="--verifyonly --timeout=300 --random"))
     windows_64_deep_factory.addStep(util_process_clean)
     windows_64_deep_factory.addStep(util_clean_buildsdir)
 
     windows_64_deep_builder = {
                 'name': "windows64-deep",
                 'slavename': "windows64-deep",
                 'factory': windows_64_deep_factory,
                 'builddir': './windows64-deep',
@@ -2062,16 +2068,17 @@ class tamarinredux:
     linux_deep_factory.addStep(compile_generic(name="Release-system-malloc", shellname="avmshell_sysmalloc", args="--enable-shell --enable-use-system-malloc", upload="true"))
     linux_deep_factory.addStep(test_generic(name="Release-system-malloc", shellname="avmshell_sysmalloc", vmargs="", config="x86-lnx-tvm-release-use-system-malloc", scriptargs=""))
     linux_deep_factory.addStep(test_misc)
     linux_deep_factory.addStep(deep_codecoverage)
     linux_deep_factory.addStep(compile_generic(name="ReleaseDebugger-air", shellname="avmshell_air", args="--enable-shell --enable-override-global-new --enable-use-system-malloc --enable-debugger", upload="true"))
     linux_deep_factory.addStep(test_generic(name="ReleaseDebugger-air", shellname="avmshell_air", vmargs="", config="", scriptargs=""))
     linux_deep_factory.addStep(test_generic(name="Release-Dgreedy", shellname="avmshell", vmargs="-Dgreedy", config="", scriptargs="--timeout=180 --random"))
     linux_deep_factory.addStep(test_generic(name="DebugDebugger_VerifyAll", shellname="avmshell_sd", vmargs="", config="", scriptargs="--verify --timeout=300 --random"))
+    linux_deep_factory.addStep(test_generic(name="DebugDebugger_VerifyOnly", shellname="avmshell_sd", vmargs="", config="", scriptargs="--verifyonly --timeout=300 --random"))
     linux_deep_factory.addStep(util_process_clean)
     linux_deep_factory.addStep(util_clean_buildsdir)
 
     linux_deep_builder = {
                 'name': "linux-deep",
                 'slavename': "linux-deep",
                 'factory': linux_deep_factory,
                 'builddir': './linux-deep',
--- a/test/acceptance/runtests.py
+++ b/test/acceptance/runtests.py
@@ -63,16 +63,17 @@ except ImportError:
     print "   (directory has been moved to test/util)."
 
 class AcceptanceRuntest(RuntestBase):
     abcOnlyExt = '.abc_'  # only run, don't compile these abc files - underscore is used so that tests are not deleted when removing old abc files
     runESC = False
     escbin = '../../esc/bin/'
     androidthreads = False
     androiddevices = []
+    verifyonly = False
 
     def __init__(self):
         # Set threads to # of available cpus/cores
         self.threads = detectCPUs()
         RuntestBase.__init__(self)
 
     def setEnvironVars(self):
         RuntestBase.setEnvironVars(self)
@@ -85,26 +86,28 @@ class AcceptanceRuntest(RuntestBase):
         print '    --escbin        location of esc/bin directory - defaults to ../../esc/bin'
         print '    --eval          use run-time compiler'
         print '    --ext           set the testfile extension (defaults to .as)'
         print '    --ats           generate ats swfs instead of running tests'
         print '    --atsdir        base output directory for ats swfs - defaults to ATS_SWFS'
         print '    --threads       number of threads to run (default=# of cpu/cores), set to 1 to have tests finish sequentially'
         print '    --androidthreads    assign a thread for each android device connected.'
         print '    --verify        run a verify pass instead of running abcs'
+        print '    --verifyonly    run a -Dverifyonly pass: only checks test exitcode'
         print '    --aotsdk        location of the AOT sdk used to compile tests to standalone executables.'
         print '    --aotout        where the resulting binaries should be put (defaults to the location of the as file).'
         print '    --aotargs       any extra arguments to pass to compile.py.'
         print '    --remoteip      IP/DNS address of the machine to run the tests on.'
         print '    --remoteuser    user name to use to connect to the remote machine.'
         exit(c)
 
     def setOptions(self):
         RuntestBase.setOptions(self)
-        self.longOptions.extend(['ext=','esc','escbin=','eval','threads=','ats','atsdir=','verify','androidthreads'])
+        self.longOptions.extend(['ext=','esc','escbin=','eval','threads=','ats',
+                                 'atsdir=','verify','verifyonly','androidthreads'])
 
     def parseOptions(self):
         opts = RuntestBase.parseOptions(self)
         for o, v in opts:
             if o in ('--ext',):
                 self.sourceExt = v
             elif o in ('--esc',):
                 self.runESC = True
@@ -120,16 +123,20 @@ class AcceptanceRuntest(RuntestBase):
             elif o in ('--ats',):
                 self.genAtsSwfs = True
                 self.rebuildtests = True
             elif o in ('--atsdir',):
                 self.atsDir = v
             elif o in ('--verify',):
                 self.verify = True
                 self.vmargs = '-Dverifyall -Dverbose=verify'
+            elif o in ('--verifyonly',):
+                self.verifyonly = True
+                if '-Dverifyonly' not in self.vmargs:
+                    self.vmargs += ' -Dverifyonly'
             elif o in ('--aotsdk',):
                 self.aotsdk = v
             elif o in ('--aotout',):
                 self.aotout = v
             elif o in ('--aotargs',):
                 self.aotextraargs = v
             elif o in ('--remoteip',):
                 self.remoteip = v
@@ -140,16 +147,18 @@ class AcceptanceRuntest(RuntestBase):
         self.setEnvironVars()
         self.loadPropertiesFile()
         self.setOptions()
         self.parseOptions()
         self.setTimestamp()
         self.checkPath()
         if not self.config:
             self.determineConfig()
+        if self.verifyonly and 'debugger' not in self.vmtype:
+            exit('You must be running a debugger build in order to use the -Dverifyonly option')
         if self.rebuildtests==False and (re.search('arm-winmobile-emulator',self.config)!=None or self.osName=='winmobile'):
             if re.search('^arm-winmobile-emulator',self.config)==None:
                 print 'ERROR: to use windows mobile build set --config arm-winmobile-emulator-tvm-release or install cygwin utility /usr/bin/file.exe'
                 sys.exit(1)
             self.setupCEEmulators()
         if self.htmlOutput and not self.rebuildtests:
             self.createOutputFile()
         # extension lists must be tuples
@@ -387,18 +396,25 @@ class AcceptanceRuntest(RuntestBase):
                 lpass += 1
         else:
             if abcargs:
                 (f,err,exitcode) = self.run_pipe('%s %s %s %s -- %s' % (self.avm, self.vmargs, extraVmArgs, testName, abcargs), outputCalls=outputCalls)
             else:
                 (f,err,exitcode) = self.run_pipe('%s %s %s %s' % (self.avm, self.vmargs, extraVmArgs, testName), outputCalls=outputCalls)
 
         # Test has been run, handle output
-
-        if not self.verify:
+        if self.verifyonly:
+            # only check the exit code when running a verifyonly pass
+            ec_lfail, ec_lexpfail, expectedExitcode = self.checkExitCode(exitcode, root, testName, f, err, settings, outputCalls)
+            if ec_lfail or ec_lexpfail:
+                lfail += ec_lfail
+                lexpfail += ec_lexpfail
+            else:
+                lpass += 1
+        elif not self.verify:
             try:
                 outputLines = []
                 if isfile(root+'.out'):
                     # override standard runtests behavior, just compare the .out file with stdout+stderr
                     actual = [line.strip() for line in f+err if line.strip()]
                     try:
                         outfile = open(root+'.out', 'r')
                         expectedOut = [line.strip() for line in outfile.readlines() if line.strip()]
@@ -459,45 +475,22 @@ class AcceptanceRuntest(RuntestBase):
                                 outputCalls.append((self.fail,(testName, 'expected failure: ' + line.strip() + ' reason: '+res, self.expfailmsgs)))
                                 lexpfail += 1
                             else:
                                 lfail += 1
                                 outputCalls.append((self.fail,(testName+extraVmArgs, line, self.failmsgs)))
             except:
                 print 'exception running avm'
                 raise
-            expectedExitcode=0
-            if isfile(root+'.err'):
-                # .err file holds both the expected (non-catchable) error (usually a VerifyError) and the expected exitcode
-                expectedErr,expectedExitcode = self.loadExpectedErr(root+'.err')
-                # check the expectedErr - error is always the last (non-empty) line of output
-                actualErr = ''
-                for line in reversed(f):
-                    # make sure we have an error
-                    if re.search('.*Error:.*', line):
-                        actualErr = self.getError(line.strip())
-                        break
-                if actualErr != expectedErr:
-                    outputCalls.append((self.fail,(testName, 'unexpected error message. expected: %s actual: %s' % (expectedErr, actualErr), self.failmsgs)))
-                    lfail += 1
-            elif isfile(root+".exitcode"):
-                try:
-                    expectedExitcode=int(open(root+".exitcode").read())
-                except:
-                    print("ERROR: reading exit code file '%s' should contain an integer")
-            res=dict_match(settings,'exitcode','expectedfail')
-            if exitcode!=expectedExitcode:
-                res2=dict_match(settings,'exitcode','skip')
-                if res2==None and res:
-                    outputCalls.append((self.js_print,(testName, 'expected failure: exitcode reason: %s'%res,self.expfailmsgs)))
-                    lexpfail += 1
-                elif res2==None:
-                    outputCalls.append((self.fail,(testName, 'unexpected exit code expected:%d actual:%d Signal Name: %s FAILED!' % (expectedExitcode,exitcode,getSignalName(abs(exitcode))), self.failmsgs)))
-                    outputCalls.append((self.fail,(testName, 'captured output: %s' % string.join([l.strip() for l in outputLines], ' | '), self.failmsgs)))
-                    lfail+= 1
+
+            # exitcode check
+            ec_lfail, ec_lexpfail, expectedExitcode = self.checkExitCode(exitcode, root, testName, f, err, settings, outputCalls)
+            if ec_lfail or ec_lexpfail:
+                lfail += ec_lfail
+                lexpfail += ec_lexpfail
             elif err:
                 # TODO: When needed, add support for expected stderr output - see https://bugzilla.mozilla.org/show_bug.cgi?id=561892
                 outputCalls.append((self.fail,(testName, "unexpected stderr expected:'%s' actual:'%s'" % ('',err), self.failmsgs)))
                 lfail += 1  # any messages to stderr automatically fail the test
             elif lpass == 0 and lfail == 0 and lunpass==0 and lexpfail==0:
                 res=dict_match(settings,'*','expectedfail')
                 if res:
                     outputCalls.append((self.fail,(testName, 'expected failure: FAILED contained no testcase messages reason: %s' % res,self.expfailmsgs)))
@@ -524,16 +517,62 @@ class AcceptanceRuntest(RuntestBase):
         if self.show_time:
             outputCalls.insert(0,(self.js_print,('%s running %s %s %s time %.1f' % (testnum, ast, extraVmArgs, abcargs, time()-starttime), '<b>', '</b><br/>')));
         else:
             outputCalls.insert(0,(self.js_print,('%s running %s %s %s' % (testnum, ast, extraVmArgs, abcargs), '<b>', '</b><br/>')));
 
 
         return outputCalls
 
+    def checkExitCode(self, exitcode, root, testName, f, err, settings, outputCalls):
+        '''Check the exitcode for a test against any expected non-zero exitcodes
+            Return the fail and the expected exitcode if non-zero
+        '''
+        lfail = 0
+        lexpfail = 0
+        expectedExitcode=0
+        if isfile(root+'.err'):
+            # .err file holds both the expected (non-catchable) error (usually a VerifyError) and the expected exitcode
+            expectedErr,expectedExitcode = self.loadExpectedErr(root+'.err')
+            # check the expectedErr - error is always the last (non-empty) line of output
+            actualErr = ''
+            for line in reversed(f):
+                # When running in --verifyonly mode, output will be VERIFY FAILED instead of VerifyError
+                line = line.replace('VERIFY FAILED', 'VerifyError')
+                # make sure we have an error
+                if re.search('.*Error:.*', line):
+                    actualErr = self.getError(line.strip())
+                    break
+            if actualErr != expectedErr:
+                outputCalls.append((self.fail,(testName, 'unexpected error message. expected: %s actual: %s'
+                                               % (expectedErr, actualErr), self.failmsgs)))
+                lfail += 1
+        elif isfile(root+".exitcode"):
+            try:
+                expectedExitcode=int(open(root+".exitcode").read())
+            except:
+                print("ERROR: reading exit code file '%s' should contain an integer")
+        res=dict_match(settings,'exitcode','expectedfail')
+        
+        if exitcode!=expectedExitcode:
+            res2=dict_match(settings,'exitcode','skip')
+            if res2==None and res:
+                outputCalls.append((self.js_print,(testName, 'expected failure: exitcode reason: %s'
+                                                   % res,self.expfailmsgs)))
+                lexpfail += 1
+            elif res2==None:
+                outputCalls.append((self.fail,(testName, 'unexpected exit code expected:%d actual:%d Signal Name: %s FAILED!'
+                                               % (expectedExitcode,exitcode,getSignalName(abs(exitcode))),
+                                               self.failmsgs)))
+                outputCalls.append((self.fail,(testName, 'captured output: %s'
+                                               % string.join([l.strip() for l in f+err], ' | '),
+                                               self.failmsgs)))
+                lfail+= 1
+        return lfail, lexpfail, expectedExitcode
+
     def loadExpectedErr(self, file):
         try:
             f = open(file, 'r')
             expectedErr = ''
             expectedExitcode = 0
             for line in f:
                 line = line.strip()
                 if line and not line.startswith('#'):
--- a/test/acceptance/testconfig.txt
+++ b/test/acceptance/testconfig.txt
@@ -321,16 +321,27 @@ ecma3/Unicode/uE000_PrivateUseArea, .*, 
 ecma3/Unicode/uAC00_HangulSyllables, .*, ats_skip, bug 499685 skip slow tests
 
 ####################
 #  --verify skips
 ####################
 abcasm/.*, .*, verify_skip, abcasm tests not compatible with --verify
 
 ####################
+#  --verifyonly skips : tests that intentionally fail will "pass" in -Dverifyonly runs
+####################
+abcasm/bug_554915a, .*-Dverifyonly.*, skip, -Dtimeout test will not timeout when only being verified
+ecma3/Exceptions/exception_011_rt, .*-Dverifyonly.*, skip, Test expects exitcode=1: exits with uncaught exception
+regress/bug_515935, .*-Dverifyonly.*, skip, out of memory test
+regress/bug_555542, .*-Dverifyonly.*, skip, uncaught ReferenceError test
+regress/bug_555705, .*-Dverifyonly.*, skip, test intentionally throws a verifyerror
+regress/bug_498979, .*-Dverifyonly.*, skip, https://bugzilla.mozilla.org/show_bug.cgi?id=585128
+abcasm/nullCheck/CallStatic, .*-Dverifyonly.*, skip, https://bugzilla.mozilla.org/show_bug.cgi?id=585128
+
+####################
 # AOT specific
 ####################
 as3/sampling/FunctionEntry,		.*AOT.*	, skip, AOT does not yet support profiling
 as3/sampling/InvocationSampling,	.*AOT.*	, skip, AOT does not yet support profiling
 as3/ShellClasses/Domain.as,		.*AOT.*	, skip, AOT does not support loadBytes()
 regress/vector_domain_bug,		.*AOT.*	, skip, AOT does not support loadBytes()
 
 ####################
--- a/test/runsmokes.txt
+++ b/test/runsmokes.txt
@@ -61,16 +61,19 @@ cd acceptance;${py} ./runtests.py  --not
 
 # ByteArray compress/uncompress: https://bugzilla.mozilla.org/show_bug.cgi?id=436372
 cd acceptance;${py} ./runtests.py --notimecheck --avm=${avmr} as3/ShellClasses/ByteArray.as
 cd acceptance;${py} ./runtests.py --notimecheck --avm=${avmr} as3/ShellClasses/File.as
 
 # https://bugzilla.mozilla.org/show_bug.cgi?id=517858
 cd acceptance;${py} ./runtests.py  --notimecheck --avm=${avmdd} mmgc/outofmemory.as
 
+# test -Dverifyonly in smokes: https://bugzilla.mozilla.org/show_bug.cgi?id=491934
+cd acceptance;${py} ./runtests.py  --notimecheck --avm=${avmdd} --verifyonly as3/Vector/typechecking.as
+
 # We have seen this test fail the GetSizeSamples test
 cd acceptance;${py} ./runtests.py  --notimecheck --avm=${avmr} as3/sampling/ProcessSampling.as
 cd acceptance;${py} ./runtests.py  --notimecheck --avm=${avmrd} as3/sampling/ProcessSampling.as
 cd acceptance;${py} ./runtests.py  --notimecheck --avm=${avmd} as3/sampling/ProcessSampling.as
 cd acceptance;${py} ./runtests.py  --notimecheck --avm=${avmdd} as3/sampling/ProcessSampling.as
 
 # Bug https://bugzilla.mozilla.org/show_bug.cgi?id=402864
 cd acceptance;${py} ./runtests.py  --notimecheck --avm=${avmr} ecma3/GlobalObject/e15_1_2_2_2.as ecma3/TypeConversion/e9_3_1_3.as