Bug 1597963 - Pass VT_ERROR for Explorer to call ShellExecuteExW with null verb. r=aklotz a=pascalc FIREFOX_71_0_BUILD2
authorToshihito Kikuchi <tkikuchi@mozilla.com>
Mon, 25 Nov 2019 22:26:42 +0200
changeset 563491 e136c48f3e881f26cf0cb28603a8860770c73acd
parent 563490 2dd6b102980ef537ea2dbf59b32a2c9366570914
child 563492 8ef3650711add852abca2661f6eb43ecc9f53d28
push id2199
push userccoroiu@mozilla.com
push dateMon, 25 Nov 2019 20:40:40 +0000
treeherdermozilla-release@e136c48f3e88 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaklotz, pascalc
bugs1597963, 1588975, 20140919, 44023
milestone71.0
Bug 1597963 - Pass VT_ERROR for Explorer to call ShellExecuteExW with null verb. r=aklotz a=pascalc The patch for Bug 1588975 specified the "open" verb to execute a target, but the default verb is not always "open". For example, the default verb for a font file is "preview". We should specify null verb to start the default operation. Now we use `IShellDispatch2.ShellExecute` to ask explorer.exe to call `ShellExecuteExW`. That method takes an optional `VARIANT` parameter as a verb. According to https://devblogs.microsoft.com/oldnewthing/20140919-00/?p=44023, we need to pass `VT_ERROR` to omit an optional parameter. If we pass other values such as `nullptr` with `VT_BSTR` or `VT_EMPTY`, explorer.exe calls `ShellExecuteExW` with the empty string `""` instead of `nullptr`, which is not considered as a valid verb if the target file is not associated with any app. Differential Revision: https://phabricator.services.mozilla.com//D54603
uriloader/exthandler/win/nsMIMEInfoWin.cpp
xpcom/io/nsLocalFileWin.cpp
--- a/uriloader/exthandler/win/nsMIMEInfoWin.cpp
+++ b/uriloader/exthandler/win/nsMIMEInfoWin.cpp
@@ -50,25 +50,27 @@ static nsresult ShellExecuteWithIFile(co
   if (NS_FAILED(rv) || execPath.IsEmpty()) {
     rv = aExecutable->GetPath(execPath);
   }
   if (NS_FAILED(rv)) {
     return rv;
   }
 
   _bstr_t execPathBStr(execPath.get());
-  _variant_t verb(L"open");
+  // Pass VT_ERROR/DISP_E_PARAMNOTFOUND to omit an optional RPC parameter
+  // to execute a file with the default verb.
+  _variant_t verbDefault(DISP_E_PARAMNOTFOUND, VT_ERROR);
   _variant_t workingDir;
   _variant_t showCmd(SW_SHOWNORMAL);
 
   // Ask Explorer to ShellExecute on our behalf, as some applications such as
   // Skype for Business do not start correctly when inheriting our process's
   // migitation policies.
   mozilla::LauncherVoidResult shellExecuteOk = mozilla::ShellExecuteByExplorer(
-      execPathBStr, aArgs, verb, workingDir, showCmd);
+      execPathBStr, aArgs, verbDefault, workingDir, showCmd);
   if (shellExecuteOk.isErr()) {
     return NS_ERROR_FILE_EXECUTION_FAILED;
   }
 
   return NS_OK;
 }
 
 NS_IMETHODIMP
--- a/xpcom/io/nsLocalFileWin.cpp
+++ b/xpcom/io/nsLocalFileWin.cpp
@@ -3040,17 +3040,19 @@ nsLocalFile::Launch() {
   if (NS_FAILED(rv)) {
     return rv;
   }
 
   // use the app registry name to launch a shell execute....
   _bstr_t execPath(mResolvedPath.get());
 
   _variant_t args;
-  _variant_t verb(L"open");
+  // Pass VT_ERROR/DISP_E_PARAMNOTFOUND to omit an optional RPC parameter
+  // to execute a file with the default verb.
+  _variant_t verbDefault(DISP_E_PARAMNOTFOUND, VT_ERROR);
   _variant_t workingDir;
   _variant_t showCmd(SW_SHOWNORMAL);
 
   // Use the directory of the file we're launching as the working
   // directory. That way if we have a self extracting EXE it won't
   // suggest to extract to the install directory.
   WCHAR workingDirectory[MAX_PATH + 1] = {L'\0'};
   wcsncpy(workingDirectory, mResolvedPath.get(), MAX_PATH);
@@ -3059,17 +3061,17 @@ nsLocalFile::Launch() {
   } else {
     NS_WARNING("Could not set working directory for launched file.");
   }
 
   // Ask Explorer to ShellExecute on our behalf, as some applications such as
   // Skype for Business do not start correctly when inheriting our process's
   // migitation policies.
   mozilla::LauncherVoidResult shellExecuteOk = mozilla::ShellExecuteByExplorer(
-      execPath, args, verb, workingDir, showCmd);
+      execPath, args, verbDefault, workingDir, showCmd);
   if (shellExecuteOk.isOk()) {
     return NS_OK;
   }
   DWORD r = shellExecuteOk.inspectErr().AsWin32Error().value();
   // if the file has no association, we launch windows'
   // "what do you want to do" dialog
   if (r == SE_ERR_NOASSOC) {
     SHELLEXECUTEINFOW seinfo;