Bug 952456 - Part 1: Implement gonk/nsClipboard for rich text and raw image r=fabrice
☠☠ backed out by 3ef1a1856a3f ☠ ☠
authorBoris Chiou <boris.chiou@gmail.com>
Mon, 17 Aug 2015 15:49:14 -0700
changeset 258189 e00a383520f6f414ca595d5e3dff8be5da4dfe7f
parent 258188 291ae45aa7ce4d72b18e013dc2f89fdea30e1611
child 258190 3e5d45dcd5c6f088b9be5188736d259c9cca7f4a
push id63843
push userryanvm@gmail.com
push dateTue, 18 Aug 2015 14:58:06 +0000
treeherdermozilla-inbound@d55e24c983aa [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfabrice
bugs952456
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 952456 - Part 1: Implement gonk/nsClipboard for rich text and raw image r=fabrice Handle text/html and image MIME types on gonk/nsClipboard
widget/gonk/GonkClipboardData.cpp
widget/gonk/GonkClipboardData.h
widget/gonk/moz.build
widget/gonk/nsClipboard.cpp
widget/gonk/nsClipboard.h
new file mode 100644
--- /dev/null
+++ b/widget/gonk/GonkClipboardData.cpp
@@ -0,0 +1,75 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "GonkClipboardData.h"
+#include "mozilla/gfx/DataSurfaceHelpers.h"
+
+namespace mozilla {
+
+void
+GonkClipboardData::SetText(const nsAutoString &aText)
+{
+  mPlain = aText;
+}
+
+bool
+GonkClipboardData::HasText() const
+{
+  return !mPlain.IsEmpty();
+}
+
+const nsAutoString&
+GonkClipboardData::GetText() const
+{
+  return mPlain;
+}
+
+void
+GonkClipboardData::SetHTML(const nsAutoString &aHTML)
+{
+  mHTML = aHTML;
+}
+
+bool
+GonkClipboardData::HasHTML() const
+{
+  return !mHTML.IsEmpty();
+}
+
+const nsAutoString&
+GonkClipboardData::GetHTML() const
+{
+  return mHTML;
+}
+
+void
+GonkClipboardData::SetImage(gfx::DataSourceSurface* aDataSource)
+{
+  // Clone a new DataSourceSurface and store it.
+  mImage = gfx::CreateDataSourceSurfaceByCloning(aDataSource);
+}
+
+bool
+GonkClipboardData::HasImage() const
+{
+  return static_cast<bool>(mImage);
+}
+
+already_AddRefed<gfx::DataSourceSurface>
+GonkClipboardData::GetImage() const
+{
+  // Return cloned DataSourceSurface.
+  RefPtr<gfx::DataSourceSurface> cloned = gfx::CreateDataSourceSurfaceByCloning(mImage);
+  return cloned.forget();
+}
+
+void
+GonkClipboardData::Clear()
+{
+  mPlain.Truncate(0);
+  mHTML.Truncate(0);
+  mImage = nullptr;
+}
+
+} // namespace mozilla
new file mode 100644
--- /dev/null
+++ b/widget/gonk/GonkClipboardData.h
@@ -0,0 +1,49 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_GonkClipboardData
+#define mozilla_GonkClipboardData
+
+#include "mozilla/RefPtr.h"
+#include "nsString.h"
+
+namespace mozilla {
+
+namespace gfx {
+class DataSourceSurface;
+}
+
+class GonkClipboardData final
+{
+public:
+  explicit GonkClipboardData() = default;
+  ~GonkClipboardData() = default;
+
+  // For text/plain
+  void SetText(const nsAutoString &aText);
+  bool HasText() const;
+  const nsAutoString& GetText() const;
+
+  // For text/html
+  void SetHTML(const nsAutoString &aHTML);
+  bool HasHTML() const;
+  const nsAutoString& GetHTML() const;
+
+  // For images
+  void SetImage(gfx::DataSourceSurface* aDataSource);
+  bool HasImage() const;
+  already_AddRefed<gfx::DataSourceSurface> GetImage() const;
+
+  // For other APIs
+  void Clear();
+
+private:
+  nsAutoString mPlain;
+  nsAutoString mHTML;
+  RefPtr<gfx::DataSourceSurface> mImage;
+};
+
+} // namespace mozilla
+
+#endif // mozilla_GonkClipboardData
--- a/widget/gonk/moz.build
+++ b/widget/gonk/moz.build
@@ -53,16 +53,17 @@ if CONFIG['ANDROID_VERSION'] >= '17':
 elif CONFIG['ANDROID_VERSION'] == '15':
     SOURCES += [
         'hwchal/HwcICS.cpp',
     ]
 
 SOURCES += [
     'GeckoTouchDispatcher.cpp',
     'GfxInfo.cpp',
+    'GonkClipboardData.cpp',
     'GonkMemoryPressureMonitoring.cpp',
     'GonkPermission.cpp',
     'HwcComposer2D.cpp',
     'HwcUtils.cpp',
     'nsAppShell.cpp',
     'nsClipboard.cpp',
     'nsIdleServiceGonk.cpp',
     'nsLookAndFeel.cpp',
@@ -77,16 +78,17 @@ SOURCES += [
 include('/ipc/chromium/chromium-config.mozbuild')
 
 FINAL_LIBRARY = 'xul'
 
 LOCAL_INCLUDES += [
     '/dom/system/android',
     '/gfx/skia/skia/include/config',
     '/gfx/skia/skia/include/core',
+    '/image',
     '/widget',
 ]
 
 DEFINES['HAVE_OFF64_T'] = True
 DEFINES['SK_BUILD_FOR_ANDROID_NDK'] = True
 
 if CONFIG['ANDROID_VERSION'] != '15':
     DEFINES['HAVE_POSIX_CLOCKS'] = True
--- a/widget/gonk/nsClipboard.cpp
+++ b/widget/gonk/nsClipboard.cpp
@@ -1,134 +1,303 @@
 /* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+#include "nsClipboard.h"
+
+#include "gfxDrawable.h"
+#include "gfxUtils.h"
+#include "ImageOps.h"
+#include "imgIContainer.h"
+#include "imgTools.h"
 #include "mozilla/dom/ContentChild.h"
-#include "nsClipboard.h"
 #include "nsClipboardProxy.h"
 #include "nsISupportsPrimitives.h"
+#include "nsComponentManagerUtils.h"
 #include "nsCOMPtr.h"
-#include "nsComponentManagerUtils.h"
+#include "nsStringStream.h"
 #include "nsXULAppAPI.h"
 
 using namespace mozilla;
 using mozilla::dom::ContentChild;
 
 #define LOG_TAG "Clipboard"
 #define LOGI(args...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, ## args)
 #define LOGE(args...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, ## args)
 
+
 NS_IMPL_ISUPPORTS(nsClipboard, nsIClipboard)
 
 nsClipboard::nsClipboard()
+  : mClipboard(mozilla::MakeUnique<GonkClipboardData>())
 {
 }
 
 NS_IMETHODIMP
 nsClipboard::SetData(nsITransferable *aTransferable,
-                     nsIClipboardOwner *anOwner, int32_t aWhichClipboard)
+                     nsIClipboardOwner *anOwner,
+                     int32_t aWhichClipboard)
 {
   if (aWhichClipboard != kGlobalClipboard) {
     return NS_ERROR_NOT_IMPLEMENTED;
   }
 
   if (!XRE_IsParentProcess()) {
     // Re-direct to the clipboard proxy.
     nsRefPtr<nsClipboardProxy> clipboardProxy = new nsClipboardProxy();
     return clipboardProxy->SetData(aTransferable, anOwner, aWhichClipboard);
   }
 
-  nsCOMPtr<nsISupports> tmp;
-  uint32_t len;
-  nsresult rv  = aTransferable->GetTransferData(kUnicodeMime, getter_AddRefs(tmp),
-                                                &len);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
+  // Get the types of supported flavors.
+  nsCOMPtr<nsISupportsArray> flavorList;
+  nsresult rv = aTransferable->FlavorsTransferableCanExport(getter_AddRefs(flavorList));
+
+  if (!flavorList || NS_FAILED(rv)) {
+    return NS_ERROR_FAILURE;
   }
-  nsCOMPtr<nsISupportsString> supportsString = do_QueryInterface(tmp);
-  // No support for non-text data
-  if (NS_WARN_IF(!supportsString)) {
-    LOGE("No support for non-text data. See bug 952456.");
-    return NS_ERROR_NOT_IMPLEMENTED;
+
+  uint32_t flavorCount = 0;
+  flavorList->Count(&flavorCount);
+  bool imageAdded = false;
+  for (uint32_t i = 0; i < flavorCount; ++i) {
+    nsCOMPtr<nsISupportsCString> currentFlavor = do_QueryElementAt(flavorList, i);
+
+    if (currentFlavor) {
+      // MIME type
+      nsXPIDLCString flavorStr;
+      currentFlavor->ToString(getter_Copies(flavorStr));
+
+      // Clip is the data which will be sent to the clipboard.
+      nsCOMPtr<nsISupports> clip;
+      uint32_t len;
+
+      if (flavorStr.EqualsLiteral(kUnicodeMime)) {
+        // text/plain
+        rv = aTransferable->GetTransferData(flavorStr, getter_AddRefs(clip), &len);
+        nsCOMPtr<nsISupportsString> wideString = do_QueryInterface(clip);
+        if (!wideString || NS_FAILED(rv)) {
+          continue;
+        }
+
+        nsAutoString utf16string;
+        wideString->GetData(utf16string);
+        mClipboard->SetText(utf16string);
+      } else if (flavorStr.EqualsLiteral(kHTMLMime)) {
+        // text/html
+        rv = aTransferable->GetTransferData(flavorStr, getter_AddRefs(clip), &len);
+        nsCOMPtr<nsISupportsString> wideString = do_QueryInterface(clip);
+        if (!wideString || NS_FAILED(rv)) {
+          continue;
+        }
+
+        nsAutoString utf16string;
+        wideString->GetData(utf16string);
+        mClipboard->SetHTML(utf16string);
+      } else if (!imageAdded && // image is added only once to the clipboard.
+                 (flavorStr.EqualsLiteral(kNativeImageMime) ||
+                  flavorStr.EqualsLiteral(kPNGImageMime) ||
+                  flavorStr.EqualsLiteral(kJPEGImageMime) ||
+                  flavorStr.EqualsLiteral(kJPGImageMime) ||
+                  flavorStr.EqualsLiteral(kGIFImageMime))) {
+        // image/[png|jpeg|jpg|gif] or application/x-moz-nativeimage
+
+        // Look through our transfer data for the image.
+        static const char* const imageMimeTypes[] = {
+          kNativeImageMime, kPNGImageMime, kJPEGImageMime, kJPGImageMime, kGIFImageMime };
+
+        nsCOMPtr<nsISupportsInterfacePointer> imgPtr;
+        for (uint32_t i = 0; !imgPtr && i < ArrayLength(imageMimeTypes); ++i) {
+          aTransferable->GetTransferData(imageMimeTypes[i], getter_AddRefs(clip), &len);
+          imgPtr = do_QueryInterface(clip);
+        }
+        if (!imgPtr) {
+          continue;
+        }
+
+        nsCOMPtr<nsISupports> imageData;
+        imgPtr->GetData(getter_AddRefs(imageData));
+        nsCOMPtr<imgIContainer> image(do_QueryInterface(imageData));
+        if (!image) {
+          continue;
+        }
+
+        RefPtr<gfx::SourceSurface> surface =
+          image->GetFrame(imgIContainer::FRAME_CURRENT,
+                          imgIContainer::FLAG_SYNC_DECODE);
+        if (!surface) {
+          continue;
+        }
+
+        RefPtr<gfx::DataSourceSurface> dataSurface;
+        if (surface->GetFormat() == gfx::SurfaceFormat::B8G8R8A8) {
+          dataSurface = surface->GetDataSurface();
+        } else {
+          // Convert format to SurfaceFormat::B8G8R8A8.
+          dataSurface = gfxUtils::CopySurfaceToDataSourceSurfaceWithFormat(surface, gfx::SurfaceFormat::B8G8R8A8);
+        }
+
+        mClipboard->SetImage(dataSurface);
+        imageAdded = true;
+      }
+    }
   }
-  nsAutoString buffer;
-  supportsString->GetData(buffer);
 
-  mClipboard = buffer;
   return NS_OK;
 }
 
 NS_IMETHODIMP
-nsClipboard::GetData(nsITransferable *aTransferable, int32_t aWhichClipboard)
+nsClipboard::GetData(nsITransferable *aTransferable,
+                     int32_t aWhichClipboard)
 {
   if (aWhichClipboard != kGlobalClipboard) {
     return NS_ERROR_NOT_IMPLEMENTED;
   }
 
   if (!XRE_IsParentProcess()) {
     // Re-direct to the clipboard proxy.
     nsRefPtr<nsClipboardProxy> clipboardProxy = new nsClipboardProxy();
     return clipboardProxy->GetData(aTransferable, aWhichClipboard);
   }
 
-  nsAutoString buffer(mClipboard);
+  // Get flavor list that includes all acceptable flavors (including
+  // ones obtained through conversion).
+  // Note: We don't need to call nsITransferable::AddDataFlavor here
+  //       because ContentParent already did.
+  nsCOMPtr<nsISupportsArray> flavorList;
+  nsresult rv = aTransferable->FlavorsTransferableCanImport(getter_AddRefs(flavorList));
 
-  nsresult rv;
-  nsCOMPtr<nsISupportsString> dataWrapper =
-    do_CreateInstance(NS_SUPPORTS_STRING_CONTRACTID, &rv);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
+  if (!flavorList || NS_FAILED(rv)) {
+    return NS_ERROR_FAILURE;
   }
 
-  rv = dataWrapper->SetData(buffer);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
+  // Walk through flavors and see which flavor matches the one being pasted.
+  uint32_t flavorCount;
+  flavorList->Count(&flavorCount);
+
+  for (uint32_t i = 0; i < flavorCount; ++i) {
+    nsCOMPtr<nsISupportsCString> currentFlavor = do_QueryElementAt(flavorList, i);
+
+    if (currentFlavor) {
+      // flavorStr is the mime type.
+      nsXPIDLCString flavorStr;
+      currentFlavor->ToString(getter_Copies(flavorStr));
+
+      // text/plain, text/Unicode
+      if (flavorStr.EqualsLiteral(kUnicodeMime) && mClipboard->HasText()) {
+        nsresult rv;
+        nsCOMPtr<nsISupportsString> dataWrapper = do_CreateInstance(NS_SUPPORTS_STRING_CONTRACTID, &rv);
+        rv = dataWrapper->SetData(mClipboard->GetText());
+        if (NS_WARN_IF(NS_FAILED(rv))) {
+          continue;
+        }
+
+        nsCOMPtr<nsISupports> genericDataWrapper = do_QueryInterface(dataWrapper);
+        uint32_t len = mClipboard->GetText().Length() * sizeof(PRUnichar);
+        rv = aTransferable->SetTransferData(flavorStr, genericDataWrapper, len);
+        if (NS_WARN_IF(NS_FAILED(rv))) {
+          continue;
+        }
+        break;
+      }
 
-  // If our data flavor has already been added, this will fail. But we don't care
-  aTransferable->AddDataFlavor(kUnicodeMime);
+      // text/html
+      if (flavorStr.EqualsLiteral(kHTMLMime) && mClipboard->HasHTML()) {
+        nsresult rv;
+        nsCOMPtr<nsISupportsString> dataWrapper = do_CreateInstance(NS_SUPPORTS_STRING_CONTRACTID, &rv);
+        rv = dataWrapper->SetData(mClipboard->GetHTML());
+        if (NS_WARN_IF(NS_FAILED(rv))) {
+          continue;
+        }
+
+        nsCOMPtr<nsISupports> genericDataWrapper = do_QueryInterface(dataWrapper);
+        uint32_t len = mClipboard->GetHTML().Length() * sizeof(PRUnichar);
+        rv = aTransferable->SetTransferData(flavorStr, genericDataWrapper, len);
+        if (NS_WARN_IF(NS_FAILED(rv))) {
+          continue;
+        }
+        break;
+      }
 
-  nsCOMPtr<nsISupports> nsisupportsDataWrapper =
-    do_QueryInterface(dataWrapper);
-  rv = aTransferable->SetTransferData(kUnicodeMime, nsisupportsDataWrapper,
-                                      buffer.Length() * sizeof(PRUnichar));
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
+      // image/[png|jpeg|jpg|gif]
+      if ((flavorStr.EqualsLiteral(kPNGImageMime) ||
+           flavorStr.EqualsLiteral(kJPEGImageMime) ||
+           flavorStr.EqualsLiteral(kJPGImageMime) ||
+           flavorStr.EqualsLiteral(kGIFImageMime)) &&
+          mClipboard->HasImage() ) {
+        // Get image buffer from clipboard.
+        RefPtr<gfx::DataSourceSurface> image = mClipboard->GetImage();
+
+        // Encode according to MIME type.
+        nsRefPtr<gfxDrawable> drawable = new gfxSurfaceDrawable(image, image->GetSize());
+        nsCOMPtr<imgIContainer> imageContainer(image::ImageOps::CreateFromDrawable(drawable));
+        nsCOMPtr<imgITools> imgTool = do_GetService(NS_IMGTOOLS_CID);
+
+        nsCOMPtr<nsIInputStream> byteStream;
+        imgTool->EncodeImage(imageContainer, flavorStr, EmptyString(), getter_AddRefs(byteStream));
+
+        // Set transferable.
+        nsresult rv = aTransferable->SetTransferData(flavorStr,
+                                                     byteStream,
+                                                     sizeof(nsIInputStream*));
+        if (NS_WARN_IF(NS_FAILED(rv))) {
+          continue;
+        }
+        break;
+      }
+    }
   }
 
   return NS_OK;
 }
 
 NS_IMETHODIMP
 nsClipboard::EmptyClipboard(int32_t aWhichClipboard)
 {
   if (aWhichClipboard != kGlobalClipboard) {
     return NS_ERROR_NOT_IMPLEMENTED;
   }
   if (XRE_IsParentProcess()) {
-    mClipboard.Truncate(0);
+    mClipboard->Clear();
   } else {
     ContentChild::GetSingleton()->SendEmptyClipboard(aWhichClipboard);
   }
 
   return NS_OK;
 }
 
 NS_IMETHODIMP
 nsClipboard::HasDataMatchingFlavors(const char **aFlavorList,
                                     uint32_t aLength, int32_t aWhichClipboard,
                                     bool *aHasType)
 {
   *aHasType = false;
+
   if (aWhichClipboard != kGlobalClipboard) {
     return NS_ERROR_NOT_IMPLEMENTED;
   }
   if (XRE_IsParentProcess()) {
-    *aHasType = !mClipboard.IsEmpty();
+    for (uint32_t i = 0; i < aLength; ++i) {
+      const char *flavor = aFlavorList[i];
+      if (!flavor) {
+        continue;
+      }
+      if (!strcmp(flavor, kUnicodeMime)) {
+        *aHasType = mClipboard->HasText();
+      } else if (!strcmp(flavor, kHTMLMime)) {
+        *aHasType = mClipboard->HasHTML();
+      } else if (!strcmp(flavor, kJPEGImageMime) ||
+                 !strcmp(flavor, kJPGImageMime) ||
+                 !strcmp(flavor, kPNGImageMime) ||
+                 !strcmp(flavor, kGIFImageMime)) {
+        // We will encode the image into any format you want, so we don't
+        // need to check each specific format
+        *aHasType = mClipboard->HasImage();
+      }
+    }
   } else {
     nsRefPtr<nsClipboardProxy> clipboardProxy = new nsClipboardProxy();
     return clipboardProxy->HasDataMatchingFlavors(aFlavorList, aLength, aWhichClipboard, aHasType);
   }
   return NS_OK;
 }
 
 NS_IMETHODIMP
--- a/widget/gonk/nsClipboard.h
+++ b/widget/gonk/nsClipboard.h
@@ -1,24 +1,27 @@
-/* -*- Mode: c++; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
- * This Source Code Form is subject to the terms of the Mozilla Public
+/* This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #ifndef nsClipbard_h__
 #define nsClipbard_h__
 
+#include "GonkClipboardData.h"
+#include "mozilla/UniquePtr.h"
 #include "nsIClipboard.h"
 
 class nsClipboard final : public nsIClipboard
 {
-  nsAutoString mClipboard;
 public:
   NS_DECL_ISUPPORTS
   NS_DECL_NSICLIPBOARD
 
   nsClipboard();
 
 protected:
   ~nsClipboard() {}
+
+private:
+  mozilla::UniquePtr<mozilla::GonkClipboardData> mClipboard;
 };
 
 #endif