media/webrtc/signaling/src/media-conduit/MediaConduitInterface.h
author Nico Grunbaum
Mon, 04 Dec 2017 14:19:46 -0800
changeset 394897 e19b017880c8d71385aed453bacbe2b473757e90
parent 394735 8a7257832c9c6888e7e3c024182a4901e0746aba
child 396653 b299ba22a6a210cc4fa270e58a1846a4c0a92c08
permissions -rw-r--r--
Bug 1422987 - Correct override warning in RtpSourceObserver work. r=mjf, a=RyanVM MozReview-Commit-ID: 6bkbBfodlUH

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef MEDIA_CONDUIT_ABSTRACTION_
#define MEDIA_CONDUIT_ABSTRACTION_

#include "nsISupportsImpl.h"
#include "nsXPCOM.h"
#include "nsDOMNavigationTiming.h"
#include "mozilla/RefPtr.h"
#include "mozilla/RefCounted.h"
#include "mozilla/UniquePtr.h"
#include "double-conversion/utils.h" // for DISALLOW_COPY_AND_ASSIGN
#include "RtpSourceObserver.h"
#include "CodecConfig.h"
#include "VideoTypes.h"
#include "MediaConduitErrors.h"

#include "ImageContainer.h"

#include "webrtc/call.h"
#include "webrtc/config.h"
#include "webrtc/common_types.h"
#include "webrtc/common_types.h"
#include "webrtc/api/video/video_frame_buffer.h"
#include "webrtc/logging/rtc_event_log/rtc_event_log.h"

#include <vector>

namespace webrtc {
class VideoFrame;
}

namespace mozilla {

using RtpExtList = std::vector<webrtc::RtpExtension>;

// Wrap the webrtc.org Call class adding mozilla add/ref support.
class WebRtcCallWrapper : public RefCounted<WebRtcCallWrapper>
{
public:
  typedef webrtc::Call::Config Config;

  static RefPtr<WebRtcCallWrapper> Create()
  {
    return new WebRtcCallWrapper();
  }

  static RefPtr<WebRtcCallWrapper> Create(UniquePtr<webrtc::Call>&& aCall)
  {
    return new WebRtcCallWrapper(std::move(aCall));
  }

  webrtc::Call* Call() const
  {
    return mCall.get();
  }

  virtual ~WebRtcCallWrapper()
  {
    if (mCall->voice_engine()) {
      webrtc::VoiceEngine* voice_engine = mCall->voice_engine();
      mCall.reset(nullptr); // Force it to release the voice engine reference
      // Delete() must be after all refs are released
      webrtc::VoiceEngine::Delete(voice_engine);
    } else {
      // Must ensure it's destroyed *before* the EventLog!
      mCall.reset(nullptr);
    }
  }

  MOZ_DECLARE_REFCOUNTED_TYPENAME(WebRtcCallWrapper)

private:
  WebRtcCallWrapper()
  {
    webrtc::Call::Config config(&mEventLog);
    mCall.reset(webrtc::Call::Create(config));
  }

  explicit WebRtcCallWrapper(UniquePtr<webrtc::Call>&& aCall)
  {
    MOZ_ASSERT(aCall);
    mCall = std::move(aCall);
  }

  DISALLOW_COPY_AND_ASSIGN(WebRtcCallWrapper);
  UniquePtr<webrtc::Call> mCall;
  webrtc::RtcEventLogNullImpl mEventLog;
};


/**
 * Abstract Interface for transporting RTP packets - audio/vidoeo
 * The consumers of this interface are responsible for passing in
 * the RTPfied media packets
 */
class TransportInterface
{
protected:
  virtual ~TransportInterface() {}

public:
  /**
   * RTP Transport Function to be implemented by concrete transport implementation
   * @param data : RTP Packet (audio/video) to be transported
   * @param len  : Length of the media packet
   * @result     : NS_OK on success, NS_ERROR_FAILURE otherwise
   */
  virtual nsresult SendRtpPacket(const uint8_t* data, size_t len) = 0;

  /**
   * RTCP Transport Function to be implemented by concrete transport implementation
   * @param data : RTCP Packet to be transported
   * @param len  : Length of the RTCP packet
   * @result     : NS_OK on success, NS_ERROR_FAILURE otherwise
   */
  virtual nsresult SendRtcpPacket(const uint8_t* data, size_t len) = 0;
  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TransportInterface)
};

/**
 * 1. Abstract renderer for video data
 * 2. This class acts as abstract interface between the video-engine and
 *    video-engine agnostic renderer implementation.
 * 3. Concrete implementation of this interface is responsible for
 *    processing and/or rendering the obtained raw video frame to appropriate
 *    output , say, <video>
 */
class VideoRenderer
{
protected:
  virtual ~VideoRenderer() {}

public:
  /**
   * Callback Function reportng any change in the video-frame dimensions
   * @param width:  current width of the video @ decoder
   * @param height: current height of the video @ decoder
   * @param number_of_streams: number of participating video streams
   */
  virtual void FrameSizeChange(unsigned int width,
                               unsigned int height,
                               unsigned int number_of_streams) = 0;

  /**
   * Callback Function reporting decoded frame for processing.
   * @param buffer: reference to decoded video frame
   * @param buffer_size: size of the decoded frame
   * @param time_stamp: Decoder timestamp, typically 90KHz as per RTP
   * @render_time: Wall-clock time at the decoder for synchronization
   *                purposes in milliseconds
   * NOTE: If decoded video frame is passed through buffer , it is the
   * responsibility of the concrete implementations of this class to own copy
   * of the frame if needed for time longer than scope of this callback.
   * Such implementations should be quick in processing the frames and return
   * immediately.
   */
  virtual void RenderVideoFrame(const webrtc::VideoFrameBuffer& buffer,
                                uint32_t time_stamp,
                                int64_t render_time) = 0;

  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(VideoRenderer)
};


/**
 * Generic Interface for representing Audio/Video Session
 * MediaSession conduit is identified by 2 main components
 * 1. Attached Transport Interface for inbound and outbound RTP transport
 * 2. Attached Renderer Interface for rendering media data off the network
 * This class hides specifics of Media-Engine implementation from the consumers
 * of this interface.
 * Also provides codec configuration API for the media sent and recevied
 */
class MediaSessionConduit
{
protected:
  virtual ~MediaSessionConduit() {}

public:
  enum Type { AUDIO, VIDEO } ;

  virtual Type type() const = 0;

  /**
   * Function triggered on Incoming RTP packet from the remote
   * endpoint by the transport implementation.
   * @param data : RTP Packet (audio/video) to be processed
   * @param len  : Length of the media packet
   * Obtained packets are passed to the Media-Engine for further
   * processing , say, decoding
   */
  virtual MediaConduitErrorCode ReceivedRTPPacket(const void *data, int len, uint32_t ssrc) = 0;

  /**
   * Function triggered on Incoming RTCP packet from the remote
   * endpoint by the transport implementation.
   * @param data : RTCP Packet (audio/video) to be processed
   * @param len  : Length of the media packet
   * Obtained packets are passed to the Media-Engine for further
   * processing , say, decoding
   */
  virtual MediaConduitErrorCode ReceivedRTCPPacket(const void *data, int len) = 0;

  virtual MediaConduitErrorCode StopTransmitting() = 0;
  virtual MediaConduitErrorCode StartTransmitting() = 0;
  virtual MediaConduitErrorCode StopReceiving() = 0;
  virtual MediaConduitErrorCode StartReceiving() = 0;


  /**
   * Function to attach transmitter transport end-point of the Media conduit.
   * @param aTransport: Reference to the concrete teansport implementation
   * When nullptr, unsets the transmitter transport endpoint.
   * Note: Multiple invocations of this call , replaces existing transport with
   * with the new one.
   * Note: This transport is used for RTP, and RTCP if no receiver transport is
   * set. In the future, we should ensure that RTCP sender reports use this
   * regardless of whether the receiver transport is set.
   */
  virtual MediaConduitErrorCode SetTransmitterTransport(RefPtr<TransportInterface> aTransport) = 0;

  /**
   * Function to attach receiver transport end-point of the Media conduit.
   * @param aTransport: Reference to the concrete teansport implementation
   * When nullptr, unsets the receiver transport endpoint.
   * Note: Multiple invocations of this call , replaces existing transport with
   * with the new one.
   * Note: This transport is used for RTCP.
   * Note: In the future, we should avoid using this for RTCP sender reports.
   */
  virtual MediaConduitErrorCode SetReceiverTransport(RefPtr<TransportInterface> aTransport) = 0;

  /* Sets the local SSRCs
   * @return true iff the local ssrcs == aSSRCs upon return
   * Note: this is an ordered list and {a,b,c} != {b,a,c}
   */
  virtual bool SetLocalSSRCs(const std::vector<unsigned int>& aSSRCs) = 0;
  virtual std::vector<unsigned int> GetLocalSSRCs() const = 0;

  /**
  * Adds negotiated RTP extensions
  * XXX Move to MediaSessionConduit
  */
  virtual void
  SetLocalRTPExtensions(bool aIsSend, const RtpExtList& extensions) = 0;
  /**
  * Returns the negotiated RTP extensions
  */
  virtual RtpExtList GetLocalRTPExtensions(bool aIsSend) const = 0;

  virtual bool GetRemoteSSRC(unsigned int* ssrc) = 0;
  virtual bool SetRemoteSSRC(unsigned int ssrc) = 0;
  virtual bool SetLocalCNAME(const char* cname) = 0;

  virtual bool SetLocalMID(const std::string& mid) = 0;

  /**
   * Functions returning stats needed by w3c stats model.
   */

  virtual bool
  GetSendPacketTypeStats(webrtc::RtcpPacketTypeCounter* aPacketCounts) = 0;

  virtual bool
  GetRecvPacketTypeStats(webrtc::RtcpPacketTypeCounter* aPacketCounts) = 0;

  virtual bool GetVideoEncoderStats(double* framerateMean,
                                    double* framerateStdDev,
                                    double* bitrateMean,
                                    double* bitrateStdDev,
                                    uint32_t* droppedFrames,
                                    uint32_t* framesEncoded) = 0;
  virtual bool GetVideoDecoderStats(double* framerateMean,
                                    double* framerateStdDev,
                                    double* bitrateMean,
                                    double* bitrateStdDev,
                                    uint32_t* discardedPackets,
                                    uint32_t* framesDecoded) = 0;
  virtual bool GetAVStats(int32_t* jitterBufferDelayMs,
                          int32_t* playoutBufferDelayMs,
                          int32_t* avSyncOffsetMs) = 0;
  virtual bool GetRTPStats(unsigned int* jitterMs,
                           unsigned int* cumulativeLost) = 0;
  virtual bool GetRTCPReceiverReport(DOMHighResTimeStamp* timestamp,
                                     uint32_t* jitterMs,
                                     uint32_t* packetsReceived,
                                     uint64_t* bytesReceived,
                                     uint32_t* cumulativeLost,
                                     int32_t* rttMs) = 0;
  virtual bool GetRTCPSenderReport(DOMHighResTimeStamp* timestamp,
                                   unsigned int* packetsSent,
                                   uint64_t* bytesSent) = 0;

  virtual uint64_t CodecPluginID() = 0;

  virtual void SetPCHandle(const std::string& aPCHandle) = 0;

  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(MediaSessionConduit)

};

// Abstract base classes for external encoder/decoder.
class CodecPluginID
{
public:
  virtual ~CodecPluginID() {}

  virtual uint64_t PluginID() const = 0;
};

class VideoEncoder : public CodecPluginID
{
public:
  virtual ~VideoEncoder() {}
};

class VideoDecoder : public CodecPluginID
{
public:
  virtual ~VideoDecoder() {}
};

/**
 * MediaSessionConduit for video
 * Refer to the comments on MediaSessionConduit above for overall
 * information
 */
class VideoSessionConduit : public MediaSessionConduit
{
public:
  /**
   * Factory function to create and initialize a Video Conduit Session
   * @param  webrtc::Call instance shared by paired audio and video
   *         media conduits
   * @result Concrete VideoSessionConduitObject or nullptr in the case
   *         of failure
   */
  static RefPtr<VideoSessionConduit> Create(RefPtr<WebRtcCallWrapper> aCall);

  enum FrameRequestType
  {
    FrameRequestNone,
    FrameRequestFir,
    FrameRequestPli,
    FrameRequestUnknown
  };

  VideoSessionConduit() : mFrameRequestMethod(FrameRequestNone),
                          mUsingNackBasic(false),
                          mUsingTmmbr(false),
                          mUsingFEC(false) {}

  virtual ~VideoSessionConduit() {}

  Type type() const override { return VIDEO; }

  void
  SetLocalRTPExtensions(bool aIsSend,
                        const RtpExtList& extensions) override = 0;

  RtpExtList GetLocalRTPExtensions(bool aIsSend) const override = 0;

  /**
   * Function to attach Renderer end-point of the Media-Video conduit.
   * @param aRenderer : Reference to the concrete Video renderer implementation
   * Note: Multiple invocations of this API shall remove an existing renderer
   * and attaches the new to the Conduit.
   */
  virtual MediaConduitErrorCode AttachRenderer(RefPtr<mozilla::VideoRenderer> aRenderer) = 0;
  virtual void DetachRenderer() = 0;

  bool SetRemoteSSRC(unsigned int ssrc) override = 0;

  /**
   * Function to deliver a capture video frame for encoding and transport
   * @param video_frame: pointer to captured video-frame.
   * @param video_frame_length: size of the frame
   * @param width, height: dimensions of the frame
   * @param video_type: Type of the video frame - I420, RAW
   * @param captured_time: timestamp when the frame was captured.
   *                       if 0 timestamp is automatcally generated
   * NOTE: ConfigureSendMediaCodec() MUST be called before this function can be invoked
   *       This ensures the inserted video-frames can be transmitted by the conduit
   */
  virtual MediaConduitErrorCode SendVideoFrame(unsigned char* video_frame,
                                               unsigned int video_frame_length,
                                               unsigned short width,
                                               unsigned short height,
                                               VideoType video_type,
                                               uint64_t capture_time) = 0;
  virtual MediaConduitErrorCode SendVideoFrame(webrtc::VideoFrame& frame) = 0;

  virtual MediaConduitErrorCode ConfigureCodecMode(webrtc::VideoCodecMode) = 0;
  /**
   * Function to configure send codec for the video session
   * @param sendSessionConfig: CodecConfiguration
   * @result: On Success, the video engine is configured with passed in codec for send
   *          On failure, video engine transmit functionality is disabled.
   * NOTE: This API can be invoked multiple time. Invoking this API may involve restarting
   *        transmission sub-system on the engine
   *
   */
  virtual MediaConduitErrorCode ConfigureSendMediaCodec(const VideoCodecConfig* sendSessionConfig) = 0;

  /**
   * Function to configurelist of receive codecs for the video session
   * @param sendSessionConfig: CodecConfiguration
   * NOTE: This API can be invoked multiple time. Invoking this API may involve restarting
   *        reception sub-system on the engine
   *
   */
  virtual MediaConduitErrorCode ConfigureRecvMediaCodecs(
      const std::vector<VideoCodecConfig* >& recvCodecConfigList) = 0;

  virtual unsigned int SendingMaxFs() = 0;

  virtual unsigned int SendingMaxFr() = 0;

  /**
    * These methods allow unit tests to double-check that the
    * rtcp-fb settings are as expected.
    */
    FrameRequestType FrameRequestMethod() const {
      return mFrameRequestMethod;
    }

    bool UsingNackBasic() const {
      return mUsingNackBasic;
    }

    bool UsingTmmbr() const {
      return mUsingTmmbr;
    }

    bool UsingFEC() const {
      return mUsingFEC;
    }

   protected:
     /* RTCP feedback settings, for unit testing purposes */
     FrameRequestType mFrameRequestMethod;
     bool mUsingNackBasic;
     bool mUsingTmmbr;
     bool mUsingFEC;
};

/**
 * MediaSessionConduit for audio
 * Refer to the comments on MediaSessionConduit above for overall
 * information
 */
class AudioSessionConduit : public MediaSessionConduit
{
public:

 /**
   * Factory function to create and initialize an Audio Conduit Session
   * @param  webrtc::Call instance shared by paired audio and video
   *         media conduits
   * @result Concrete AudioSessionConduitObject or nullptr in the case
   *         of failure
   */
  static RefPtr<AudioSessionConduit> Create();

  virtual ~AudioSessionConduit() {}

  Type type() const override { return AUDIO; }

  void
  SetLocalRTPExtensions(bool aIsSend,
                        const RtpExtList& extensions) override {};

  RtpExtList
  GetLocalRTPExtensions(bool aIsSend) const override {return RtpExtList();}
  /**
   * Function to deliver externally captured audio sample for encoding and transport
   * @param audioData [in]: Pointer to array containing a frame of audio
   * @param lengthSamples [in]: Length of audio frame in samples in multiple of 10 milliseconds
  *                             Ex: Frame length is 160, 320, 440 for 16, 32, 44 kHz sampling rates
                                    respectively.
                                    audioData[] is lengthSamples in size
                                    say, for 16kz sampling rate, audioData[] should contain 160
                                    samples of 16-bits each for a 10m audio frame.
   * @param samplingFreqHz [in]: Frequency/rate of the sampling in Hz ( 16000, 32000 ...)
   * @param capture_delay [in]:  Approx Delay from recording until it is delivered to VoiceEngine
                                 in milliseconds.
   * NOTE: ConfigureSendMediaCodec() SHOULD be called before this function can be invoked
   *       This ensures the inserted audio-samples can be transmitted by the conduit
   *
   */
  virtual MediaConduitErrorCode SendAudioFrame(const int16_t audioData[],
                                               int32_t lengthSamples,
                                               int32_t samplingFreqHz,
                                               uint32_t channels,
                                               int32_t capture_delay) = 0;

  /**
   * Function to grab a decoded audio-sample from the media engine for rendering
   * / playoutof length 10 milliseconds.
   *
   * @param speechData [in]: Pointer to a array to which a 10ms frame of audio will be copied
   * @param samplingFreqHz [in]: Frequency of the sampling for playback in Hertz (16000, 32000,..)
   * @param capture_delay [in]: Estimated Time between reading of the samples to rendering/playback
   * @param lengthSamples [out]: Will contain length of the audio frame in samples at return.
                                 Ex: A value of 160 implies 160 samples each of 16-bits was copied
                                     into speechData
   * NOTE: This function should be invoked every 10 milliseconds for the best
   *          peformance
   * NOTE: ConfigureRecvMediaCodec() SHOULD be called before this function can be invoked
   *       This ensures the decoded samples are ready for reading.
   *
   */
  virtual MediaConduitErrorCode GetAudioFrame(int16_t speechData[],
                                              int32_t samplingFreqHz,
                                              int32_t capture_delay,
                                              int& lengthSamples) = 0;

   /**
    * Function to configure send codec for the audio session
    * @param sendSessionConfig: CodecConfiguration
    * NOTE: See VideoConduit for more information
    */

  virtual MediaConduitErrorCode ConfigureSendMediaCodec(const AudioCodecConfig* sendCodecConfig) = 0;

   /**
    * Function to configure list of receive codecs for the audio session
    * @param sendSessionConfig: CodecConfiguration
    * NOTE: See VideoConduit for more information
    */
  virtual MediaConduitErrorCode ConfigureRecvMediaCodecs(
                                const std::vector<AudioCodecConfig* >& recvCodecConfigList) = 0;
   /**
    * Function to enable the audio level extension
    * @param aEnabled: enable extension
    * @param aId: the RTP extension header ID to use
    * @param aDirectionIsSend: indicates whether to set the extension on the
    *                          sender or the receiver side
    * returns an error if the extension could not be set
    */
  virtual MediaConduitErrorCode
  EnableAudioLevelExtension(bool aEnabled,
                            uint8_t aId,
                            bool aDirectionIsSend) = 0;
  virtual MediaConduitErrorCode
  EnableMIDExtension(bool enabled, uint8_t id) = 0;

  virtual bool SetDtmfPayloadType(unsigned char type, int freq) = 0;

  virtual bool InsertDTMFTone(int channel, int eventCode, bool outOfBand,
                              int lengthMs, int attenuationDb) = 0;

  virtual void GetRtpSources(const int64_t aTimeNow,
                             nsTArray<dom::RTCRtpSourceEntry>& outSources) = 0;

};
}
#endif