鸟语天空

WebRTC For Unity

post by:追风剑情 2022-2-7 12:14

WebRTC官网
 WebRTC for Unity
Unity WebRTC Package
webrtc-uwp
WebRTC-universal-samples
interview-uwp
uwp-webrtcortc
HoloLens2-Unity-ResearchModeStreamer
HoloLens2ForCV
Unity Render Streaming Package

WebRTC 带宽占用
分辨率	需要的带宽
4K(3840×2160)	15-20 Mbps
1080p(1920x1080)	4-8 Mbps
720p(1280x720)	1-4 Mbps
VGA(640x480)	500K-1Mbps
QVGA(320x320)	300K-500K

Unity2019.4 安装
[官方文档] 安装

Unity2020.3 安装

com.unity.webrtc@3.0.0-pre.7

Unity 版本支持

建议使用最新的长期支持（LTS）版本的Unity。

WebRTC 3.0.0 兼容的 Unity 版本如下：

Unity 2020.3
Unity 2021.3
Unity 2022.3
Unity 2023.1

支持的平台

Windows 10 (x64 only)
Linux (Ubuntu 16.04, 18.04, 20.04)
macOS (Intel and Apple Slicon)
iOS
Android (ARM64 only. ARMv7 is not supported)

补充说明

请注意，以下为不受支持的平台。

Windows UWP platform is not supported
Building for iOS Simulator is not supported
WebGL platform is not supported

Player Settings
Build On Android
Scripting backend - IL2CPP
Target Architectures - ARM64 (Do disable ARMv7)
Internet Access - Require
Graphics API - OpenGLES3
Optimized Frame Pacing - disable

Build on iOS

您必须在从Unity导出的Xcode工程中关闭 bitcode 选项。

On the Xcode Build Settings tab, in the Build Options group, set Enable Bitcode to No

Auto Graphics API 默认使用 Vulkan，WebRTC 不支持 Vulkan。

打开统计界面 [Window]->Analysis->WebRTC Stats

源码分析

RTCSdpType 枚举值说明
枚举值	说明
offer	SDP提案
pranswer	SDP应答，但不是最终的应答
answer	SDP最终应答
rollback	回滚，取消当前的SDP协商，回退到上一个稳定状态

无论是Offer端还是Answer端都必须调用下面这句代码，才会开始推流。
StartCoroutine(WebRTC.Update());

从摄像机创建媒体流

public static MediaStream CaptureStream(this Camera cam, int width, int height,
            RenderTextureDepth depth = RenderTextureDepth.Depth24)
{
	var stream = new MediaStream();
	var track = cam.CaptureStreamTrack(width, height, depth);
	stream.AddTrack(track);
	return stream;
}

添加ICE候选地址

private void AddIceCandidate(string candidate, string sdpMid, int sdpMLineIndex)
{
	RTCIceCandidateInit iceInit = new RTCIceCandidateInit();
	iceInit.candidate = candidate;
	iceInit.sdpMid = sdpMid;
	iceInit.sdpMLineIndex = sdpMLineIndex;

	RTCIceCandidate ice = new RTCIceCandidate(iceInit);
	pc.AddIceCandidate(ice);
}

创建PeerConnection

private void CreatePeerConnection()
{
	if (pc != null)
		Dispose();
	pc = new RTCPeerConnection();
	pc.OnNegotiationNeeded = OnNegotiationNeeded;
	pc.OnIceCandidate = OnIceCandidate;
	pc.OnTrack = e =>
	{
		receiveStream?.AddTrack(e.Track);
	};
}

创建视频流

//创建视频流轨道 (方法一)
public void CreateVideoStreamTrack(Camera cam, int width=640, int height=480)
{
	//调用Camera的扩展方法CaptureStream()
	VideoStreamTrack track = cam.CaptureStream(width, height);
	return track;
}

//创建视频流轨道 (方法二)
public void CreateVideoStreamTrack(Camera cam, int width=640, int height=480)
{
	//自己创建渲染源(Texture)
	int depthValue = (int)RenderTextureDepth.Depth24;
	var format = WebRTC.GetSupportedRenderTextureFormat(SystemInfo.graphicsDeviceType);
	var rt = new UnityEngine.RenderTexture(width, height, depthValue, format);
	rt.Create();
	cam.targetTexture = rt;
	VideoStreamTrack track = new VideoStreamTrack(rt, null);
	return track;
}

创建接收流

private void CreateReceiveStream()
{
	receiveStream = new MediaStream();
	receiveStream.OnAddTrack = e =>
	{
		if (e.Track is VideoStreamTrack track)
		{
			track.OnVideoReceived += tex =>
			{
				//显示视频画面
				if (receiveImage == null)
					return;
				receiveImage.texture = tex;
				receiveImage.color = Color.white;
			};
		}
	};
}

发送Offer

private void OnNegotiationNeeded()
{
	//可协商事件触发后，再发送Offer
	StartCoroutine(SendOffer());
}

IEnumerator SendOffer()
{
	var op = pc.CreateOffer();
	//确保Offer异步创建完成
	yield return op;

	if (op.IsError)
	{
		Debug.LogError("Create offer failed!");
		yield break;
	}

	RTCSessionDescription desc = op.Desc;
	var op1 = pc.SetLocalDescription(ref desc);
	if (op1.IsError)
	{
		Debug.LogError("SetLocalDescription Error");
		yield break;
	}

	Debug.LogFormat("send offer sdp={0}", op.Desc.sdp);
	//TODO:: 将sdp信息发送到对端
}

发送Answer

IEnumerator SendAnswer()
{
	var op = pc.CreateAnswer();
	//确保Answer异步创建完成
	yield return op;

	if (op.IsError)
	{
		Debug.LogError("Create answer failed!");
		yield break;
	}

	RTCSessionDescription desc = op.Desc;
	var op1 = pc.SetLocalDescription(ref desc);
	if (op1.IsError)
	{
		Debug.LogError("SetLocalDescription Error");
		yield break;
	}
	Debug.Log("SetLocalDescription complete");

	Debug.LogFormat("send answer sdp={0}", op.Desc.sdp);
	//TODO:: 将sdp信息发送到对端
}

释放

public void Dispose()
{
	//释放连接
	if (pc != null)
	{
		pc.Dispose();
		pc = null;
	}
	//释放视频流
	if (videoStream != null)
	{
		foreach (var track in videoStream.GetTracks())
		{
			track.Dispose();
		}
		videoStream.Dispose();
		videoStream = null;
	}
	//释放接收流
	if (receiveStream != null)
	{
		foreach (var track in receiveStream.GetTracks())
		{
			track.Dispose();
		}
		receiveStream.Dispose();
		receiveStream = null;
	}
	//释放发送器
	if (pcSenders != null)
	{
		pcSenders.ForEach(x => x.Dispose());
		pcSenders = null;
	}
}

单向传输

一方设置为 a=sendonly，另一方设置为 a=recvonly，才能实现单向传输。

foreach (var transceiver in peerConnection.GetTransceivers())
{
	transceiver.Direction = RTCRtpTransceiverDirection.SendOnly;
	//transceiver.Direction = RTCRtpTransceiverDirection.RecvOnly;
}

辅助类

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
using Unity.WebRTC;
using UNetwork;
using UNetwork.Protocol;
/// <summary>
/// WebRTC P2P连接
/// </summary>
public class PeerConnectionBehaviour : MonoBehaviour
{
    [SerializeField]
    private Camera videoCamera;
    private MediaStream videoStream;
    private RTCPeerConnection pc;
    private List<RTCRtpSender> pcSenders;
    private static bool videoUpdateStarted;
    private MediaStream receiveStream;
    [SerializeField]
    private RawImage receiveImage;
    [SerializeField]
    private int videoWidth = 1280;
    [SerializeField]
    private int videoHeight = 720;
    [SerializeField]
    private string myUserId; //自己的用户ID
    [SerializeField]
    private string toUserId; //对方的用户ID
    private RTCSdpType? sdpType = null;

    void Start()
    {
        //每帧更新视频画面
        if (!videoUpdateStarted)
        {
            StartCoroutine(WebRTC.Update());
            videoUpdateStarted = true;
        }

        UnityNetwork.Instance.OnWebRtcSdpEvent += OnWebRtcSdp;
        UnityNetwork.Instance.OnWebRtcIceEvent += OnWebRtcIce;
    }

    private void OnDestroy()
    {
        UnityNetwork.Instance.OnWebRtcSdpEvent -= OnWebRtcSdp;
        UnityNetwork.Instance.OnWebRtcIceEvent -= OnWebRtcIce;
        Dispose();
    }

    // 收到 SDP
    private void OnWebRtcSdp(WebRtcSDP des)
    {
        if (des.fromUserId != toUserId)
            return;

        RTCSessionDescription desc = new RTCSessionDescription();
        desc.type = (RTCSdpType)des.sdpType;
        desc.sdp = des.sdp;

        switch (desc.type)
        {
            case RTCSdpType.Offer:
                Debug.LogFormat("receive offer sdp\n{0}", desc.sdp);
                sdpType = RTCSdpType.Answer;
                CreatePeerConnection();
                //CreateVideoStream();
                CreateReceiveStream();
                pc.SetRemoteDescription(ref desc);
                StartCoroutine(SendAnswer());
                break;
            case RTCSdpType.Answer:
                Debug.LogFormat("receive answer sdp\n{0}", desc.sdp);
                pc.SetRemoteDescription(ref desc);
                break;
        }
    }

    // 收到 ICE
    private void OnWebRtcIce(WebRtcICE ice)
    {
        if (ice.fromUserId != toUserId)
            return;

        Debug.LogFormat("receive ice\ncandidate={0}\nsdpMid={1}\nsdpMLineIndex={2}", 
            ice.candidate, ice.sdpMid, ice.sdpMLineIndex);
        RTCIceCandidateInit iceInit = new RTCIceCandidateInit();
        iceInit.candidate = ice.candidate;
        iceInit.sdpMid = ice.sdpMid;
        iceInit.sdpMLineIndex = ice.sdpMLineIndex;

        RTCIceCandidate _ice = new RTCIceCandidate(iceInit);
        pc.AddIceCandidate(_ice);
    }

    private void AddIceCandidate(string candidate, string sdpMid, int sdpMLineIndex)
    {
        RTCIceCandidateInit iceInit = new RTCIceCandidateInit();
        iceInit.candidate = candidate;
        iceInit.sdpMid = sdpMid;
        iceInit.sdpMLineIndex = sdpMLineIndex;

        RTCIceCandidate ice = new RTCIceCandidate(iceInit);
        pc.AddIceCandidate(ice);
    }

    // 可以进行媒体协商时触发，向媒体流中添加轨道后会自动触发。
    private void OnNegotiationNeeded()
    {
        Debug.Log("OnNegotiationNeeded");
        if (sdpType == RTCSdpType.Offer)
            StartCoroutine(SendOffer());
    }

    // 需要同步ICE信息时触发
    private void OnIceCandidate(RTCIceCandidate candidate)
    {
        Debug.Log("send ice");
        NetworkHelper.SendWebRtcICE(myUserId, toUserId, candidate.Candidate, candidate.SdpMid, (int)candidate.SdpMLineIndex);
    }

    // 创建连接
    private void CreatePeerConnection()
    {
        if (pc != null)
            Dispose();
        pc = new RTCPeerConnection();
        pc.OnNegotiationNeeded = OnNegotiationNeeded;
        pc.OnIceCandidate = OnIceCandidate;
        pc.OnConnectionStateChange = (state) => {
            Debug.LogFormat("peer connection state change. {0}", state);
        };
        pc.OnTrack = e =>
        {
            receiveStream?.AddTrack(e.Track);
        };
        pcSenders = new List<RTCRtpSender>();
    }

    // 创建视频流
    private void CreateVideoStream()
    {
        if (videoCamera != null)
        {
            videoStream = videoCamera.CaptureStream(videoWidth, videoHeight);
            foreach (var track in videoStream.GetTracks())
            {
                pcSenders.Add(pc.AddTrack(track, videoStream));
            }
        }
    }

    // 创建接收流
    private void CreateReceiveStream()
    {
        receiveStream = new MediaStream();
        receiveStream.OnAddTrack = e =>
        {
            if (e.Track is VideoStreamTrack track)
            {
                track.OnVideoReceived += tex =>
                {
                    //显示视频画面
                    if (receiveImage == null)
                        return;
                    receiveImage.texture = tex;
                    receiveImage.color = Color.white;
                };
            }
        };
    }

    // 发送Offer
    IEnumerator SendOffer()
    {
        var op = pc.CreateOffer();
        //确保Offer异步创建完成
        yield return op;

        if (op.IsError)
        {
            Debug.LogError("Create offer failed!");
            yield break;
        }

        RTCSessionDescription desc = op.Desc;
        var op1 = pc.SetLocalDescription(ref desc);
        if (op1.IsError)
        {
            Debug.LogError("SetLocalDescription Error");
            yield break;
        }
        Debug.Log("SetLocalDescription complete");

        Debug.LogFormat("send offer sdp={0}", op.Desc.sdp);
        string sdp = op.Desc.sdp;
        NetworkHelper.SendWebRtcSDP(myUserId, toUserId, (int)RTCSdpType.Offer, sdp);
    }

    // 发送Answer
    IEnumerator SendAnswer()
    {
        var op = pc.CreateAnswer();
        //确保Answer异步创建完成
        yield return op;

        if (op.IsError)
        {
            Debug.LogError("Create answer failed!");
            yield break;
        }

        RTCSessionDescription desc = op.Desc;
        var op1 = pc.SetLocalDescription(ref desc);
        if (op1.IsError)
        {
            Debug.LogError("SetLocalDescription Error");
            yield break;
        }
        Debug.Log("SetLocalDescription complete");

        Debug.LogFormat("send answer sdp={0}", op.Desc.sdp);
        string sdp = op.Desc.sdp;
        NetworkHelper.SendWebRtcSDP(myUserId, toUserId, (int)RTCSdpType.Answer, sdp);
    }

    // 启动WebRTC通讯
    public void StartRTC()
    {
        sdpType = RTCSdpType.Offer;
        CreatePeerConnection();
        CreateVideoStream();
        //CreateReceiveStream();
    }

    // 释放
    public void Dispose()
    {
        //释放连接
        if (pc != null)
        {
            pc.Dispose();
            pc = null;
        }
        //释放视频流
        if (videoStream != null)
        {
            foreach (var track in videoStream.GetTracks())
            {
                track.Dispose();
            }
            videoStream.Dispose();
            videoStream = null;
        }
        //释放接收流
        if (receiveStream != null)
        {
            foreach (var track in receiveStream.GetTracks())
            {
                track.Dispose();
            }
            receiveStream.Dispose();
            receiveStream = null;
        }
        //释放发送器
        if (pcSenders != null)
        {
            pcSenders.ForEach(x => x.Dispose());
            pcSenders = null;
        }
    }

    private void OnApplicationQuit()
    {
        Dispose();
    }
}

评论：

发表评论：