sfu-ws example with trickle ICE not able to send rtp out to browser
sampleref opened this issue · comments
I tried to modify the given https://github.com/pion/example-webrtc-applications/tree/master/sfu-ws example to support trickleICE as below,
- Changes for trickle enable
m := webrtc.MediaEngine{}
s := webrtc.SettingEngine{}
s.SetTrickle(true)
// Setup the codecs you want to use.
m.RegisterCodec(webrtc.NewRTPVP8Codec(webrtc.DefaultPayloadTypeVP8, 90000))
m.RegisterCodec(webrtc.NewRTPOpusCodec(webrtc.DefaultPayloadTypeOpus, 48000))
// Create the API object with the MediaEngine
api := webrtc.NewAPI(webrtc.WithMediaEngine(m), webrtc.WithSettingEngine(s))
- Below WsConn Method is similar to Room with Trickle ICE support for a single pubReceiverC and subSenderC
func WsConn(w http.ResponseWriter, r *http.Request) {
// Websocket client
c, err := upgrader.Upgrade(w, r, nil)
lutil.CheckError(err)
// Make sure we close the connection when the function returns
defer c.Close()
for {
var msg JsonMsg
// Read in a new message as JSON and map it to a Message object
err := c.ReadJSON(&msg)
if err != nil {
log.Printf("error: %v", err)
delete(ConnMap, c)
break
}
if msg.PeerType == "pub" {
if msg.Sdp != "" {
// Create a new RTCPeerConnection
pubReceiverC, err = Api.NewPeerConnection(peerConnectionConfig)
lutil.CheckError(err)
_, err = pubReceiverC.AddTransceiverFromKind(webrtc.RTPCodecTypeAudio)
lutil.CheckError(err)
_, err = pubReceiverC.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo)
lutil.CheckError(err)
pubReceiverC.OnTrack(func(remoteTrack *webrtc.Track, receiver *webrtc.RTPReceiver) {
if remoteTrack.PayloadType() == webrtc.DefaultPayloadTypeVP8 || remoteTrack.PayloadType() == webrtc.DefaultPayloadTypeVP9 || remoteTrack.PayloadType() == webrtc.DefaultPayloadTypeH264 {
// Create a local video track, all our SFU clients will be fed via this track
var err error
videoTrackLockC.Lock()
videoTrackC, err = pubReceiverC.NewTrack(remoteTrack.PayloadType(), remoteTrack.SSRC(), "video", "pion")
videoTrackLockC.Unlock()
lutil.CheckError(err)
// Send a PLI on an interval so that the publisher is pushing a keyframe every rtcpPLIInterval
go func() {
ticker := time.NewTicker(rtcpPLIInterval)
for range ticker.C {
lutil.CheckError(pubReceiverC.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: videoTrackC.SSRC()}}))
}
}()
rtpBuf := make([]byte, 1400)
for {
i, err := remoteTrack.Read(rtpBuf)
lutil.CheckError(err)
videoTrackLockC.RLock()
_, err = videoTrackC.Write(rtpBuf[:i])
videoTrackLockC.RUnlock()
if err != io.ErrClosedPipe {
lutil.CheckError(err)
}
}
} else {
// Create a local audio track, all our SFU clients will be fed via this track
var err error
audioTrackLockC.Lock()
audioTrackC, err = pubReceiverC.NewTrack(remoteTrack.PayloadType(), remoteTrack.SSRC(), "audio", "pion")
audioTrackLockC.Unlock()
lutil.CheckError(err)
rtpBuf := make([]byte, 1400)
for {
i, err := remoteTrack.Read(rtpBuf)
lutil.CheckError(err)
audioTrackLockC.RLock()
_, err = audioTrackC.Write(rtpBuf[:i])
audioTrackLockC.RUnlock()
if err != io.ErrClosedPipe {
lutil.CheckError(err)
}
}
}
})
// Set the remote SessionDescription
lutil.CheckError(pubReceiverC.SetRemoteDescription(
webrtc.SessionDescription{
SDP: string(msg.Sdp),
Type: webrtc.SDPTypeOffer,
}))
// Create answer
answer, err := pubReceiverC.CreateAnswer(nil)
lutil.CheckError(err)
// Sets the LocalDescription, and starts our UDP listeners
lutil.CheckError(pubReceiverC.SetLocalDescription(answer))
// Send server sdp to publisher
var sdpMsg = JsonMsg{PeerId: msg.PeerId, Sdp: answer.SDP, PeerType: msg.PeerType, IceCandidate: struct {
Candidate string
SDPMid string
SDPMLineIndex uint16
}{Candidate: "", SDPMid: "", SDPMLineIndex: 0}}
lutil.CheckError(c.WriteJSON(sdpMsg))
// Register incoming channel
pubReceiverC.OnDataChannel(func(d *webrtc.DataChannel) {
d.OnMessage(func(msg webrtc.DataChannelMessage) {
// Broadcast the data to subSenders
broadcastHub.broadcastChannel <- msg.Data
})
})
// Register for ice candidate generation
pubReceiverC.OnICECandidate(func(candidate *webrtc.ICECandidate) {
var iceMsg = JsonMsg{PeerId: msg.PeerId, Sdp: "", IceCandidate: struct {
Candidate string
SDPMid string
SDPMLineIndex uint16
}{Candidate: candidate.ToJSON().Candidate, SDPMid: *candidate.ToJSON().SDPMid, SDPMLineIndex: *candidate.ToJSON().SDPMLineIndex},
PeerType: msg.PeerType}
lutil.CheckError(c.WriteJSON(iceMsg))
})
} else if &msg.IceCandidate != nil && msg.IceCandidate.Candidate != "" {
var iceCandidate = webrtc.ICECandidateInit{Candidate: msg.IceCandidate.Candidate, SDPMid: &msg.IceCandidate.SDPMid,
SDPMLineIndex: &msg.IceCandidate.SDPMLineIndex, UsernameFragment: ""}
pubReceiverC.AddICECandidate(iceCandidate)
} else {
fmt.Println("PUB: None of matching msg sdp or ice")
}
} else {
if msg.Sdp != "" {
// Create a new RTCPeerConnection
subSenderC, err = Api.NewPeerConnection(peerConnectionConfig)
lutil.CheckError(err)
_, err = subSenderC.AddTransceiverFromKind(webrtc.RTPCodecTypeAudio)
lutil.CheckError(err)
_, err = subSenderC.AddTransceiverFromKind(webrtc.RTPCodecTypeVideo)
lutil.CheckError(err)
// Waiting for publisher track finish
for {
videoTrackLockC.RLock()
if videoTrackC == nil {
videoTrackLockC.RUnlock()
fmt.Println("Waiting for local video track to come up...")
time.Sleep(100 * time.Millisecond)
} else {
videoTrackLockC.RUnlock()
break
}
}
// Add local video track
videoTrackLockC.RLock()
_, err = subSenderC.AddTrack(videoTrackC)
videoTrackLockC.RUnlock()
lutil.CheckError(err)
// Add local audio track
audioTrackLockC.RLock()
_, err = subSenderC.AddTrack(audioTrackC)
audioTrackLockC.RUnlock()
lutil.CheckError(err)
// Set the remote SessionDescription
lutil.CheckError(subSenderC.SetRemoteDescription(
webrtc.SessionDescription{
SDP: string(msg.Sdp),
Type: webrtc.SDPTypeOffer,
}))
// Create answer
answer, err := subSenderC.CreateAnswer(nil)
lutil.CheckError(err)
// Sets the LocalDescription, and starts our UDP listeners
lutil.CheckError(subSenderC.SetLocalDescription(answer))
// Send server sdp to publisher
var sdpMsg = JsonMsg{PeerId: msg.PeerId, Sdp: answer.SDP, PeerType: msg.PeerType, IceCandidate: struct {
Candidate string
SDPMid string
SDPMLineIndex uint16
}{Candidate: "", SDPMid: "", SDPMLineIndex: 0}}
lutil.CheckError(c.WriteJSON(sdpMsg))
// Register incoming channel
subSenderC.OnDataChannel(func(d *webrtc.DataChannel) {
d.OnMessage(func(msg webrtc.DataChannelMessage) {
// Broadcast the data to subSenders
broadcastHub.broadcastChannel <- msg.Data
})
})
// Register for ice candidate generation
subSenderC.OnICECandidate(func(candidate *webrtc.ICECandidate) {
var iceMsg = JsonMsg{PeerId: msg.PeerId, Sdp: "", IceCandidate: struct {
Candidate string
SDPMid string
SDPMLineIndex uint16
}{Candidate: candidate.ToJSON().Candidate, SDPMid: *candidate.ToJSON().SDPMid, SDPMLineIndex: *candidate.ToJSON().SDPMLineIndex},
PeerType: msg.PeerType}
lutil.CheckError(c.WriteJSON(iceMsg))
})
} else if &msg.IceCandidate != nil && msg.IceCandidate.Candidate != "" {
var iceCandidate = webrtc.ICECandidateInit{Candidate: msg.IceCandidate.Candidate, SDPMid: &msg.IceCandidate.SDPMid,
SDPMLineIndex: &msg.IceCandidate.SDPMLineIndex, UsernameFragment: ""}
subSenderC.AddICECandidate(iceCandidate)
} else {
fmt.Println("SUB: None of matching msg sdp or ice")
}
}
}
}
- I have small JSON wrapper struct to handle JSON Message for SDP and ICE Messages as below, which is used in above function
type JsonMsg struct {
PeerId string
Sdp string
IceCandidate struct {
Candidate string
SDPMid string
SDPMLineIndex uint16
}
PeerType string
}
Rest all is same with a set of variables
videoTrackC *webrtc.Track
audioTrackC *webrtc.Track
videoTrackLockC = sync.RWMutex{}
audioTrackLockC = sync.RWMutex{}
pubReceiverC *webrtc.PeerConnection
subSenderC *webrtc.PeerConnection
The behavior seen is when tried from Chrome Browser with 2 instances one as publisher and other as subscriber, the default Room function works fine
But for above written WsConn function with Trickle ICE
- pubReceiverC ICE is connected and RTP packets are seen sent from browser and received at remoteTrack.Read(rtpBuf)
- subSenderC ICE is connected fine, Added with local tracks, is not sending any RTP, to browser
The only changes I did to given example is to modify websocket json a bit to support SDP and ICE with some extra info and apply the ICE messages as they arrive to respective peer connection
awesome work! I am going to migrate sfu-ws to /v3 this weekend. Really appreciate you starting early :)
Hey @sampleref
I am going to rewrite this entirely. The more time I spent with it the more I realized we needed massive changes. I will close this PR with a new sfu-ws example!
Hello @Sean-Der
Sure, Thanks for updating this example
I hope the master version which is updated recently would be using /v3 and can be referred
@sampleref when you get a chance tell me what you think! Trickle ICE is always on by default and tried to simplify as much as possible.
Sure I shall verify this at earliest and update here
Trickle ICE for which I actually started this discussion is working with /v3, Thanks for the support
Hello @Sean-Der
I have created a kind of push to talk app https://github.com/sampleref/gortpdemo/blob/master/ptt/ptthandler.go
where
-
Local tracks one audio and video is shared to all recipients on peer connection, example
https://github.com/sampleref/gortpdemo/blob/03ebf6ceff899ad17130a5c39fb235dbd680d87b/ptt/ptthandler.go#L87 -
Multiple chrome peers connect with offer and answer and stay connected, but rtp from them is not sent to anyone
-
On press of a key(floor) on being available, the RTP from that peer/track is pushed into local tracks in the above where RTP in turn flows to all of the recipients added with this local tracks
Given above scenario, what I observe is most of the attempts on writing RTP packets to local tracks which happen on and off from multiple peer connections( though only one at a time) three scenarios happen
- No RTP packets are pushed to Peers/Browsers from go code
- RTP packets are send out, and shown receiving in chrome://webrtc-internals but not played out
- Immediately on allowing at https://github.com/sampleref/gortpdemo/blob/03ebf6ceff899ad17130a5c39fb235dbd680d87b/ptt/ptthandler.go#L158 No RTP is sent, but after some random duration it works
Above 3 scenarios, not sure if any buffering is affecting this behavior here
I am using /v3 and not exactly sure if my use case can be addressed in above way
Please suggest if there is some better approach doing this or if anything missing
@sampleref That's not good! So here are the things that stand out to me.
- An
apiobject must not be shared amount PeerConnection, that could be causing problems! - TrackLocal is thread safe, you don't need to hold the lock when calling Write
- You have a timing issue. The browser sees a giant gap in audio and is trying to get the lost samples. I think what you may need to always send packets, but instead send an empty Payload.
I will pull down the repo and give it a shot. Asking in https://pion.ly/slack might be better! GitHub issues are async (and I am the only one that really responds to them) but you get the whole community in Slack
I could address first 2 items mentioned, in https://github.com/sampleref/gortpdemo/blob/master/ptt/ptthandler.go
Still unable to listen audio on receiver track on browser, but shown as packets/bytes received in chrome://webrtc-internals
-
An
apiobject must not be shared amount PeerConnection, that could be causing problems! - Done, using new api for each peer connection -
TrackLocal is thread safe, you don't need to hold the lock when calling Write - Done
-
You have a timing issue. The browser sees a giant gap in audio and is trying to get the lost samples. I think what you may need to always send packets, but instead send an empty Payload.
-> This one I am unable to figure out how to debug/validate, Can you please elaborate how to identify that this gap in audio is happening and how to approach - "always send packets, but instead send an empty Payload"