How to create ICE candidates?

[Harsh4999]

So I have a script which is able to accept the offer send answer and also gather candidates which are sent in connection process but I need to send back ICE candidates in order to complete the connection process but I dont know how to do so.

I wanted to know if it is possible to create ICE candidates with AIORTC at all?
Current status of my script

Connected to the signaling server
TRACK RECEIVED <aiortc.rtcrtpreceiver.RemoteStreamTrack object at 0x10b14b410>
TRACK KIND audio
TRACK RECEIVED <aiortc.rtcrtpreceiver.RemoteStreamTrack object at 0x10b16d550>
TRACK KIND video
Signaling state change: stable
ICE gathering complete
Signaling state change: stable
ICE gathering complete
ICE gathering state changed to gathering
ICE gathering state changed to complete
All ICE candidates have been gathered.
ICE connection state is checking
Connection state change: connecting

After I receive some offer my script just gets stuck on this state and never moves ahead. I think the only missing thing in here is sending back of ICE candidates because I am seeing getting answer back on the other end. I am creating offer from JS script from browser and it is able to also send ICE candidates automatically but for some reason I could not find any function which can help me create ICE candidates in aiortc.

Do let me know if its possible or if I am doing something wrong in here for connection process.

Adding my python script for reference:

import asyncio
import json
import websockets
from aiortc import RTCIceCandidate, RTCPeerConnection, RTCSessionDescription, RTCIceServer, RTCConfiguration

class WebRTCClient:
    def __init__(self, uri):
        self.uri = uri
        ice_servers = [
            RTCIceServer(urls="stun:stun.l.google.com:19302")  # Google's public STUN server
        ]
        self.pc = RTCPeerConnection(RTCConfiguration(iceServers=ice_servers))
        self.ice_candidates = []

    async def connect_to_websocket(self):
        self.websocket = await websockets.connect(self.uri)
        print("Connected to the signaling server")
        asyncio.create_task(self.receive_messages())

        @self.pc.on("track")
        async def on_track(track):
            print('TRACK RECEIVED', track)
            print('TRACK KIND', track.kind)

        @self.pc.on("signalingstatechange")
        async def on_signalingstatechange():
            print('Signaling state change:', self.pc.signalingState)
            if self.pc.signalingState == 'stable':
                print('ICE gathering complete')
                # Log all gathered candidates
                for candidate in self.ice_candidates:
                    print('Gathered candidate:', candidate)

        @self.pc.on('iceconnectionstatechange')
        async def on_iceconnectionstatechange():
            print("ICE connection state is", self.pc.iceConnectionState)
            if self.pc.iceConnectionState == "failed":
                print("ICE Connection has failed, attempting to restart ICE")
                await self.pc.restartIce()

        @self.pc.on('connectionstatechange')
        async def on_connectionstatechange():
            print('Connection state change:', self.pc.connectionState)
            if self.pc.connectionState == 'connected':
                print('Peers successfully connected')


        @self.pc.on('icegatheringstatechange')
        async def on_icegatheringstatechange():
            print('ICE gathering state changed to', self.pc.iceGatheringState)
            if self.pc.iceGatheringState == 'complete':
                print('All ICE candidates have been gathered.')
                # Log all gathered candidates
                for candidate in self.ice_candidates:
                    print('Gathered candidate:', candidate)

    async def create_offer(self):
        await self.setup_media()  # Setup media before creating an offer
        offer = await self.pc.createOffer()
        await self.pc.setLocalDescription(offer)
        await self.send_message({'event': 'offer', 'data': {'type': offer.type, 'sdp': offer.sdp}})

    async def setup_media(self):
        self.pc.addTransceiver('audio')
        self.pc.addTransceiver('video')

    async def receive_messages(self):
        async for message in self.websocket:
            data = json.loads(message)
            await self.handle_message(data)

    async def handle_message(self, message):
        event = message.get('event')
        data = message.get('data')
        if event == 'offer':
            await self.handle_offer(data)
        elif event == 'candidate':
            await self.handle_candidate(data)
        elif event == 'answer':
            await self.handle_answer(data)

    async def handle_offer(self, offer):
        await self.pc.setRemoteDescription(RTCSessionDescription(sdp=offer['sdp'], type=offer['type']))
        answer = await self.pc.createAnswer()
        await self.pc.setLocalDescription(answer)
        await self.send_message({'event': 'answer', 'data': {'type': answer.type, 'sdp': answer.sdp}})

    async def handle_candidate(self, candidate):
        # print('Received ICE candidate:', candidate)
        ip = candidate['candidate'].split(' ')[4]
        port = candidate['candidate'].split(' ')[5]
        protocol = candidate['candidate'].split(' ')[7]
        priority = candidate['candidate'].split(' ')[3]
        foundation = candidate['candidate'].split(' ')[0]
        component = candidate['candidate'].split(' ')[1]
        type = candidate['candidate'].split(' ')[7]
        rtc_candidate = RTCIceCandidate(
            ip=ip,
            port=port,
            protocol=protocol,
            priority=priority,
            foundation=foundation,
            component=component,
            type=type,
            sdpMid=candidate['sdpMid'],
            sdpMLineIndex=candidate['sdpMLineIndex']
        )
        await self.pc.addIceCandidate(rtc_candidate)
        self.ice_candidates.append(rtc_candidate)

    async def handle_answer(self, answer):
        await self.pc.setRemoteDescription(RTCSessionDescription(sdp=answer['sdp'], type=answer['type']))

    async def send_message(self, message):
        await self.websocket.send(json.dumps(message))

    async def close(self):
        await self.pc.close()
        await self.websocket.close()

async def main():
    client = WebRTCClient('ws://localhost:8080/socket')
    await client.connect_to_websocket()
    # await client.create_offer()  # Create an offer after connecting to the websocket
    await asyncio.sleep(3600)  # Keep the session alive for debugging
    await client.close()

if __name__ == '__main__':
    asyncio.run(main())

[rawlines]

Hello, i don't know if you still need help with this or not. But i think i must reply in order to help other people facing this same issue...

It was making me crazy because i'm implementing a WebRTC Application where a camera attached to a raspberry must transmit in realtime the video to a webpage. The backend of the camera uses python with aiortc and the frontend is a regular webrtc javascript application.

The camera and the frontend might be executed in complete different networks, so a TURN server is needed. I was using azure IotHUB as signaling service, and my own deployment of coturn server in Azure VM. All was working fine within the same networks but things started to go crazy when try to stablish a connection from different networks....

The coturn server was working OK and all the tests i made with him works good, and also the frontend application works well and the errors reported by chrome://webrtc shows like if the remote peer wasn't establishing/accepting or whatever the connection. After going crazy and searching a lot i faced with this issue, i've tested your code, specially the handle_candidate method but it looks like aiortc is doing nothing with the candidates added after the offer, looking a bit more i found some posts which says says that the underlying library of aiortc doesn't use ICE Trickle, so my final test was to disable ICE server signaling through my signaling server and send all the ICE candidates with the offer to the remote camera. and finally with this, it worked. aiortc uses this candidates and the connection could be stablished through turn server.

The idea is to concatenate the candidates into the offer SDP and disable all kind of ICE Trickling with aiortc.
This is how i'm creating and sending the offer:

    const iceConfig: RTCConfiguration = {
        iceServers: [
            {
                urls: 'stun:my.stun:555',
            },
            {
                urls: 'turn:my.turn:556?transport=udp',
                username: 'my-user',
                credential: 'my-password',
            },
        ]
    };
    peerConnection = new RTCPeerConnection(iceConfig);
    console.log('Created peer connection:', peerConnection);

    const iceCandidates: RTCIceCandidateInit[] = [];

    const iceGatheringComplete = new Promise((resolve) => {
        peerConnection!.onicegatheringstatechange = (event) => {
            console.log('icegatheringstatechange -> ', peerConnection?.iceGatheringState);

            if (peerConnection!.iceGatheringState === 'complete') {
                console.log('iceCandidates -> ', iceCandidates);
                resolve(null);
            }
        };
    });

    peerConnection.oniceconnectionstatechange = (event) => {
        console.log('iceconnectionstatechange -> ', peerConnection?.iceConnectionState);
    };

    peerConnection.onsignalingstatechange = (event) => {
        console.log('signalingstatechange -> ', peerConnection?.signalingState);
    };

    peerConnection.onicecandidateerror = (event) => {
        console.error('icecandidateerror -> ', peerConnection?.iceConnectionState);
    };

    peerConnection.onicecandidate = (event) => {
        if (event.candidate) {
            console.log('icecandidate -> ', event.candidate);
            iceCandidates.push(event.candidate.toJSON());
        }
    };

    peerConnection.ontrack = (event) => {
        console.log('Got remote track:', event);
        const [remoteStream] = event.streams;
        remoteVideo.srcObject = remoteStream;
        remoteVideo.play();
    };

    peerConnection.addTransceiver('video', { direction: 'recvonly' });

    const offer = await peerConnection.createOffer();

    await peerConnection.setLocalDescription(offer);
    console.log('Created offer:', offer);

    // Send ICE Candidates with the offer (we are not trickleing it)
    await iceGatheringComplete;
    offer.sdp += iceCandidates.map((candidate) => `a=${candidate.candidate}`).join('\r\n') + '\r\n';

    const answer = await signaling.offerStreaming(selected_device.value!.deviceId, offer);
    if (answer === null) {
        console.error('Failed to get answer');
        return;
    }

    peerConnection.setRemoteDescription(answer);
    console.log('Received answer:', answer);