VoIPCallPeerConnectionClient.swift 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825
  1. // _____ _
  2. // |_ _| |_ _ _ ___ ___ _ __ __ _
  3. // | | | ' \| '_/ -_) -_) ' \/ _` |_
  4. // |_| |_||_|_| \___\___|_|_|_\__,_(_)
  5. //
  6. // Threema iOS Client
  7. // Copyright (c) 2019-2020 Threema GmbH
  8. //
  9. // This program is free software: you can redistribute it and/or modify
  10. // it under the terms of the GNU Affero General Public License, version 3,
  11. // as published by the Free Software Foundation.
  12. //
  13. // This program is distributed in the hope that it will be useful,
  14. // but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16. // GNU Affero General Public License for more details.
  17. //
  18. // You should have received a copy of the GNU Affero General Public License
  19. // along with this program. If not, see <https://www.gnu.org/licenses/>.
  20. import Foundation
  21. import AVFoundation
  22. import WebRTC
  23. import ThreemaFramework
  24. protocol VoIPCallPeerConnectionClientDelegate: class {
  25. func peerConnectionClient(_ client: VoIPCallPeerConnectionClient, changeState: VoIPCallService.CallState)
  26. func peerConnectionClient(_ client: VoIPCallPeerConnectionClient, audioMuted: Bool)
  27. func peerConnectionClient(_ client: VoIPCallPeerConnectionClient, speakerActive: Bool)
  28. func peerConnectionClient(_ client: VoIPCallPeerConnectionClient, removedCandidates: [RTCIceCandidate])
  29. func peerConnectionClient(_ client: VoIPCallPeerConnectionClient, addedCandidate: RTCIceCandidate)
  30. func peerConnectionClient(_ client: VoIPCallPeerConnectionClient, didChangeConnectionState state: RTCIceConnectionState)
  31. func peerConnectionClient(_ client: VoIPCallPeerConnectionClient, receivingVideo: Bool)
  32. func peerConnectionClient(_ client: VoIPCallPeerConnectionClient, didReceiveData: Data)
  33. }
  34. final class VoIPCallPeerConnectionClient: NSObject {
  35. // The `RTCPeerConnectionFactory` is in charge of creating new RTCPeerConnection instances.
  36. // A new RTCPeerConnection should be created every new call, but the factory is shared.
  37. private static var factory: RTCPeerConnectionFactory = {
  38. let decoderFactory = RTCDefaultVideoDecoderFactory()
  39. let encoderFactory = RTCDefaultVideoEncoderFactory()
  40. return RTCPeerConnectionFactory(encoderFactory: encoderFactory, decoderFactory: decoderFactory)
  41. }()
  42. weak var delegate: VoIPCallPeerConnectionClientDelegate?
  43. let peerConnection: RTCPeerConnection
  44. var remoteVideoQualityProfile: CallsignalingProtocol.ThreemaVideoCallQualityProfile? {
  45. didSet {
  46. let newProfile = CallsignalingProtocol.findCommonProfile(remoteProfile: remoteVideoQualityProfile, networkIsRelayed: networkIsRelayed)
  47. self.setOutgoingVideoLimits(maxBitrate: Int(newProfile.bitrate) * 1000, maxFps: Int(newProfile.maxFps), w: UInt32(newProfile.maxResolution.width), h: UInt32(newProfile.maxResolution.height))
  48. }
  49. }
  50. var isRemoteVideoActivated: Bool = false {
  51. didSet {
  52. self.delegate?.peerConnectionClient(self, receivingVideo: true)
  53. }
  54. }
  55. private var peerConnectionParameters: PeerConnectionParameters
  56. private let rtcAudioSession = RTCAudioSession.sharedInstance()
  57. private let audioQueue = DispatchQueue(label: "VoIPCallAudioQueue")
  58. private var dataChannelQueue = Queue<Any>()
  59. private let dataChannelLockQueue = DispatchQueue(label: "VoIIPCallPeerConnectionClientLockQueue")
  60. private var videoCapturer: RTCVideoCapturer?
  61. private var localVideoTrack: RTCVideoTrack?
  62. private var localVideoSender: RTCRtpSender?
  63. private var remoteVideoTrack: RTCVideoTrack?
  64. private var dataChannel: RTCDataChannel?
  65. private var statsTimer: Timer?
  66. private var receivingVideoTimer: Timer?
  67. private var contact: Contact?
  68. private let internetReachability: Reachability = Reachability.forInternetConnection()
  69. private var lastInternetStatus: NetworkStatus?
  70. private(set) var networkIsRelayed: Bool = false // will be checked every 30 seconds after connection is established
  71. private var previousPeriodDebugState: VoIPStatsState? = nil
  72. private var previousVideoState: VoIPStatsState? = nil
  73. private static let logStatsIntervalConnecting = 2.0
  74. private static let logStatsIntervalConnected = 30.0
  75. private static let checkReceivingVideoInterval = 2.0
  76. public struct PeerConnectionParameters {
  77. public var isVideoCallAvailable: Bool = true
  78. public var videoCodecHwAcceleration: Bool = true
  79. public var forceTurn: Bool = false
  80. public var gatherContinually: Bool = false
  81. public var allowIpv6: Bool = true
  82. internal var isDataChannelAvailable: Bool = false
  83. }
  84. static func instantiate(contact: Contact, peerConnectionParameters: PeerConnectionParameters, completion: @escaping (Result<VoIPCallPeerConnectionClient,Error>) -> Void) {
  85. VoIPCallPeerConnectionClient.defaultRTCConfiguration(peerConnectionParameters: peerConnectionParameters) { (result) in
  86. do {
  87. let client = VoIPCallPeerConnectionClient.init(contact: contact, peerConnectionParameters: peerConnectionParameters, config: try result.get())
  88. completion(.success(client))
  89. } catch let e {
  90. completion(.failure(e))
  91. }
  92. }
  93. }
  94. /**
  95. Init new peer connection with a contact
  96. - parameter contact: Call contact
  97. */
  98. required init(contact: Contact, peerConnectionParameters: PeerConnectionParameters, config: RTCConfiguration) {
  99. self.peerConnectionParameters = peerConnectionParameters
  100. let constraints = VoIPCallPeerConnectionClient.defaultPeerConnectionConstraints()
  101. peerConnection = VoIPCallPeerConnectionClient.factory.peerConnection(with: config, constraints: constraints, delegate: nil)
  102. self.contact = contact
  103. super.init()
  104. self.createMediaSenders()
  105. configureAudioSession()
  106. peerConnection.delegate = self
  107. NotificationCenter.default.addObserver(self, selector: #selector(networkStatusDidChange), name: NSNotification.Name.reachabilityChanged, object: nil)
  108. NotificationCenter.default.addObserver(forName: Notification.Name(kThreemaVideoCallsQualitySettingChanged), object: nil, queue: nil) { (notification) in
  109. self.setQualityProfileForVideoSource()
  110. }
  111. if let protobufMessage = CallsignalingProtocol.encodeVideoQuality(CallsignalingProtocol.localPeerQualityProfile().profile!) {
  112. sendDataToRemote(protobufMessage)
  113. }
  114. }
  115. }
  116. extension VoIPCallPeerConnectionClient {
  117. // MARK:- Audio control
  118. /**
  119. Mute the audio of the rtc session
  120. */
  121. func muteAudio(completion: @escaping () -> ()) {
  122. setAudioEnabled(false)
  123. delegate?.peerConnectionClient(self, audioMuted: true)
  124. if let protobufMessage = CallsignalingProtocol.encodeMute(true) {
  125. sendDataToRemote(protobufMessage)
  126. }
  127. completion()
  128. }
  129. /**
  130. Unmute the audio of the rtc session
  131. */
  132. func unmuteAudio(completion: @escaping () -> ()) {
  133. setAudioEnabled(true)
  134. delegate?.peerConnectionClient(self, audioMuted: false)
  135. if let protobufMessage = CallsignalingProtocol.encodeMute(false) {
  136. sendDataToRemote(protobufMessage)
  137. }
  138. completion()
  139. }
  140. /**
  141. Activate RTC audio
  142. */
  143. func activateRTCAudio(speakerActive: Bool) {
  144. audioQueue.async { [weak self] in
  145. guard let self = self else {
  146. return
  147. }
  148. self.rtcAudioSession.lockForConfiguration()
  149. do {
  150. try self.rtcAudioSession.setCategory(AVAudioSession.Category.playAndRecord.rawValue, with: [.duckOthers, .allowBluetooth, .allowBluetoothA2DP])
  151. try self.rtcAudioSession.setMode(AVAudioSession.Mode.voiceChat.rawValue)
  152. try self.rtcAudioSession.overrideOutputAudioPort(speakerActive ? .speaker : .none)
  153. try self.rtcAudioSession.setActive(true)
  154. } catch let error {
  155. debugPrint("Error setting AVAudioSession category: \(error)")
  156. }
  157. self.rtcAudioSession.unlockForConfiguration()
  158. self.rtcAudioSession.isAudioEnabled = true
  159. }
  160. }
  161. /**
  162. Disable the speaker for the rtc session
  163. */
  164. func speakerOff() {
  165. audioQueue.async { [weak self] in
  166. guard let self = self else {
  167. return
  168. }
  169. self.rtcAudioSession.lockForConfiguration()
  170. do {
  171. try self.rtcAudioSession.setCategory(AVAudioSession.Category.playAndRecord.rawValue, with: [.duckOthers, .allowBluetooth, .allowBluetoothA2DP])
  172. try self.rtcAudioSession.setMode(AVAudioSession.Mode.voiceChat.rawValue)
  173. try self.rtcAudioSession.overrideOutputAudioPort(.none)
  174. try self.rtcAudioSession.setActive(true)
  175. } catch let error {
  176. debugPrint("Error setting AVAudioSession category: \(error)")
  177. }
  178. self.rtcAudioSession.unlockForConfiguration()
  179. self.delegate?.peerConnectionClient(self, speakerActive: false)
  180. }
  181. }
  182. /**
  183. Enable the speaker for the rtc session
  184. */
  185. func speakerOn() {
  186. audioQueue.async { [weak self] in
  187. guard let self = self else {
  188. return
  189. }
  190. self.rtcAudioSession.lockForConfiguration()
  191. do {
  192. try self.rtcAudioSession.setCategory(AVAudioSession.Category.playAndRecord.rawValue, with: [.duckOthers, .allowBluetooth, .allowBluetoothA2DP])
  193. try self.rtcAudioSession.setMode(AVAudioSession.Mode.videoChat.rawValue)
  194. try self.rtcAudioSession.overrideOutputAudioPort(.speaker)
  195. try self.rtcAudioSession.setActive(true)
  196. } catch let error {
  197. debugPrint("Couldn't force audio to speaker: \(error)")
  198. }
  199. self.rtcAudioSession.unlockForConfiguration()
  200. self.delegate?.peerConnectionClient(self, speakerActive: true)
  201. }
  202. }
  203. /**
  204. Set the audio track for the peer connection
  205. */
  206. private func setAudioEnabled(_ isEnabled: Bool) {
  207. let audioTracks = self.peerConnection.transceivers.compactMap { return $0.sender.track as? RTCAudioTrack }
  208. audioTracks.forEach { $0.isEnabled = isEnabled }
  209. DDLogNotice("Threema call: set audio to \(isEnabled)")
  210. }
  211. }
  212. extension VoIPCallPeerConnectionClient {
  213. // MARK: class functions
  214. /**
  215. Configure the peer connection
  216. - parameter alwaysRelayCall: true or false, if user enabled always relay call setting
  217. - returns: RTCConfiguration for the peer connection
  218. */
  219. internal class func defaultRTCConfiguration(peerConnectionParameters: PeerConnectionParameters, completion: @escaping (Result<RTCConfiguration,Error>) -> Void) {
  220. // forceTurn determines whether to use dual stack enabled TURN servers.
  221. // In normal mode, the device is either:
  222. // a) IPv4 only or dual stack. It can then be reached directly or via relaying over IPv4 TURN servers.
  223. // b) IPv6 only and then **must** be reachable via a peer-to-peer connection.
  224. //
  225. // When enforcing relayed mode, the device may have an IPv6 only configuration, so we need to be able
  226. // to reach our TURN servers via IPv6 or no connection can be established at all.
  227. VoIPIceServerSource.obtainIceServers(dualStack: peerConnectionParameters.forceTurn) { (result) in
  228. do {
  229. let configuration = RTCConfiguration.init()
  230. configuration.iceServers = [try result.get()]
  231. if peerConnectionParameters.forceTurn == true {
  232. configuration.iceTransportPolicy = .relay
  233. }
  234. configuration.bundlePolicy = .maxBundle
  235. configuration.rtcpMuxPolicy = .require
  236. configuration.tcpCandidatePolicy = .disabled
  237. configuration.sdpSemantics = .unifiedPlan
  238. configuration.continualGatheringPolicy = peerConnectionParameters.gatherContinually ? .gatherContinually : .gatherOnce
  239. configuration.keyType = .ECDSA
  240. configuration.cryptoOptions = RTCCryptoOptions(srtpEnableGcmCryptoSuites: true, srtpEnableAes128Sha1_32CryptoCipher: false, srtpEnableAes128Sha1_80CryptoCipher: false, srtpEnableEncryptedRtpHeaderExtensions: true, sframeRequireFrameEncryption: false)
  241. configuration.offerExtmapAllowMixed = true
  242. completion(.success(configuration))
  243. } catch let error {
  244. completion(.failure(error))
  245. }
  246. }
  247. }
  248. /**
  249. Get the rtc media constraints for the peer connection
  250. - returns: RTCMediaConstraints for the peer connection
  251. */
  252. internal class func defaultPeerConnectionConstraints() -> RTCMediaConstraints {
  253. let optionalConstraints = ["DtlsSrtpKeyAgreement": kRTCMediaConstraintsValueTrue]
  254. let constraints = RTCMediaConstraints.init(mandatoryConstraints: nil, optionalConstraints: optionalConstraints)
  255. return constraints
  256. }
  257. /**
  258. Get the rtc media constraints for the offer or answer
  259. - returns: RTCMediaConstraints for the offer or answer
  260. */
  261. internal class func mediaConstrains(isVideoCallAvailable: Bool) -> RTCMediaConstraints {
  262. let mandatoryConstraints = [kRTCMediaConstraintsOfferToReceiveAudio: kRTCMediaConstraintsValueTrue, kRTCMediaConstraintsOfferToReceiveVideo: isVideoCallAvailable ? kRTCMediaConstraintsValueTrue : kRTCMediaConstraintsValueFalse]
  263. let constraints = RTCMediaConstraints(mandatoryConstraints: mandatoryConstraints, optionalConstraints: nil)
  264. return constraints
  265. }
  266. }
  267. extension VoIPCallPeerConnectionClient {
  268. // MARK: public functions
  269. func startCaptureLocalVideo(renderer: RTCVideoRenderer, useBackCamera: Bool, switchCamera: Bool = false) {
  270. lastInternetStatus = internetReachability.currentReachabilityStatus()
  271. guard let capturer = self.videoCapturer as? RTCCameraVideoCapturer else {
  272. return
  273. }
  274. let newProfile = CallsignalingProtocol.findCommonProfile(remoteProfile: self.remoteVideoQualityProfile, networkIsRelayed: networkIsRelayed)
  275. self.setOutgoingVideoLimits(maxBitrate: Int(newProfile.bitrate) * 1000, maxFps: Int(newProfile.maxFps), w: UInt32(newProfile.maxResolution.width), h: UInt32(newProfile.maxResolution.height))
  276. let localCaptureQualityProfile = CallsignalingProtocol.localCaptureQualityProfile()
  277. if useBackCamera == true, let backCamera = (RTCCameraVideoCapturer.captureDevices().first { $0.position == .back }) {
  278. let format = selectFormatForDevice(device: backCamera, width: Int32(localCaptureQualityProfile.maxResolution.width), height: Int32(localCaptureQualityProfile.maxResolution.height), capturer: capturer)
  279. capturer.startCapture(with: backCamera,
  280. format: format,
  281. fps: Int(localCaptureQualityProfile.maxFps))
  282. } else {
  283. guard
  284. let frontCamera = (RTCCameraVideoCapturer.captureDevices().first { $0.position == .front }) else {
  285. return
  286. }
  287. let format = selectFormatForDevice(device: frontCamera, width: Int32(localCaptureQualityProfile.maxResolution.width), height: Int32(localCaptureQualityProfile.maxResolution.height), capturer: capturer)
  288. capturer.startCapture(with: frontCamera,
  289. format: format,
  290. fps: Int(localCaptureQualityProfile.maxFps))
  291. }
  292. internetReachability.startNotifier()
  293. self.localVideoTrack?.add(renderer)
  294. if switchCamera == false {
  295. if let protobufMessage = CallsignalingProtocol.encodeVideoCapture(true) {
  296. sendDataToRemote(protobufMessage)
  297. }
  298. }
  299. }
  300. func endCaptureLocalVideo(renderer: RTCVideoRenderer, switchCamera: Bool = false) {
  301. if switchCamera == false {
  302. if let protobufMessage = CallsignalingProtocol.encodeVideoCapture(false) {
  303. sendDataToRemote(protobufMessage)
  304. }
  305. }
  306. internetReachability.stopNotifier()
  307. guard let capturer = self.videoCapturer as? RTCCameraVideoCapturer else {
  308. return
  309. }
  310. capturer.stopCapture()
  311. self.localVideoTrack?.remove(renderer)
  312. }
  313. func renderRemoteVideo(to renderer: RTCVideoRenderer) {
  314. self.remoteVideoTrack?.add(renderer)
  315. }
  316. func endRemoteVideo(renderer: RTCVideoRenderer) {
  317. self.remoteVideoTrack?.remove(renderer)
  318. }
  319. func stopVideoCall() {
  320. guard let capturer = self.videoCapturer as? RTCCameraVideoCapturer else {
  321. return
  322. }
  323. capturer.stopCapture()
  324. localVideoTrack = nil
  325. remoteVideoTrack = nil
  326. }
  327. }
  328. extension VoIPCallPeerConnectionClient {
  329. // MARK: private functions
  330. /**
  331. Get the rtc media constraints for the audio
  332. - returns: RTCMediaConstraints for the audio
  333. */
  334. private func defaultAudioConstraints() -> RTCMediaConstraints {
  335. return RTCMediaConstraints.init(mandatoryConstraints: nil, optionalConstraints: nil)
  336. }
  337. /**
  338. Create an audio and video track and add it as local stream to the peer connection
  339. */
  340. private func createMediaSenders() {
  341. let streamId = "3MACALL"
  342. // Audio
  343. let audioTrack = self.createAudioTrack()
  344. self.peerConnection.add(audioTrack, streamIds: [streamId])
  345. if peerConnectionParameters.isVideoCallAvailable {
  346. // Video
  347. let videoTrack = self.createVideoTrack()
  348. self.localVideoTrack = videoTrack
  349. self.localVideoSender = self.peerConnection.add(videoTrack, streamIds: [streamId])
  350. self.remoteVideoTrack = self.peerConnection.transceivers.first { $0.mediaType == .video }?.receiver.track as? RTCVideoTrack
  351. }
  352. if let dataChannel = createDataChannel() {
  353. dataChannel.delegate = self
  354. self.dataChannel = dataChannel
  355. }
  356. }
  357. /**
  358. Create an audio track and add it as local stream to the peer connection
  359. */
  360. private func createAudioTrack() -> RTCAudioTrack {
  361. let audioConstrains = RTCMediaConstraints(mandatoryConstraints: nil, optionalConstraints: nil)
  362. let audioSource = VoIPCallPeerConnectionClient.factory.audioSource(with: audioConstrains)
  363. let audioTrack = VoIPCallPeerConnectionClient.factory.audioTrack(with: audioSource, trackId: "3MACALLa0")
  364. return audioTrack
  365. }
  366. /**
  367. Create a video track and add it as local stream to the peer connection
  368. */
  369. private func createVideoTrack() -> RTCVideoTrack {
  370. let videoSource = VoIPCallPeerConnectionClient.factory.videoSource()
  371. #if TARGET_OS_SIMULATOR
  372. self.videoCapturer = RTCFileVideoCapturer(delegate: videoSource)
  373. #else
  374. self.videoCapturer = RTCCameraVideoCapturer(delegate: videoSource)
  375. #endif
  376. let videoTrack = VoIPCallPeerConnectionClient.factory.videoTrack(with: videoSource, trackId: "3MACALLv0")
  377. return videoTrack
  378. }
  379. /**
  380. Create a data channel and add it to the peer connection
  381. */
  382. private func createDataChannel() -> RTCDataChannel? {
  383. let config = RTCDataChannelConfiguration()
  384. config.channelId = 0
  385. config.isNegotiated = true
  386. config.isOrdered = true
  387. guard let dataChannel = self.peerConnection.dataChannel(forLabel: "3MACALLdc0", configuration: config) else {
  388. debugPrint("Warning: Couldn't create data channel.")
  389. return nil
  390. }
  391. return dataChannel
  392. }
  393. /**
  394. Configure the audio session category to .playAndRecord in the mode .voiceChat
  395. */
  396. private func configureAudioSession() {
  397. self.rtcAudioSession.lockForConfiguration()
  398. do {
  399. try self.rtcAudioSession.setCategory(AVAudioSession.Category.playAndRecord.rawValue, with: [.duckOthers, .allowBluetooth, .allowBluetoothA2DP])
  400. try self.rtcAudioSession.setMode(AVAudioSession.Mode.voiceChat.rawValue)
  401. } catch let error {
  402. debugPrint("Error changeing AVAudioSession category: \(error)")
  403. }
  404. self.rtcAudioSession.unlockForConfiguration()
  405. }
  406. /**
  407. set outgoing video limits
  408. */
  409. private func setOutgoingVideoLimits(maxBitrate: Int, maxFps: Int, w: UInt32, h: UInt32) {
  410. setOutgoingVideoEncoderLimits(maxBitrate: maxBitrate, maxFps: maxFps)
  411. setOutgoingVideoResolution(w: w, h: h, maxFps: UInt32(maxFps))
  412. }
  413. /**
  414. set outgoing video encoder limits for bitrate and fps
  415. */
  416. private func setOutgoingVideoEncoderLimits(maxBitrate: Int, maxFps: Int) {
  417. guard let sender = localVideoSender else {
  418. debugPrint("setOutgoingVideoBandwidthLimit: Could not find local video sender")
  419. return
  420. }
  421. let parameters = sender.parameters
  422. parameters.degradationPreference = NSNumber(value: RTCDegradationPreference.balanced.rawValue)
  423. for encoding in parameters.encodings {
  424. DDLogNotice("Threema Call: rtp encoding before -> maxBitrateBps: \(encoding.maxBitrateBps ?? 0), maxFramerate: \(encoding.maxFramerate ?? 0)")
  425. encoding.maxBitrateBps = NSNumber(value: maxBitrate)
  426. encoding.maxFramerate = NSNumber(value: maxFps)
  427. DDLogNotice("Threema Call: rtp encoding after -> maxBitrateBps: \(encoding.maxBitrateBps ?? 0), maxFramerate: \(encoding.maxFramerate ?? 0)")
  428. }
  429. sender.parameters = parameters
  430. }
  431. private func setOutgoingVideoResolution(w: UInt32, h: UInt32, maxFps: UInt32) {
  432. guard let videoSource = self.localVideoTrack?.source else {
  433. return
  434. }
  435. videoSource.adaptOutputFormat(toWidth: Int32(w), height: Int32(h), fps: Int32(maxFps))
  436. }
  437. /**
  438. Select the correct format for the capture device
  439. */
  440. private func selectFormatForDevice(device: AVCaptureDevice, width: Int32, height: Int32, capturer: RTCCameraVideoCapturer) -> AVCaptureDevice.Format {
  441. let targetHeight = height
  442. let targetWidth = width
  443. var selectedFormat: AVCaptureDevice.Format?
  444. var currentDiff = Int32.max
  445. let supportedFormats = RTCCameraVideoCapturer.supportedFormats(for: device)
  446. for format in supportedFormats {
  447. let dimension: CMVideoDimensions = CMVideoFormatDescriptionGetDimensions(
  448. format.formatDescription
  449. )
  450. let diff =
  451. abs(targetWidth - dimension.width) +
  452. abs(targetHeight - dimension.height)
  453. let pixelFormat = CMFormatDescriptionGetMediaSubType(format.formatDescription)
  454. if (diff < currentDiff) {
  455. selectedFormat = format
  456. currentDiff = diff
  457. }else if(diff == currentDiff && pixelFormat == capturer.preferredOutputPixelFormat()){
  458. selectedFormat = format
  459. }
  460. }
  461. return selectedFormat!
  462. }
  463. private func sendDataToRemote(_ data: Data) {
  464. if peerConnectionParameters.isDataChannelAvailable {
  465. sendCachedDataChannelDataToRemote()
  466. let buffer = RTCDataBuffer(data: data, isBinary: true)
  467. self.dataChannel?.sendData(buffer)
  468. } else {
  469. dataChannelLockQueue.sync {
  470. dataChannelQueue.enqueue(data)
  471. }
  472. }
  473. }
  474. private func sendCachedDataChannelDataToRemote() {
  475. while dataChannelQueue.elements.count > 0 {
  476. var element: Data?
  477. dataChannelLockQueue.sync {
  478. element = dataChannelQueue.dequeue() as? Data
  479. }
  480. if element != nil {
  481. let buffer = RTCDataBuffer(data: element!, isBinary: true)
  482. self.dataChannel?.sendData(buffer)
  483. }
  484. }
  485. }
  486. private func setQualityProfileForVideoSource() {
  487. let newProfile = CallsignalingProtocol.findCommonProfile(remoteProfile: self.remoteVideoQualityProfile, networkIsRelayed: networkIsRelayed)
  488. self.setOutgoingVideoLimits(maxBitrate: Int(newProfile.bitrate) * 1000, maxFps: Int(newProfile.maxFps), w: UInt32(newProfile.maxResolution.width), h: UInt32(newProfile.maxResolution.height))
  489. if let protobufMessage = CallsignalingProtocol.encodeVideoQuality(CallsignalingProtocol.localPeerQualityProfile().profile!) {
  490. self.sendDataToRemote(protobufMessage)
  491. }
  492. }
  493. }
  494. extension VoIPCallPeerConnectionClient {
  495. // MARK: Signaling
  496. func offer(completion: @escaping (_ sdp: RTCSessionDescription?, _ error: VoIPCallSdpPatcher.SdpError?) -> Void) {
  497. let constrains = VoIPCallPeerConnectionClient.mediaConstrains(isVideoCallAvailable: peerConnectionParameters.isVideoCallAvailable)
  498. self.peerConnection.offer(for: constrains) { (sdp, error) in
  499. guard let sdp = sdp else {
  500. return
  501. }
  502. let extensionConfig: VoIPCallSdpPatcher.RtpHeaderExtensionConfig = self.contact?.isVideoCallAvailable() ?? false ? .ENABLE_WITH_ONE_AND_TWO_BYTE_HEADER : .DISABLE
  503. do {
  504. let patchedSdpString = try VoIPCallSdpPatcher(extensionConfig).patch(type: .LOCAL_OFFER, sdp: sdp.sdp)
  505. let patchedSdp = RTCSessionDescription(type: sdp.type, sdp: patchedSdpString)
  506. self.peerConnection.setLocalDescription(patchedSdp, completionHandler: { (error) in
  507. completion(patchedSdp, nil)
  508. })
  509. }
  510. catch let sdpError {
  511. completion(nil, sdpError as? VoIPCallSdpPatcher.SdpError)
  512. }
  513. }
  514. }
  515. func answer(completion: @escaping (_ sdp: RTCSessionDescription) -> Void) {
  516. let constrains = VoIPCallPeerConnectionClient.mediaConstrains(isVideoCallAvailable: peerConnectionParameters.isVideoCallAvailable)
  517. self.peerConnection.answer(for: constrains) { (sdp, error) in
  518. guard let sdp = sdp else {
  519. return
  520. }
  521. self.peerConnection.setLocalDescription(sdp, completionHandler: { (error) in
  522. completion(sdp)
  523. })
  524. }
  525. }
  526. func set(remoteSdp: RTCSessionDescription, completion: @escaping (Error?) -> ()) {
  527. self.peerConnection.setRemoteDescription(remoteSdp, completionHandler: completion)
  528. }
  529. func set(addRemoteCandidate: RTCIceCandidate) {
  530. self.peerConnection.add(addRemoteCandidate)
  531. }
  532. func set(removeRemoteCandidates: [RTCIceCandidate]) {
  533. self.peerConnection.remove(removeRemoteCandidates)
  534. }
  535. }
  536. extension VoIPCallPeerConnectionClient: RTCPeerConnectionDelegate {
  537. // MARK: RTCPeerConnectionDelegates
  538. func peerConnection(_ peerConnection: RTCPeerConnection, didChange stateChanged: RTCSignalingState) {
  539. debugPrint("peerConnection new signaling state: \(stateChanged.rawValue)")
  540. }
  541. func peerConnection(_ peerConnection: RTCPeerConnection, didAdd stream: RTCMediaStream) {
  542. debugPrint("peerConnection did add stream")
  543. }
  544. func peerConnection(_ peerConnection: RTCPeerConnection, didRemove stream: RTCMediaStream) {
  545. debugPrint("peerConnection did remove stream")
  546. }
  547. func peerConnectionShouldNegotiate(_ peerConnection: RTCPeerConnection) {
  548. debugPrint("peerConnection should negotiate")
  549. }
  550. func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceConnectionState) {
  551. debugPrint("peerConnection new connection state: \(newState.rawValue)")
  552. if newState == .checking {
  553. // Schedule 'connecting' stats timer
  554. let options = VoIPStatsOptions.init()
  555. options.selectedCandidatePair = false
  556. options.transport = true
  557. options.crypto = true
  558. options.inboundRtp = true
  559. options.outboundRtp = true
  560. options.tracks = true
  561. options.candidatePairsFlag = .OVERVIEW_AND_DETAILED
  562. self.schedulePeriodStats(options: options, period: VoIPCallPeerConnectionClient.logStatsIntervalConnecting)
  563. }
  564. if VoIPCallStateManager.shared.currentCallState() == .initalizing && (newState == .connected || newState == .completed) {
  565. let options = VoIPStatsOptions.init()
  566. options.selectedCandidatePair = true
  567. options.transport = true
  568. options.crypto = true
  569. options.inboundRtp = true
  570. options.outboundRtp = true
  571. options.tracks = true
  572. options.candidatePairsFlag = .OVERVIEW
  573. self.schedulePeriodStats(options: options, period: VoIPCallPeerConnectionClient.logStatsIntervalConnected)
  574. if peerConnectionParameters.isVideoCallAvailable {
  575. let receivedVideoOptions = VoIPStatsOptions.init()
  576. receivedVideoOptions.framesReceived = true
  577. self.scheduleVideoStats(options: receivedVideoOptions, period: VoIPCallPeerConnectionClient.checkReceivingVideoInterval)
  578. }
  579. }
  580. if newState == .disconnected || newState == .failed || newState == .closed || newState == .new {
  581. if receivingVideoTimer != nil {
  582. receivingVideoTimer?.invalidate()
  583. receivingVideoTimer = nil
  584. }
  585. }
  586. self.delegate?.peerConnectionClient(self, didChangeConnectionState: newState)
  587. }
  588. func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceGatheringState) {
  589. debugPrint("peerConnection new gathering state: \(newState.rawValue)")
  590. }
  591. func peerConnection(_ peerConnection: RTCPeerConnection, didGenerate candidate: RTCIceCandidate) {
  592. self.delegate?.peerConnectionClient(self, addedCandidate: candidate)
  593. }
  594. func peerConnection(_ peerConnection: RTCPeerConnection, didRemove candidates: [RTCIceCandidate]) {
  595. self.delegate?.peerConnectionClient(self, removedCandidates: candidates)
  596. }
  597. func peerConnection(_ peerConnection: RTCPeerConnection, didOpen dataChannel: RTCDataChannel) {
  598. debugPrint("peerConnection did open data channel")
  599. }
  600. }
  601. extension VoIPCallPeerConnectionClient: RTCDataChannelDelegate {
  602. // MARK: RTCDataChannelDelegate
  603. func dataChannelDidChangeState(_ dataChannel: RTCDataChannel) {
  604. debugPrint("dataChannel did change state: \(dataChannel.readyState.rawValue)")
  605. peerConnectionParameters.isDataChannelAvailable = dataChannel.readyState == .open
  606. sendCachedDataChannelDataToRemote()
  607. }
  608. func dataChannel(_ dataChannel: RTCDataChannel, didReceiveMessageWith buffer: RTCDataBuffer) {
  609. self.delegate?.peerConnectionClient(self, didReceiveData: buffer.data)
  610. }
  611. }
  612. extension VoIPCallPeerConnectionClient {
  613. // MARK: VoIP Stats
  614. func schedulePeriodStats(options: VoIPStatsOptions, period: TimeInterval) {
  615. if statsTimer != nil && statsTimer?.isValid == true {
  616. statsTimer?.invalidate()
  617. statsTimer = nil
  618. }
  619. // Create new timer with <period> (but immediately log once)
  620. var dict = [AnyHashable: Any]()
  621. dict.updateValue(peerConnection, forKey: "connection")
  622. dict.updateValue(options, forKey: "options")
  623. self.logDebugStats(dict: dict)
  624. DispatchQueue.main.async {
  625. self.statsTimer = Timer.scheduledTimer(withTimeInterval: period, repeats: true, block: { (timer) in
  626. self.logDebugStats(dict: dict)
  627. })
  628. }
  629. }
  630. func scheduleVideoStats(options: VoIPStatsOptions, period: TimeInterval) {
  631. if receivingVideoTimer != nil && receivingVideoTimer?.isValid == true {
  632. receivingVideoTimer?.invalidate()
  633. receivingVideoTimer = nil
  634. }
  635. // Create new timer with <period>
  636. var dict = [AnyHashable: Any]()
  637. dict.updateValue(peerConnection, forKey: "connection")
  638. dict.updateValue(options, forKey: "options")
  639. self.checkIsReceivingVideo(dict: dict)
  640. DispatchQueue.main.async {
  641. self.receivingVideoTimer = Timer.scheduledTimer(withTimeInterval: period, repeats: true, block: { (timer) in
  642. self.checkIsReceivingVideo(dict: dict)
  643. })
  644. }
  645. }
  646. func logDebugEndStats(completion: @escaping () -> ()) {
  647. if statsTimer != nil && statsTimer?.isValid == true {
  648. statsTimer?.invalidate()
  649. statsTimer = nil
  650. // Hijack the existing dict, override options and set callback
  651. let options = VoIPStatsOptions.init()
  652. options.selectedCandidatePair = false
  653. options.transport = true
  654. options.crypto = true
  655. options.inboundRtp = true
  656. options.outboundRtp = true
  657. options.tracks = true
  658. options.candidatePairsFlag = .OVERVIEW_AND_DETAILED
  659. // One-shot stats fetch before disconnect
  660. self.logDebugStats(dict: ["connection": peerConnection, "options": options, "callback": completion])
  661. } else {
  662. completion()
  663. }
  664. }
  665. func logDebugStats(dict: [AnyHashable: Any]) {
  666. let connection = dict["connection"] as! RTCPeerConnection
  667. let options = dict["options"] as! VoIPStatsOptions
  668. connection.statistics { (report) in
  669. let stats = VoIPStats.init(report: report, options: options, transceivers: connection.transceivers, previousState: self.previousPeriodDebugState)
  670. self.previousPeriodDebugState = stats.buildVoIPStatsState()
  671. self.networkIsRelayed = stats.usesRelay()
  672. var statsString = stats.getRepresentation()
  673. statsString += "\n\(CallsignalingProtocol.printDebugQualityProfiles(remoteProfile: self.remoteVideoQualityProfile, networkIsRelayed: self.networkIsRelayed))"
  674. ValidationLogger.shared()?.logString("Call: Stats\n \(statsString)")
  675. if let callback = dict["callback"] as? (() -> Void) {
  676. callback()
  677. }
  678. }
  679. }
  680. func checkIsReceivingVideo(dict: [AnyHashable: Any]) {
  681. let connection = dict["connection"] as! RTCPeerConnection
  682. let options = dict["options"] as! VoIPStatsOptions
  683. if isRemoteVideoActivated {
  684. self.delegate?.peerConnectionClient(self, receivingVideo: true)
  685. } else {
  686. if remoteVideoTrack != nil {
  687. connection.statistics { (report) in
  688. let stats = VoIPStats.init(report: report, options: options, transceivers: connection.transceivers, previousState: self.previousVideoState)
  689. self.previousVideoState = stats.buildVoIPStatsState()
  690. self.delegate?.peerConnectionClient(self, receivingVideo: stats.isReceivingVideo())
  691. }
  692. }
  693. }
  694. }
  695. }
  696. extension VoIPCallPeerConnectionClient {
  697. // MARK: Network Status Changed
  698. @objc func networkStatusDidChange(notice: Notification) {
  699. if CallsignalingProtocol.isThreemaVideoCallQualitySettingAuto() {
  700. let currentInternetStatus = internetReachability.currentReachabilityStatus()
  701. if lastInternetStatus != currentInternetStatus {
  702. lastInternetStatus = currentInternetStatus
  703. setQualityProfileForVideoSource()
  704. }
  705. }
  706. }
  707. }