From f7d0186bf3bba6e5a2ae8620d4e5088bb819afb8 Mon Sep 17 00:00:00 2001 From: CMK Date: Fri, 11 Nov 2022 21:28:19 +0800 Subject: [PATCH] feat: add compress progress display. Set video compress config to 720p at 60 fps --- .../xcshareddata/swiftpm/Package.resolved | 9 ++ MastodonSDK/Package.swift | 2 + .../MastodonSDK/Query/SerialStream.swift | 4 + .../Attachment/AttachmentView.swift | 46 ++++++++-- .../AttachmentViewModel+Compress.swift | 86 +++++++++++++++---- .../AttachmentViewModel+Upload.swift | 1 + .../Attachment/AttachmentViewModel.swift | 60 +++++++++---- .../ComposeContentViewModel.swift | 2 + 8 files changed, 166 insertions(+), 44 deletions(-) diff --git a/Mastodon.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Mastodon.xcworkspace/xcshareddata/swiftpm/Package.resolved index 64dc691bb..409b8820d 100644 --- a/Mastodon.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Mastodon.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -90,6 +90,15 @@ "version" : "2.2.5" } }, + { + "identity" : "nextlevelsessionexporter", + "kind" : "remoteSourceControl", + "location" : "https://github.com/NextLevel/NextLevelSessionExporter.git", + "state" : { + "revision" : "b6c0cce1aa37fe1547d694f958fac3c3524b74da", + "version" : "0.4.6" + } + }, { "identity" : "nuke", "kind" : "remoteSourceControl", diff --git a/MastodonSDK/Package.swift b/MastodonSDK/Package.swift index ca241038b..b364e6519 100644 --- a/MastodonSDK/Package.swift +++ b/MastodonSDK/Package.swift @@ -49,6 +49,7 @@ let package = Package( .package(url: "https://github.com/SDWebImage/SDWebImage.git", from: "5.12.0"), .package(url: "https://github.com/eneko/Stripes.git", from: "0.2.0"), .package(url: "https://github.com/onevcat/Kingfisher.git", from: "7.4.1"), + .package(url: "https://github.com/NextLevel/NextLevelSessionExporter.git", from: "0.4.6"), ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. @@ -124,6 +125,7 @@ let package = Package( .product(name: "PanModal", package: "PanModal"), .product(name: "Stripes", package: "Stripes"), .product(name: "Kingfisher", package: "Kingfisher"), + .product(name: "NextLevelSessionExporter", package: "NextLevelSessionExporter"), ] ), .testTarget( diff --git a/MastodonSDK/Sources/MastodonSDK/Query/SerialStream.swift b/MastodonSDK/Sources/MastodonSDK/Query/SerialStream.swift index 5d806b6ba..5808b9f6d 100644 --- a/MastodonSDK/Sources/MastodonSDK/Query/SerialStream.swift +++ b/MastodonSDK/Sources/MastodonSDK/Query/SerialStream.swift @@ -82,6 +82,10 @@ final class SerialStream: NSObject { self.progress.completedUnitCount += Int64(writeResult) self.logger.log(level: .debug, "\((#file as NSString).lastPathComponent, privacy: .public)[\(#line, privacy: .public)], \(#function, privacy: .public): estimate progress: \(self.progress.completedUnitCount)/\(self.progress.totalUnitCount)") + + if writeResult == -1 { + break + } } } diff --git a/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentView.swift b/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentView.swift index 3fbccbbc7..2dc8bf12f 100644 --- a/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentView.swift +++ b/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentView.swift @@ -48,7 +48,7 @@ public struct AttachmentView: View { // loaded // uploading… or upload failed // could retry upload when error emit - if viewModel.output != nil { + if viewModel.output != nil, viewModel.uploadState != .finish { VisualEffectView(effect: blurEffect) VStack { let action: AttachmentViewModel.Action = { @@ -74,8 +74,18 @@ public struct AttachmentView: View { .padding() .background(Color(Asset.Scene.Compose.Attachment.indicatorButtonBackground.color)) .overlay( - CircleProgressView(progress: viewModel.fractionCompleted) - .animation(.default, value: viewModel.fractionCompleted) + Group { + switch viewModel.uploadState { + case .compressing: + CircleProgressView(progress: viewModel.videoCompressProgress) + .animation(.default, value: viewModel.videoCompressProgress) + case .uploading: + CircleProgressView(progress: viewModel.fractionCompleted) + .animation(.default, value: viewModel.fractionCompleted) + default: + EmptyView() + } + } ) .clipShape(Circle()) .padding() @@ -84,11 +94,20 @@ public struct AttachmentView: View { let title: String = { switch action { case .remove: - let totalSizeInByte = viewModel.outputSizeInByte - let uploadSizeInByte = Double(totalSizeInByte) * min(1.0, viewModel.fractionCompleted) - let total = viewModel.byteCountFormatter.string(fromByteCount: Int64(totalSizeInByte)) - let upload = viewModel.byteCountFormatter.string(fromByteCount: Int64(uploadSizeInByte)) - return "\(upload) / \(total)" + switch viewModel.uploadState { + case .compressing: + return "Comporessing..." // TODO: i18n + default: + if viewModel.fractionCompleted < 0.9 { + let totalSizeInByte = viewModel.outputSizeInByte + let uploadSizeInByte = Double(totalSizeInByte) * min(1.0, viewModel.fractionCompleted + 0.1) // 9:1 + let total = viewModel.byteCountFormatter.string(fromByteCount: Int64(totalSizeInByte)) + let upload = viewModel.byteCountFormatter.string(fromByteCount: Int64(uploadSizeInByte)) + return "\(upload) / \(total)" + } else { + return "Server Processing..." // TODO: i18n + } + } case .retry: return "Upload Failed" // TODO: i18n } @@ -97,7 +116,13 @@ public struct AttachmentView: View { switch action { case .remove: if viewModel.progress.fractionCompleted < 1, viewModel.uploadState == .uploading { - return viewModel.remainTimeLocalizedString ?? "" + if viewModel.progress.fractionCompleted < 0.9 { + return viewModel.remainTimeLocalizedString ?? "" + } else { + return "" + } + } else if viewModel.videoCompressProgress < 1, viewModel.uploadState == .compressing { + return viewModel.percentageFormatter.string(from: NSNumber(floatLiteral: viewModel.videoCompressProgress)) ?? "" } else { return "" } @@ -113,6 +138,9 @@ public struct AttachmentView: View { .font(.system(size: 12, weight: .regular)) .foregroundColor(.white) .padding(.horizontal) + .lineLimit(nil) + .multilineTextAlignment(.center) + .frame(maxWidth: 240) } } } // end ZStack diff --git a/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentViewModel+Compress.swift b/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentViewModel+Compress.swift index e9c1df676..0fd0ab085 100644 --- a/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentViewModel+Compress.swift +++ b/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentViewModel+Compress.swift @@ -5,29 +5,81 @@ // Created by MainasuK on 2022/11/11. // +import os.log import UIKit import AVKit +import SessionExporter +import MastodonCore extension AttachmentViewModel { func comporessVideo(url: URL) async throws -> URL { - let task = Task { () -> URL in - let urlAsset = AVURLAsset(url: url) - guard let exportSession = AVAssetExportSession(asset: urlAsset, presetName: AVAssetExportPresetMediumQuality) else { - throw AttachmentError.invalidAttachmentType - } - let outputURL = try FileManager.default.createTemporaryFileURL( - filename: UUID().uuidString, - pathExtension: url.pathExtension - ) - exportSession.outputURL = outputURL - exportSession.outputFileType = AVFileType.mp4 - exportSession.shouldOptimizeForNetworkUse = true - await exportSession.export() - return outputURL + let urlAsset = AVURLAsset(url: url) + let exporter = NextLevelSessionExporter(withAsset: urlAsset) + exporter.outputFileType = .mp4 + + let outputURL = try FileManager.default.createTemporaryFileURL( + filename: UUID().uuidString, + pathExtension: url.pathExtension + ) + exporter.outputURL = outputURL + + let compressionDict: [String: Any] = [ + AVVideoAverageBitRateKey: NSNumber(integerLiteral: 3000000), // 3000k + AVVideoProfileLevelKey: AVVideoProfileLevelH264HighAutoLevel as String, + AVVideoAverageNonDroppableFrameRateKey: NSNumber(floatLiteral: 30), // 30 FPS + ] + exporter.videoOutputConfiguration = [ + AVVideoCodecKey: AVVideoCodecType.h264, + AVVideoWidthKey: NSNumber(integerLiteral: 1280), + AVVideoHeightKey: NSNumber(integerLiteral: 720), + AVVideoScalingModeKey: AVVideoScalingModeResizeAspectFill, + AVVideoCompressionPropertiesKey: compressionDict + ] + exporter.audioOutputConfiguration = [ + AVFormatIDKey: kAudioFormatMPEG4AAC, + AVEncoderBitRateKey: NSNumber(integerLiteral: 128000), // 128k + AVNumberOfChannelsKey: NSNumber(integerLiteral: 2), + AVSampleRateKey: NSNumber(value: Float(44100)) + ] + + // needs set to LOW priority to prevent priority inverse issue + let task = Task(priority: .utility) { + _ = try await exportVideo(by: exporter) } + _ = try await task.value - self.compressVideoTask = task - - return try await task.value + return outputURL } + + private func exportVideo(by exporter: NextLevelSessionExporter) async throws -> URL { + guard let outputURL = exporter.outputURL else { + throw AppError.badRequest + } + return try await withCheckedThrowingContinuation { continuation in + exporter.export(progressHandler: { progress in + DispatchQueue.main.async { [weak self] in + guard let self = self else { return } + self.videoCompressProgress = Double(progress) + os_log(.info, log: .debug, "%{public}s[%{public}ld], %{public}s: export progress: %.2f", ((#file as NSString).lastPathComponent), #line, #function, progress) + } + }, completionHandler: { result in + switch result { + case .success(let status): + switch status { + case .completed: + print("NextLevelSessionExporter, export completed, \(exporter.outputURL?.description ?? "")") + continuation.resume(with: .success(outputURL)) + default: + if Task.isCancelled { + exporter.cancelExport() + os_log(.info, log: .debug, "%{public}s[%{public}ld], %{public}s: cancel export", ((#file as NSString).lastPathComponent), #line, #function) + } + print("NextLevelSessionExporter, did not complete") + } + case .failure(let error): + continuation.resume(with: .failure(error)) + } + }) + } + } // end func } diff --git a/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentViewModel+Upload.swift b/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentViewModel+Upload.swift index 67f0e71ed..e26e97d35 100644 --- a/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentViewModel+Upload.swift +++ b/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentViewModel+Upload.swift @@ -55,6 +55,7 @@ extension Data { extension AttachmentViewModel { public enum UploadState { case none + case compressing case ready case uploading case fail diff --git a/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentViewModel.swift b/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentViewModel.swift index 7bbaaefaf..57f1d6b95 100644 --- a/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentViewModel.swift +++ b/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/Attachment/AttachmentViewModel.swift @@ -11,6 +11,7 @@ import Combine import PhotosUI import Kingfisher import MastodonCore +import func QuartzCore.CACurrentMediaTime public protocol AttachmentViewModelDelegate: AnyObject { func attachmentViewModel(_ viewModel: AttachmentViewModel, uploadStateValueDidChange state: AttachmentViewModel.UploadState) @@ -35,6 +36,12 @@ final public class AttachmentViewModel: NSObject, ObservableObject, Identifiable formatter.countStyle = .memory return formatter }() + + let percentageFormatter: NumberFormatter = { + let formatter = NumberFormatter() + formatter.numberStyle = .percent + return formatter + }() // input public let api: APIService @@ -43,7 +50,7 @@ final public class AttachmentViewModel: NSObject, ObservableObject, Identifiable @Published var caption = "" @Published var sizeLimit = SizeLimit() - var compressVideoTask: Task? + // var compressVideoTask: Task? // output @Published public private(set) var output: Output? @@ -54,11 +61,14 @@ final public class AttachmentViewModel: NSObject, ObservableObject, Identifiable @Published public private(set) var uploadState: UploadState = .none @Published public private(set) var uploadResult: UploadResult? @Published var error: Error? + + var uploadTask: Task<(), Never>? + @Published var videoCompressProgress: Double = 0 + let progress = Progress() // upload progress @Published var fractionCompleted: Double = 0 - var displayLink: CADisplayLink! private var lastTimestamp: TimeInterval? private var lastUploadSizeInByte: Int64 = 0 private var averageUploadSpeedInByte: Int64 = 0 @@ -78,11 +88,15 @@ final public class AttachmentViewModel: NSObject, ObservableObject, Identifiable super.init() // end init - self.displayLink = CADisplayLink( - target: self, - selector: #selector(AttachmentViewModel.step(displayLink:)) - ) - displayLink.add(to: .current, forMode: .common) + Timer.publish(every: 1.0 / 60.0, on: .main, in: .common) // 60 FPS + .autoconnect() + .share() + .receive(on: DispatchQueue.main) + .sink { [weak self] _ in + guard let self = self else { return } + self.step() + } + .store(in: &disposeBag) progress .observe(\.fractionCompleted, options: [.initial, .new]) { [weak self] progress, _ in @@ -120,12 +134,14 @@ final public class AttachmentViewModel: NSObject, ObservableObject, Identifiable .assign(to: &$thumbnail) defer { - Task { @MainActor in + let uploadTask = Task { @MainActor in do { var output = try await load(input: input) switch output { case .video(let fileURL, let mimeType): + self.output = output + self.update(uploadState: .compressing) let compressedFileURL = try await comporessVideo(url: fileURL) output = .video(compressedFileURL, mimeType: mimeType) try? FileManager.default.removeItem(at: fileURL) // remove old file @@ -142,12 +158,17 @@ final public class AttachmentViewModel: NSObject, ObservableObject, Identifiable self.error = error } } // end Task + self.uploadTask = uploadTask + Task { + await uploadTask.value + } } } deinit { - displayLink.invalidate() - displayLink.remove(from: .current, forMode: .common) + os_log(.info, log: .debug, "%{public}s[%{public}ld], %{public}s", ((#file as NSString).lastPathComponent), #line, #function) + + uploadTask?.cancel() switch output { case .image: @@ -172,31 +193,34 @@ extension AttachmentViewModel { return formatter }() - @objc private func step(displayLink: CADisplayLink) { + @objc private func step() { + + let uploadProgress = min(progress.fractionCompleted + 0.1, 1) // the progress split into 9:1 blocks (download : waiting) + guard let lastTimestamp = self.lastTimestamp else { - self.lastTimestamp = displayLink.timestamp - self.lastUploadSizeInByte = Int64(Double(outputSizeInByte) * progress.fractionCompleted) + self.lastTimestamp = CACurrentMediaTime() + self.lastUploadSizeInByte = Int64(Double(outputSizeInByte) * uploadProgress) return } - let duration = displayLink.timestamp - lastTimestamp + let duration = CACurrentMediaTime() - lastTimestamp guard duration >= 1.0 else { return } // update every 1 sec let old = self.lastUploadSizeInByte - self.lastUploadSizeInByte = Int64(Double(outputSizeInByte) * progress.fractionCompleted) + self.lastUploadSizeInByte = Int64(Double(outputSizeInByte) * uploadProgress) let newSpeed = self.lastUploadSizeInByte - old let lastAverageSpeed = self.averageUploadSpeedInByte let newAverageSpeed = Int64(AttachmentViewModel.SpeedSmoothingFactor * Double(newSpeed) + (1 - AttachmentViewModel.SpeedSmoothingFactor) * Double(lastAverageSpeed)) - let remainSizeInByte = Double(outputSizeInByte) * (1 - progress.fractionCompleted) + let remainSizeInByte = Double(outputSizeInByte) * (1 - uploadProgress) let speed = Double(newAverageSpeed) if speed != .zero { // estimate by speed let uploadRemainTimeInSecond = remainSizeInByte / speed // estimate by progress 1s for 10% - let remainPercentage = 1 - progress.fractionCompleted + let remainPercentage = 1 - uploadProgress let estimateRemainTimeByProgress = remainPercentage / 0.1 // max estimate var remainTimeInSecond = max(estimateRemainTimeByProgress, uploadRemainTimeInSecond) @@ -216,7 +240,7 @@ extension AttachmentViewModel { remainTimeLocalizedString = nil } - self.lastTimestamp = displayLink.timestamp + self.lastTimestamp = CACurrentMediaTime() self.averageUploadSpeedInByte = newAverageSpeed } } diff --git a/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/ComposeContentViewModel.swift b/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/ComposeContentViewModel.swift index 03db3b010..9a12f5e84 100644 --- a/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/ComposeContentViewModel.swift +++ b/MastodonSDK/Sources/MastodonUI/Scene/ComposeContent/ComposeContentViewModel.swift @@ -427,6 +427,8 @@ extension ComposeContentViewModel: AttachmentViewModelDelegate { switch attachmentViewModel.uploadState { case .none: return + case .compressing: + return case .ready: let count = self.attachmentViewModels.count logger.log(level: .debug, "\((#file as NSString).lastPathComponent, privacy: .public)[\(#line, privacy: .public)], \(#function, privacy: .public): upload \(i)/\(count) attachment")