Mastodon
/
Bubble
mirror of https://github.com/lumaa-dev/BubbleApp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Alt text + Hugging Face AI

This commit is contained in:
Lumaa 2024-02-18 12:44:58 +01:00
parent 3187e95eca
commit d4fe9ce42f
5 changed files with 319 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Threaded.xcodeproj/project.pbxproj
View File

@ -7,6 +7,8 @@
objects = {
/* Begin PBXBuildFile section */
B9029FC22B81259400AA9B68 /* Secret.plist in Resources */ = {isa = PBXBuildFile; fileRef = B9029FC12B81259400AA9B68 /* Secret.plist */; };
B9029FC42B8125CE00AA9B68 /* HuggingFace.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9029FC32B8125CE00AA9B68 /* HuggingFace.swift */; };
B915C4422B6F908C00042DDB /* ProfileView.swift in Sources */ = {isa = PBXBuildFile; fileRef = B915C4412B6F908C00042DDB /* ProfileView.swift */; };
B93757112B7FB8D400652F91 /* AltClients.swift in Sources */ = {isa = PBXBuildFile; fileRef = B93757102B7FB8D400652F91 /* AltClients.swift */; };
B93ADFCB2B7625CD00FF9172 /* DiscoveryView.swift in Sources */ = {isa = PBXBuildFile; fileRef = B93ADFCA2B7625CD00FF9172 /* DiscoveryView.swift */; };
@ -117,6 +119,8 @@
/* End PBXCopyFilesBuildPhase section */
/* Begin PBXFileReference section */
B9029FC12B81259400AA9B68 /* Secret.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Secret.plist; sourceTree = "<group>"; };
B9029FC32B8125CE00AA9B68 /* HuggingFace.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HuggingFace.swift; sourceTree = "<group>"; };
B915C4412B6F908C00042DDB /* ProfileView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileView.swift; sourceTree = "<group>"; };
B93757102B7FB8D400652F91 /* AltClients.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AltClients.swift; sourceTree = "<group>"; };
B93ADFCA2B7625CD00FF9172 /* DiscoveryView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiscoveryView.swift; sourceTree = "<group>"; };
@ -298,6 +302,7 @@
isa = PBXGroup;
children = (
B9FB94A02B2EF23100D81C07 /* Info.plist */,
B9029FC12B81259400AA9B68 /* Secret.plist */,
B9FB945A2B2DEECE00D81C07 /* ThreadedApp.swift */,
B9EBE8572B474FD600FB594D /* AppDelegate.swift */,
B9FB946E2B2DF3BB00D81C07 /* Components */,
@ -328,6 +333,7 @@
B9D9C6BF2B6A56D500C26A41 /* Notifications */,
B9FB946F2B2DF3CD00D81C07 /* Navigator.swift */,
B9FB94A12B2EF24A00D81C07 /* AppInfo.swift */,
B9029FC32B8125CE00AA9B68 /* HuggingFace.swift */,
B98BC74C2B46CFCE00595441 /* UserPreferences.swift */,
B9FB94872B2E223E00D81C07 /* Emoji.swift */,
B9FB949A2B2EF09A00D81C07 /* Client.swift */,
@ -525,6 +531,7 @@
B9DC69302B79378400E625B9 /* ThreadedPlus.storekit in Resources */,
B9FB94642B2DEECF00D81C07 /* Preview Assets.xcassets in Resources */,
B9FB94612B2DEECF00D81C07 /* Assets.xcassets in Resources */,
B9029FC22B81259400AA9B68 /* Secret.plist in Resources */,
B9FB94902B2E2B0E00D81C07 /* Localizable.xcstrings in Resources */,
B9CFC43B2B4F08C9004CFCB7 /* LaunchStoryboard.storyboard in Resources */,
B97BCE2A2B3ED2C80044756D /* LICENSE in Resources */,
@ -582,6 +589,7 @@
B97491E32B6E96700098BC48 /* SymbolWidth.swift in Sources */,
B9DC692B2B78E60300E625B9 /* PostsView.swift in Sources */,
B9FB94BC2B2F035500D81C07 /* Tag.swift in Sources */,
B9029FC42B8125CE00AA9B68 /* HuggingFace.swift in Sources */,
B9BED51A2B5D662D00C9B715 /* ShareSheetController.swift in Sources */,
B93757112B7FB8D400652F91 /* AltClients.swift in Sources */,
B9BED5162B5D5E6500C9B715 /* PostInteractor.swift in Sources */,

51
Threaded/Data/HuggingFace.swift
View File

@ -1,3 +1,54 @@
//Made by Lumaa
import Foundation
import UIKit
@Observable
final class HuggingFace: ObservableObject {
static var token: String = ""
static let altGenUrl: URL = URL(string: "https://api-inference.huggingface.co/models/Salesforce/blip-image-captioning-large")!
var lastImgGeneration: String? = nil
init() {
self.lastImgGeneration = nil
}
static func getToken() -> String? {
guard let path = Bundle.main.path(forResource: "Secret", ofType: "plist") else { return nil }
let url = URL(fileURLWithPath: path)
let data = try! Data(contentsOf: url)
guard let plist = try! PropertyListSerialization.propertyList(from: data, options: .mutableContainers, format: nil) as? [String: String] else { return nil }
Self.token = plist["AI_Token"] ?? ""
return Self.token
}
func altGeneration(image: UIImage) -> String? {
if let imageData = image.jpegData(compressionQuality: 0.5) {
let base64Image = imageData.base64EncodedString()
let parameters = ["image": base64Image]
let headers = ["Authorization": "Bearer \(Self.token)"]
var request = URLRequest(url: Self.altGenUrl)
request.httpMethod = "POST"
request.allHTTPHeaderFields = headers
request.httpBody = try? JSONSerialization.data(withJSONObject: parameters, options: [])
let semaphore = DispatchSemaphore(value: 0)
var jsonResponse: [[String: Any]]?
URLSession.shared.dataTask(with: request) { (data, response, error) in
defer { semaphore.signal() }
if let data = data {
jsonResponse = try? JSONSerialization.jsonObject(with: data, options: []) as? [[String: Any]]
print(jsonResponse?[0]["generated_text"] ?? "idfk")
}
}.resume()
semaphore.wait()
return (jsonResponse?[0]["generated_text"] as! String)
}
return nil
}
}

80
Threaded/Localizable.xcstrings
View File

@ -983,6 +983,86 @@
}
}
},
"posting.alt.apply" : {
"localizations" : {
"en" : {
"stringUnit" : {
"state" : "translated",
"value" : "Apply"
}
},
"fr" : {
"stringUnit" : {
"state" : "translated",
"value" : "Appliquer"
}
}
}
},
"posting.alt.cancel" : {
"localizations" : {
"en" : {
"stringUnit" : {
"state" : "translated",
"value" : "Cancel"
}
},
"fr" : {
"stringUnit" : {
"state" : "translated",
"value" : "Annuler"
}
}
}
},
"posting.alt.generate" : {
"localizations" : {
"en" : {
"stringUnit" : {
"state" : "translated",
"value" : "Generate using an AI"
}
},
"fr" : {
"stringUnit" : {
"state" : "translated",
"value" : "Générer en utilisant une IA"
}
}
}
},
"posting.alt.header" : {
"localizations" : {
"en" : {
"stringUnit" : {
"state" : "translated",
"value" : "Alt text"
}
},
"fr" : {
"stringUnit" : {
"state" : "translated",
"value" : "Texte Alt"
}
}
}
},
"posting.alt.prompt" : {
"localizations" : {
"en" : {
"stringUnit" : {
"state" : "translated",
"value" : "Describe the image"
}
},
"fr" : {
"stringUnit" : {
"state" : "translated",
"value" : "Décrivez l'image"
}
}
}
},
"setting.appearence" : {
"localizations" : {
"en" : {

4
Threaded/Views/ContentView.swift
View File

@ -6,6 +6,7 @@ import SwiftUI
struct ContentView: View {
@UIApplicationDelegateAdaptor private var appDelegate: AppDelegate
private var huggingFace: HuggingFace = HuggingFace()
@State private var preferences: UserPreferences = .defaultPreferences
@StateObject private var uniNavigator = UniversalNavigator()
@StateObject private var accountManager: AccountManager = AccountManager.shared
@ -50,6 +51,7 @@ struct ContentView: View {
.environment(uniNavigator)
.environment(accountManager)
.environment(appDelegate)
.environment(huggingFace)
.environmentObject(preferences)
.navigationBarTitleDisplayMode(.inline)
.onAppear {
@ -64,6 +66,8 @@ struct ContentView: View {
await recognizeAccount()
}
}
_ = HuggingFace.getToken()
}
.environment(\.openURL, OpenURLAction { url in
// Open internal URL.

192
Threaded/Views/PostingView.swift
View File

@ -20,10 +20,12 @@ struct PostingView: View {
@State private var selectingPhotos: Bool = false
@State private var mediaContainers: [MediaContainer] = []
@State private var mediaAttributes: [StatusData.MediaAttribute] = []
@State private var selectedPhotos: [PhotosPickerItem] = []
@State private var player: AVPlayer?
@State private var selectingEmoji: Bool = false
@State private var makingAlt: MediaContainer? = nil
@State private var loadingContent: Bool = false
@State private var postingStatus: Bool = false
@ -44,6 +46,11 @@ struct PostingView: View {
.presentationDragIndicator(.visible)
.presentationBackgroundInteraction(.enabled(upThrough: .height(200))) // Allow users to move the cursor while adding emojis
}
.sheet(item: $makingAlt) { container in
AltTextView(container: container, mediaContainers: $mediaContainers, mediaAttributes: $mediaAttributes)
.presentationDetents([.height(235), .medium])
.presentationDragIndicator(.visible)
}
} else {
loading
.background(Color.appBackground)
@ -76,7 +83,7 @@ struct PostingView: View {
.foregroundStyle(Color(uiColor: UIColor.label))
if !mediaContainers.isEmpty {
mediasView(containers: mediaContainers) //TODO: ALT text
mediasView(containers: mediaContainers)
}
editorButtons
@ -199,13 +206,13 @@ struct PostingView: View {
private let containerHeight: CGFloat = 450
@ViewBuilder
private func mediasView(containers: [MediaContainer]) -> some View {
private func mediasView(containers: [MediaContainer], actions: Bool = true) -> some View {
ViewThatFits {
hMedias(containers)
hMedias(containers, actions: actions)
.frame(maxHeight: containerHeight)
ScrollView(.horizontal, showsIndicators: false) {
hMedias(containers)
hMedias(containers, actions: actions)
}
.frame(maxHeight: containerHeight)
.scrollClipDisabled()
@ -213,7 +220,7 @@ struct PostingView: View {
}
@ViewBuilder
private func hMedias(_ containers: [MediaContainer]) -> some View {
private func hMedias(_ containers: [MediaContainer], actions: Bool) -> some View {
HStack(alignment: .firstTextBaseline, spacing: 10) {
ForEach(containers) { container in
ZStack(alignment: .topLeading) {
@ -237,6 +244,7 @@ struct PostingView: View {
}
}
.overlay(alignment: .topTrailing) {
if actions {
Button {
deleteAction(container: container)
} label: {
@ -249,6 +257,21 @@ struct PostingView: View {
}
}
}
.overlay(alignment: .topLeading) {
if actions && container.mediaAttachment != nil {
Button {
makingAlt = container
} label: {
Text(String("ALT"))
.font(.subheadline.smallCaps())
.padding(7.5)
.background(Material.ultraThick)
.clipShape(Capsule())
.padding(5)
}
}
}
}
}
.frame(maxHeight: containerHeight)
}
@ -374,7 +397,6 @@ struct PostingView: View {
mediaContainers.removeAll { removedIDs.contains($0.id) }
let newPickerItems = selectedPhotos.filter { !oldValue.contains($0) }
print("newPickerItems: \(newPickerItems.count)")
if !newPickerItems.isEmpty {
loadingContent = true
Task {
@ -564,25 +586,6 @@ struct PostingView: View {
}
}
func addDescription(container: PostingView.MediaContainer, description: String) async {
guard let client = accountManager.getClient(), let attachment = container.mediaAttachment else { return }
if let index = indexOf(container: container) {
do {
let media: MediaAttachment = try await client.put(endpoint: Media.media(id: attachment.id,
json: .init(description: description)))
mediaContainers[index] = MediaContainer(id: container.id, image: nil, movieTransferable: nil, gifTransferable: nil, mediaAttachment: media, error: nil)
} catch { print(error) }
}
}
private var mediaAttributes: [StatusData.MediaAttribute] = []
mutating func editDescription(container: PostingView.MediaContainer, description: String) async {
guard let attachment = container.mediaAttachment else { return }
if indexOf(container: container) != nil {
mediaAttributes.append(StatusData.MediaAttribute(id: attachment.id, description: description, thumbnail: nil, focus: nil))
}
}
private func uploadMedia(data: Data, mimeType: String) async throws -> MediaAttachment? {
guard let client = accountManager.getClient() else { return nil }
return try await client.mediaUpload(endpoint: Media.medias, version: .v2, method: "POST", mimeType: mimeType, filename: "file", data: data)
@ -650,4 +653,143 @@ extension PostingView {
let mediaAttachment: MediaAttachment?
let error: Error?
}
struct AltTextView: View {
@Environment(AccountManager.self) private var accountManager: AccountManager
@Environment(HuggingFace.self) private var huggingFace: HuggingFace
@Environment(\.dismiss) private var dismiss
var container: MediaContainer
@Binding var mediaContainers: [MediaContainer]
@Binding var mediaAttributes: [StatusData.MediaAttribute]
@State private var tasking: Bool = false
@State private var applying: Bool = false
@State private var alt: String = ""
@FocusState private var altFocused: Bool
var body: some View {
NavigationStack {
List {
TextField(String(""), text: $alt, prompt: Text("posting.alt.prompt"), axis: .vertical)
.labelsHidden()
.keyboardType(.asciiCapable)
.focused($altFocused)
Button {
tasking = true
let img = container.image
if img == nil, let media = container.mediaAttachment {
guard media.supportedType == .image else { return }
downloadImage(from: media.url ?? URL(string: "https://cdn.pixabay.com/photo/2023/08/28/20/32/flower-8220018_1280.jpg")!) { image in
if let uiimage = image {
alt = huggingFace.altGeneration(image: uiimage) ?? ""
tasking = false
} else {
print("Couldn't download image")
}
}
} else {
alt = huggingFace.altGeneration(image: img!) ?? ""
tasking = false
}
} label: {
if !tasking {
Label("posting.alt.generate", systemImage: "printer")
.foregroundStyle(Color.blue)
} else {
ProgressView()
.progressViewStyle(.circular)
.foregroundStyle(Color(uiColor: UIColor.label))
}
}
.tint(tasking ? Color(uiColor: UIColor.label) : Color.blue)
.disabled(tasking)
}
.navigationTitle(Text("posting.alt.header"))
.navigationBarTitleDisplayMode(.inline)
.toolbar {
ToolbarItem(placement: .primaryAction) {
Button {
applying = true
if let mediaAttachment = container.mediaAttachment {
if let str = mediaAttachment.description, !str.isEmpty {
Task {
await editDescription(container: container, description: alt)
applying = false
dismiss()
}
} else {
Task {
await addDescription(container: container, description: alt)
applying = false
dismiss()
}
}
}
} label: {
if applying {
ProgressView()
.progressViewStyle(.circular)
} else {
Text("posting.alt.apply")
}
}
}
ToolbarItem(placement: .cancellationAction) {
Button {
dismiss()
} label: {
Text("posting.alt.cancel")
}
}
}
.onAppear {
altFocused = true
}
}
}
private func downloadImage(from url: URL, completion: @escaping (UIImage?) -> Void) {
URLSession.shared.dataTask(with: url) { data, response, error in
guard let data = data, error == nil else {
completion(nil)
return
}
DispatchQueue.main.async {
completion(UIImage(data: data))
}
}.resume()
}
private func addDescription(container: MediaContainer, description: String) async {
guard let client = accountManager.getClient(), let attachment = container.mediaAttachment else { return }
if let index = indexOf(container: container) {
do {
let media: MediaAttachment = try await client.put(endpoint: Media.media(id: attachment.id,
json: .init(description: description)))
mediaContainers[index] = MediaContainer(
id: container.id,
image: nil,
movieTransferable: nil,
gifTransferable: nil,
mediaAttachment: media,
error: nil
)
} catch {}
}
}
private func editDescription(container: MediaContainer, description: String) async {
guard let attachment = container.mediaAttachment else { return }
if indexOf(container: container) != nil {
mediaAttributes.append(StatusData.MediaAttribute(id: attachment.id, description: description, thumbnail: nil, focus: nil))
}
}
private func indexOf(container: MediaContainer) -> Int? {
mediaContainers.firstIndex(where: { $0.id == container.id })
}
}
}