From a3818cb03ecd947c7568cd0bd047db93d07faadb Mon Sep 17 00:00:00 2001 From: Ray Kitajima Date: Thu, 17 Jul 2025 19:47:20 +0900 Subject: [PATCH 1/5] feat: Image Playground --- Sources/SwiftApiAdapter/Loader.swift | 93 ++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/Sources/SwiftApiAdapter/Loader.swift b/Sources/SwiftApiAdapter/Loader.swift index d0f8608..c58d548 100644 --- a/Sources/SwiftApiAdapter/Loader.swift +++ b/Sources/SwiftApiAdapter/Loader.swift @@ -2,8 +2,101 @@ import Foundation import SwiftyJSON import SwiftSoup +#if canImport(ImagePlayground) +import ImagePlayground +#endif + +#if canImport(UIKit) +import UIKit +#endif + +#if canImport(AppKit) && !targetEnvironment(macCatalyst) +import AppKit +#endif + +#if canImport(ImagePlayground) +@available(iOS 18.4, macOS 15.4, visionOS 2.4, *) +extension ImagePlaygroundStyle { + init?(string: String) { + switch string.lowercased() { + case "animation": self = .animation + case "illustration": self = .illustration + case "sketch": self = .sketch + default: return nil + } + } +} +#endif + +private func ipgPNGData(from cgImage: CGImage) -> Data? { + #if canImport(UIKit) + return UIImage(cgImage: cgImage).pngData() + #elseif canImport(AppKit) && !targetEnvironment(macCatalyst) + let rep = NSBitmapImageRep(cgImage: cgImage) + return rep.representation(using: .png, properties: [:]) + #else + return nil + #endif +} + +enum ImagePlaygroundGeneratorError: Error { + case osTooOld + case generationFailed +} + +struct ImagePlaygroundGenerator { + static func generate(prompt: String, + styleString: String, + limit: Int) async throws -> String { + #if canImport(ImagePlayground) + guard #available(iOS 18.4, macOS 15.4, visionOS 2.4, *) else { + throw ImagePlaygroundGeneratorError.osTooOld + } + + let style = ImagePlaygroundStyle(string: styleString) ?? .animation + let creator = try await ImageCreator() + let seq = creator.images(for: [.text(prompt)], style: style, limit: limit) + + if let first = try await seq.first(where: { _ in true }), + let png = ipgPNGData(from: first.cgImage) { + return png.base64EncodedString() + } + throw ImagePlaygroundGeneratorError.generationFailed + #else + throw ImagePlaygroundGeneratorError.osTooOld + #endif + } +} + +private struct ImagePlaygroundRequestBody: Decodable { + var prompt: String + var style: String + var limit: Int? +} + public class ApiContentLoader { public static func load(contextId: UUID, apiContent: ApiContent) async throws -> ApiContentRack? { + if apiContent.endpoint.hasPrefix("imageplayground://") { + if let data = apiContent.body.data(using: .utf8) { + do { + let body = try JSONDecoder().decode(ImagePlaygroundRequestBody.self, from: data) + let base64 = try await ImagePlaygroundGenerator.generate( + prompt: body.prompt, + styleString: body.style, + limit: body.limit ?? 1 + ) + return ApiContentRack(id: apiContent.id, arguments: ["base64image": base64]) + } catch { + #if DEBUG + print("[ApiContentLoader] ImagePlayground generation failed: \(error)") + #endif + return nil + } + } else { + return nil + } + } + let endpoint = apiContent.endpoint guard let url = URL(string: endpoint) else { From 7c944d2e00e3f51244b92f95443d04880ea87a16 Mon Sep 17 00:00:00 2001 From: Ray Kitajima Date: Fri, 18 Jul 2025 21:17:34 +0900 Subject: [PATCH 2/5] feat: Foundation Models --- Sources/SwiftApiAdapter/Loader.swift | 68 ++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/Sources/SwiftApiAdapter/Loader.swift b/Sources/SwiftApiAdapter/Loader.swift index c58d548..86e7d70 100644 --- a/Sources/SwiftApiAdapter/Loader.swift +++ b/Sources/SwiftApiAdapter/Loader.swift @@ -6,6 +6,10 @@ import SwiftSoup import ImagePlayground #endif +#if canImport(FoundationModels) +import FoundationModels +#endif + #if canImport(UIKit) import UIKit #endif @@ -74,6 +78,48 @@ private struct ImagePlaygroundRequestBody: Decodable { var limit: Int? } +enum FoundationModelsGeneratorError: Error { + case osTooOld + case generationFailed +} + +struct FoundationModelsGenerator { + static func generate(prompt: String, + instructions: String? = nil, + temperature: Double? = nil, + maxTokens: Int? = nil) async throws -> String { + + #if canImport(FoundationModels) + // Create or reuse a session + let session: LanguageModelSession = { + if let instructions, !instructions.isEmpty { + return LanguageModelSession(instructions: instructions) + } else { + return LanguageModelSession() + } + }() + + // Prepare generation options (all are optional) + var opts = GenerationOptions() + if let temperature { opts.temperature = temperature } + if let maxTokens { opts.maximumTokens = maxTokens } + + // Ask the model + let response = try await session.respond(to: prompt, options: opts) + return response.content // :contentReference[oaicite:0]{index=0} + #else + throw FoundationModelsGeneratorError.osTooOld + #endif + } +} + +private struct FoundationModelsRequestBody: Decodable { + let prompt: String + let instructions: String? + let temperature: Double? + let maxTokens: Int? +} + public class ApiContentLoader { public static func load(contextId: UUID, apiContent: ApiContent) async throws -> ApiContentRack? { if apiContent.endpoint.hasPrefix("imageplayground://") { @@ -97,6 +143,28 @@ public class ApiContentLoader { } } + if apiContent.endpoint.hasPrefix("foundationmodels://") { + guard let data = apiContent.body.data(using: .utf8) else { return nil } + do { + let body = try JSONDecoder().decode(FoundationModelsRequestBody.self, from: data) + let generated = try await FoundationModelsGenerator.generate( + prompt: body.prompt, + instructions: body.instructions, + temperature: body.temperature, + maxTokens: body.maxTokens + ) + return ApiContentRack( + id: apiContent.id, + arguments: ["content": generated] + ) + } catch { + #if DEBUG + print("[ApiContentLoader] FoundationModels generation failed: \(error)") + #endif + return nil + } + } + let endpoint = apiContent.endpoint guard let url = URL(string: endpoint) else { From 67f07ecfb615202b96d4dce7529e6e446ec4f43b Mon Sep 17 00:00:00 2001 From: Ray Kitajima Date: Sun, 27 Jul 2025 20:01:46 +0900 Subject: [PATCH 3/5] fix: FoundationModels integration --- Sources/SwiftApiAdapter/Loader.swift | 40 +++++++++++++++++++--------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/Sources/SwiftApiAdapter/Loader.swift b/Sources/SwiftApiAdapter/Loader.swift index 86e7d70..784ed1e 100644 --- a/Sources/SwiftApiAdapter/Loader.swift +++ b/Sources/SwiftApiAdapter/Loader.swift @@ -83,6 +83,7 @@ enum FoundationModelsGeneratorError: Error { case generationFailed } +@available(iOS 26.0, macOS 26.0, visionOS 26.0, *) struct FoundationModelsGenerator { static func generate(prompt: String, instructions: String? = nil, @@ -101,8 +102,8 @@ struct FoundationModelsGenerator { // Prepare generation options (all are optional) var opts = GenerationOptions() - if let temperature { opts.temperature = temperature } - if let maxTokens { opts.maximumTokens = maxTokens } + if let temperature { opts.temperature = temperature } + if let maxTokens { opts.maximumResponseTokens = maxTokens } // Ask the model let response = try await session.respond(to: prompt, options: opts) @@ -145,18 +146,33 @@ public class ApiContentLoader { if apiContent.endpoint.hasPrefix("foundationmodels://") { guard let data = apiContent.body.data(using: .utf8) else { return nil } + do { let body = try JSONDecoder().decode(FoundationModelsRequestBody.self, from: data) - let generated = try await FoundationModelsGenerator.generate( - prompt: body.prompt, - instructions: body.instructions, - temperature: body.temperature, - maxTokens: body.maxTokens - ) - return ApiContentRack( - id: apiContent.id, - arguments: ["content": generated] - ) + + #if canImport(FoundationModels) + if #available(iOS 26.0, macOS 26.0, visionOS 26.0, *) { + let generated = try await FoundationModelsGenerator.generate( + prompt: body.prompt, + instructions: body.instructions, + temperature: body.temperature, + maxTokens: body.maxTokens + ) + + return ApiContentRack( + id: apiContent.id, + arguments: ["content": generated] + ) + } else { + #if DEBUG + print("[ApiContentLoader] FoundationModels not supported on this OS version") + #endif + return nil + } + #else + return nil + #endif + } catch { #if DEBUG print("[ApiContentLoader] FoundationModels generation failed: \(error)") From 1eed46fce2ea8057589f69917d9457a5536ca1c0 Mon Sep 17 00:00:00 2001 From: Ray Kitajima Date: Sun, 27 Jul 2025 20:08:15 +0900 Subject: [PATCH 4/5] cleanup --- Sources/SwiftApiAdapter/Loader.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/SwiftApiAdapter/Loader.swift b/Sources/SwiftApiAdapter/Loader.swift index 784ed1e..8f764ff 100644 --- a/Sources/SwiftApiAdapter/Loader.swift +++ b/Sources/SwiftApiAdapter/Loader.swift @@ -107,7 +107,7 @@ struct FoundationModelsGenerator { // Ask the model let response = try await session.respond(to: prompt, options: opts) - return response.content // :contentReference[oaicite:0]{index=0} + return response.content #else throw FoundationModelsGeneratorError.osTooOld #endif From aa36be7300da48e3aa1921a814bb1d4bbcffee96 Mon Sep 17 00:00:00 2001 From: Ray Kitajima Date: Sun, 27 Jul 2025 21:49:03 +0900 Subject: [PATCH 5/5] fix: return arguments in FM --- Sources/SwiftApiAdapter/Loader.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/SwiftApiAdapter/Loader.swift b/Sources/SwiftApiAdapter/Loader.swift index 8f764ff..7936a50 100644 --- a/Sources/SwiftApiAdapter/Loader.swift +++ b/Sources/SwiftApiAdapter/Loader.swift @@ -161,7 +161,7 @@ public class ApiContentLoader { return ApiContentRack( id: apiContent.id, - arguments: ["content": generated] + arguments: ["text": generated] ) } else { #if DEBUG