Merge pull request #1 from EthanLipnik/main

Fix iOS support, update Readme, and fix gitignore

Author: mortenjust

.gitignore

@@ -2,3 +2,19 @@ maple-diffusion/bins/*.txt
 maple-diffusion/bins/*.bin
 xcuserdata/
 .DS_Store
+.build
+DerivedData
+/.previous-build
+xcuserdata
+*~
+\#*
+.\#*
+.*.sw[nop]
+*.xcscmblueprint
+/default.profraw
+Utilities/Docker/*.tar.gz
+.swiftpm
+Package.resolved
+/build
+*.pyc
+.docc-build

MapleDiffusion/.swiftpm/xcode/package.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist

@@ -5,7 +5,7 @@ import PackageDescription
 
 let package = Package(
     name: "MapleDiffusion",
-    platforms: [ .macOS(.v12) ],
+    platforms: [.iOS(.v15), .macOS(.v12)],
 
     products: [
         // Products define the executables and libraries a package produces, and make them visible to other packages.

Package.swift

@@ -4,30 +4,57 @@ Maple Diffusion runs Stable Diffusion models **locally** on macOS / iOS devices,
 
 ![](demonstration.jpg)
 
-Maple Diffusion should be capable of generating a reasonable image in a minute or two on a recent iPhone (I get around ~2.3s / step on an iPhone 13 Pro).
+Maple Diffusion should be capable of generating a reasonable image [in a minute or two](https://twitter.com/madebyollin/status/1579213789823893504) on a recent iPhone (I get around ~2.3s / step on an iPhone 13 Pro).
 
 To attain usable performance without tripping over iOS's 4GB memory limit, Maple Diffusion relies internally on FP16 (NHWC) tensors, operator fusion from MPSGraph, and a truly pitiable degree of swapping models to device storage.
 
 On macOS, Maple Diffusion uses slightly more memory (~6GB), to reach <1s / step.
 
 ![](screenshot.jpg)
 
+# Projects using Maple Diffusion
+
+* https://github.com/mortenjust/maple-diffusion/ is a fork with several improvements
+
+# Device Requirements
+
+Maple Diffusion should run on any Apple Silicon Mac (M1, M2, etc.). Intel macs seem to not support Float16 or something.
+
+Maple Diffusion should run on any iOS device with [sufficient RAM](https://blakespot.com/ios_device_specifications_grid.html) (≥6144MB RAM definitely works; 4096MB *might* but I wouldn't bet on it; anything lower than that won't work). Recent iPads should work out of the box; recent iPhones should work if you can get the `Increase Memory Limit` capability working (to unlock 4GB of RAM).
+
+iOS 16.1 (beta) is reportedly broken and always generating a gray image :(
+
 # Usage
 
 To build and run Maple Diffusion:
 
-1. Download a Stable Diffusion model checkpoint ([`sd-v1-4.ckpt`](https://huggingface.co/CompVis/stable-diffusion-v1-4), or some derivation thereof)
+1. Download a Stable Diffusion PyTorch model checkpoint ([`sd-v1-4.ckpt`](https://huggingface.co/CompVis/stable-diffusion-v1-4), or some derivation thereof)
 
 2. Download this repo
 
    ```bash
    git clone https://github.com/madebyollin/maple-diffusion.git && cd maple-diffusion
    ```
 
-3. Convert the model into a bunch of fp16 binary blobs. You might need to install PyTorch and stuff.
+3. Setup & install Python with PyTorch, if you haven't already.
+
+   ```bash
+   # may need to install conda first https://github.com/conda-forge/miniforge#homebrew
+   conda deactivate
+   conda remove -n maple-diffusion --all
+   conda create -n maple-diffusion python=3.10
+   conda activate maple-diffusion
+   pip install torch typing_extensions numpy Pillow requests pytorch_lightning
+   ```
+
+4. Convert the PyTorch model checkpoint into a bunch of fp16 binary blobs.
 
    ```bash
    ./maple-convert.py ~/Downloads/sd-v1-4.ckpt
    ```
 
-4. Open, build, and run the `maple-diffusion` Xcode project. You might need to set up code signing and stuff
+5. Open the `maple-diffusion` Xcode project. Select the device you want to run on from the `Product > Destination` menu.
+
+6. [Manually add](https://github.com/madebyollin/maple-diffusion/issues/5#issuecomment-1279111878) the `Increased Memory Limit` capability to the `maple-diffusion` target (this step might not be needed on iPads, but it's definitely needed on iPhones - the default limit is 3GB).
+
+7. Build & run the project on your device with the `Product > Run` menu.

README.md

@@ -44,7 +44,7 @@ func makeUpsampleNearest(graph: MPSGraph, xIn: MPSGraphTensor, scaleFactor: Int=
     return graph.resize(xIn, size: [NSNumber(value:xIn.shape![1].intValue * scaleFactor), NSNumber(value:xIn.shape![2].intValue * scaleFactor)], mode: MPSGraphResizeMode.nearest, centerResult: true, alignCorners: false, layout: MPSGraphTensorNamedDataLayout.NHWC, name: nil)
 }
 
-@available(macOS 12.3, *)
+@available(iOS 15.4, macOS 12.3, *)
 func makeGroupNorm(graph: MPSGraph, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {
     var x = xIn
     if (xIn.shape!.count == 3) {
@@ -66,12 +66,12 @@ func makeSwish(graph: MPSGraph, xIn: MPSGraphTensor) -> MPSGraphTensor {
     return graph.multiplication(xIn, graph.sigmoid(with: xIn, name: nil), name: nil)
 }
 
-@available(macOS 12.3, *)
+@available(iOS 15.4, macOS 12.3, *)
 func makeGroupNormSwish(graph: MPSGraph, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {
     return makeSwish(graph: graph, xIn: makeGroupNorm(graph: graph, xIn: xIn, name: name))
 }
 
-@available(macOS 12.3, *)
+@available(iOS 15.4, macOS 12.3, *)
 func makeDecoderResBlock(graph: MPSGraph, xIn: MPSGraphTensor, name: String, outChannels: NSNumber) -> MPSGraphTensor {
     var x = xIn
     x = makeGroupNormSwish(graph: graph, xIn: x, name: name + ".norm1")
@@ -85,7 +85,7 @@ func makeDecoderResBlock(graph: MPSGraph, xIn: MPSGraphTensor, name: String, out
     return graph.addition(x, xIn, name: "skip")
 }
 
-@available(macOS 12.3, *)
+@available(iOS 15.4, macOS 12.3, *)
 func makeDecoderAttention(graph: MPSGraph, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {
     var x = makeGroupNorm(graph: graph, xIn: xIn, name: name + ".norm")
     let c = x.shape![3]
@@ -113,7 +113,7 @@ func makeByteConverter(graph: MPSGraph, xIn: MPSGraphTensor) -> MPSGraphTensor {
     return graph.concatTensors([x, alpha], dimension: 3, name: nil)
 }
 
-@available(macOS 12.3, *)
+@available(iOS 15.4, macOS 12.3, *)
 func makeDecoder(graph: MPSGraph, xIn: MPSGraphTensor) -> MPSGraphTensor {
     var x = xIn
     let name = "first_stage_model.decoder"
@@ -185,7 +185,7 @@ func makeTimeEmbed(graph: MPSGraph, xIn: MPSGraphTensor, name: String) -> MPSGra
     return makeLinear(graph: graph, xIn: x, name: name + ".2", outChannels: 1280)
 }
 
-@available(macOS 12.3, *)
+@available(iOS 15.4, macOS 12.3, *)
 func makeUNetResBlock(graph: MPSGraph, xIn: MPSGraphTensor, embIn: MPSGraphTensor, name: String, inChannels: NSNumber, outChannels: NSNumber) -> MPSGraphTensor {
     var x = xIn
     x = makeGroupNormSwish(graph: graph, xIn: x, name: name + ".in_layers.0")
@@ -286,7 +286,7 @@ func makeBasicTransformerBlock(graph: MPSGraph, xIn: MPSGraphTensor, name: Strin
     return graph.addition(ff, x, name: nil)
 }
 
-@available(macOS 12.3, *)
+@available(iOS 15.4, macOS 12.3, *)
 func makeSpatialTransformerBlock(graph: MPSGraph, xIn: MPSGraphTensor, name: String, contextIn: MPSGraphTensor, saveMemory: Bool) -> MPSGraphTensor {
     let n, h, w, c: NSNumber
     (n, h, w, c) = (xIn.shape![0], xIn.shape![1], xIn.shape![2], xIn.shape![3])
@@ -300,7 +300,7 @@ func makeSpatialTransformerBlock(graph: MPSGraph, xIn: MPSGraphTensor, name: Str
     return graph.addition(x, xIn, name: nil)
 }
 
-@available(macOS 12.3, *)
+@available(iOS 15.4, macOS 12.3, *)
 func makeOutputBlock(graph: MPSGraph, xIn: MPSGraphTensor, embIn: MPSGraphTensor, condIn: MPSGraphTensor, inChannels: NSNumber, outChannels: NSNumber, dHead: NSNumber, name: String, saveMemory: Bool, spatialTransformer: Bool = true, upsample: Bool = false) -> MPSGraphTensor {
     var x = xIn
     x = makeUNetResBlock(graph: graph, xIn: x, embIn: embIn, name: name + ".0", inChannels: inChannels, outChannels: outChannels)
@@ -315,7 +315,7 @@ func makeOutputBlock(graph: MPSGraph, xIn: MPSGraphTensor, embIn: MPSGraphTensor
 }
 
 
-@available(macOS 12.3, *)
+@available(iOS 15.4, macOS 12.3, *)
 func makeUNetAnUnexpectedJourney(graph: MPSGraph, xIn: MPSGraphTensor, tembIn: MPSGraphTensor, condIn: MPSGraphTensor, name: String, saveMemory: Bool = true) -> [MPSGraphTensor] {
     let emb = makeTimeEmbed(graph: graph, xIn: tembIn, name: name + ".time_embed")
 
@@ -382,7 +382,7 @@ func makeUNetAnUnexpectedJourney(graph: MPSGraph, xIn: MPSGraphTensor, tembIn: M
     return savedInputs + [emb] + [x]
 }
 
-@available(macOS 12.3, *)
+@available(iOS 15.4, macOS 12.3, *)
 func makeUNetTheDesolationOfSmaug(graph: MPSGraph, savedInputsIn: [MPSGraphTensor], name: String, saveMemory: Bool = true) -> [MPSGraphTensor] {
     var savedInputs = savedInputsIn
     let condIn = savedInputs.popLast()!
@@ -408,7 +408,7 @@ func makeUNetTheDesolationOfSmaug(graph: MPSGraph, savedInputsIn: [MPSGraphTenso
     return savedInputs + [emb] + [x]
 }
 
-@available(macOS 12.3, *)
+@available(iOS 15.4, macOS 12.3, *)
 func makeUNetTheBattleOfTheFiveArmies(graph: MPSGraph, savedInputsIn: [MPSGraphTensor], name: String, saveMemory: Bool = true) -> MPSGraphTensor {
     var savedInputs = savedInputsIn
     let condIn = savedInputs.popLast()!
@@ -456,7 +456,7 @@ func makeSqrtOneMinus(graph: MPSGraph, xIn: MPSGraphTensor) -> MPSGraphTensor {
     return graph.squareRoot(with: graph.subtraction(graph.constant(1.0, dataType: MPSDataType.float16), xIn, name: nil), name: nil)
 }
 
-@available(macOS 12.3, *)
+@available(iOS 15.4, macOS 12.3, *)
 func makeDiffusionStep(graph: MPSGraph, xIn: MPSGraphTensor, etaUncondIn: MPSGraphTensor, etaCondIn: MPSGraphTensor, tIn: MPSGraphTensor, tPrevIn: MPSGraphTensor, guidanceScaleIn: MPSGraphTensor) -> MPSGraphTensor {
 
     // superconditioning
@@ -530,7 +530,7 @@ func makeTextEncoder(graph: MPSGraph, xIn: MPSGraphTensor, name: String) -> MPSG
     return x
 }
 
-@available(macOS 12.3, *)
+@available(iOS 15.4, macOS 12.3, *)
 func makeTextEmbeddings(graph: MPSGraph, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {
     var tokenEmbeddings = loadConstant(graph: graph, name: name + ".token_embedding.weight", shape: [1, 49408, 768])
     tokenEmbeddings = graph.broadcast(tokenEmbeddings, shape: [2, 49408, 768], name: nil)
@@ -540,7 +540,7 @@ func makeTextEmbeddings(graph: MPSGraph, xIn: MPSGraphTensor, name: String) -> M
     return graph.addition(embeddings, positionEmbeddings, name: nil)
 }
 
-@available(macOS 12.3, *)
+@available(iOS 15.4, macOS 12.3, *)
 func makeTextGuidance(graph: MPSGraph, xIn: MPSGraphTensor, name: String) -> MPSGraphTensor {
     var x = makeTextEmbeddings(graph: graph, xIn: xIn, name: name + ".embeddings")
     x = makeTextEncoder(graph: graph, xIn: x, name: name + ".encoder")

Sources/MapleDiffusion/Functions.swift

@@ -1,7 +1,6 @@
 
 import MetalPerformanceShadersGraph
 import Foundation
-import AppKit
 import Combine
 
 
@@ -34,7 +33,7 @@ public struct ProgressMonitor : Equatable {
 
 }
 
-@available(macOS 12.3, *)
+@available(iOS 15.4, macOS 12.3, *)
 public class MapleDiffusion : ObservableObject {
     @Published public var isModelLoaded = false