Use CoreML on Apple platforms

Author: exPHAT

Package.swift

@@ -9,7 +9,9 @@ let package = Package(
  targets: [
  .target(name: "SwiftWhisper", dependencies: [.target(name: "whisper_cpp")]),
  .target(name: "whisper_cpp", cSettings: [
- .define("GGML_USE_ACCELERATE", .when(platforms: [.macOS, .macCatalyst, .iOS]))
+ .define("GGML_USE_ACCELERATE", .when(platforms: [.macOS, .macCatalyst, .iOS])),
+ .define("WHISPER_USE_COREML", .when(platforms: [.macOS, .macCatalyst, .iOS])),
+ .define("WHISPER_COREML_ALLOW_FALLBACK", .when(platforms: [.macOS, .macCatalyst, .iOS]))
  ]),
  .testTarget(name: "WhisperTests", dependencies: [.target(name: "SwiftWhisper")], resources: [.copy("TestResources/")])
  ],

README.md

@@ -72,7 +72,12 @@ protocol WhisperDelegate {
 
 ### Downloading Models :inbox_tray:
 
-You can find the pre-trained models at [here](https://huggingface.co/ggerganov/whisper.cpp) for download.
+You can find the pre-trained models [here](https://huggingface.co/ggerganov/whisper.cpp) for download.
+
+### CoreML Support :brain:
+
+If a CoreML model under the same name as the normal model can be found with the suffix altered to `-encoder.mlmodelc` (Example: `tiny.bin` would also sit beside a `tiny-encoder.mlmodelc` file), and you are using the `Whisper(fromFileURL:)` initializer, then the CoreML will be used for transcription! You can verify it's active by checking the console output. 
+ 
 
 ### Converting audio to 16kHz PCM :wrench:
 
@@ -113,7 +118,7 @@ func convertAudioFileToPCMArray(fileURL: URL, completionHandler: @escaping (Resu
 }
 ```
 
-### Speed boost :rocket:
+### Development speed boost :rocket:
 
 You may find the performance of the transcription slow when compiling your app for the `Debug` build configuration. This is because the compiler doesn't fully optimize SwiftWhisper unless the build configuration is set to `Release`.
 

Sources/whisper_cpp/coreml/whisper-encoder-impl.h

@@ -0,0 +1 @@
+../../../whisper.cpp/coreml/whisper-encoder-impl.h

Sources/whisper_cpp/coreml/whisper-encoder-impl.m

@@ -0,0 +1 @@
+../../../whisper.cpp/coreml/whisper-encoder-impl.m

Sources/whisper_cpp/coreml/whisper-encoder.h

@@ -0,0 +1 @@
+../../../whisper.cpp/coreml/whisper-encoder.h

Sources/whisper_cpp/coreml/whisper-encoder.mm

@@ -0,0 +1 @@
+../../../whisper.cpp/coreml/whisper-encoder.mm

Tests/WhisperTests/TestResources/tiny-encoder.mlmodelc/metadata.json

@@ -0,0 +1,64 @@
+[
+ {
+ "metadataOutputVersion" : "3.0",
+ "storagePrecision" : "Float16",
+ "outputSchema" : [
+ {
+ "hasShapeFlexibility" : "0",
+ "isOptional" : "0",
+ "dataType" : "Float32",
+ "formattedType" : "MultiArray (Float32)",
+ "shortDescription" : "",
+ "shape" : "[]",
+ "name" : "output",
+ "type" : "MultiArray"
+ }
+ ],
+ "modelParameters" : [
+
+ ],
+ "specificationVersion" : 6,
+ "mlProgramOperationTypeHistogram" : {
+ "Linear" : 24,
+ "Matmul" : 8,
+ "Cast" : 2,
+ "Conv" : 2,
+ "Softmax" : 4,
+ "Add" : 9,
+ "LayerNorm" : 9,
+ "Mul" : 8,
+ "Transpose" : 17,
+ "Gelu" : 6,
+ "Reshape" : 16
+ },
+ "computePrecision" : "Mixed (Float16, Float32, Int32)",
+ "isUpdatable" : "0",
+ "availability" : {
+ "macOS" : "12.0",
+ "tvOS" : "15.0",
+ "watchOS" : "8.0",
+ "iOS" : "15.0",
+ "macCatalyst" : "15.0"
+ },
+ "modelType" : {
+ "name" : "MLModelType_mlProgram"
+ },
+ "userDefinedMetadata" : {
+
+ },
+ "inputSchema" : [
+ {
+ "hasShapeFlexibility" : "0",
+ "isOptional" : "0",
+ "dataType" : "Float32",
+ "formattedType" : "MultiArray (Float32 1 × 80 × 3000)",
+ "shortDescription" : "",
+ "shape" : "[1, 80, 3000]",
+ "name" : "logmel_data",
+ "type" : "MultiArray"
+ }
+ ],
+ "generatedClassName" : "coreml_encoder_tiny",
+ "method" : "predict"
+ }
+]