Merged PR 48123: Release 0.14.35.3

Release 0.14.35.1

Directory.Build.props

@@ -6,7 +6,7 @@
     <Company>Microsoft Corporation</Company>
     <Owners>microsoft,psi</Owners>
     <Authors>Microsoft</Authors>
-    <AssemblyVersion>0.13.38.2</AssemblyVersion>
+    <AssemblyVersion>0.14.35.3</AssemblyVersion>
     <FileVersion>$(AssemblyVersion)</FileVersion>
     <Version>$(AssemblyVersion)-beta</Version>
     <SignAssembly>false</SignAssembly>

Sources/Audio/Microsoft.Psi.Audio/WaveFileStreamReader.cs

@@ -6,6 +6,7 @@ namespace Microsoft.Psi.Audio
     using System;
     using System.Collections.Generic;
     using System.IO;
+    using System.Runtime.Serialization;
     using System.Threading;
     using Microsoft.Psi;
     using Microsoft.Psi.Data;
@@ -160,7 +161,7 @@ public IStreamReader OpenNew()
         }
 
         /// <inheritdoc />
-        public IStreamMetadata OpenStream<T>(string name, Action<T, Envelope> target, Func<T> allocator = null)
+        public IStreamMetadata OpenStream<T>(string name, Action<T, Envelope> target, Func<T> allocator = null, Action<SerializationException> errorHandler = null)
         {
             ValidateStreamName(name);
 

Sources/Calibration/Microsoft.Psi.Calibration/ProjectTo3D.cs

@@ -20,7 +20,7 @@ public sealed class ProjectTo3D : ConsumerProducer<(Shared<DepthImage>, List<Poi
         /// <summary>
         /// Initializes a new instance of the <see cref="ProjectTo3D"/> class.
         /// </summary>
-        /// <param name="pipeline">Pipeline this component is a part of.</param>
+        /// <param name="pipeline">The pipeline to add the component to.</param>
         public ProjectTo3D(Pipeline pipeline)
             : base(pipeline)
         {

Sources/Data/Microsoft.Psi.Data/Json/JsonGenerator.cs

@@ -22,7 +22,7 @@ public class JsonGenerator : Generator, IDisposable
         /// <summary>
         /// Initializes a new instance of the <see cref="JsonGenerator"/> class.
         /// </summary>
-        /// <param name="pipeline">Pipeline this component is a part of.</param>
+        /// <param name="pipeline">The pipeline to add the component to.</param>
         /// <param name="name">The name of the application that generated the persisted files, or the root name of the files.</param>
         /// <param name="path">The directory in which the main persisted file resides.</param>
         public JsonGenerator(Pipeline pipeline, string name, string path)
@@ -33,7 +33,7 @@ public JsonGenerator(Pipeline pipeline, string name, string path)
         /// <summary>
         /// Initializes a new instance of the <see cref="JsonGenerator"/> class.
         /// </summary>
-        /// <param name="pipeline">Pipeline this component is a part of.</param>
+        /// <param name="pipeline">The pipeline to add the component to.</param>
         /// <param name="reader">The underlying store reader.</param>
         protected JsonGenerator(Pipeline pipeline, JsonStoreReader reader)
             : base(pipeline)

Sources/Data/Microsoft.Psi.Data/Json/JsonStreamReader.cs

@@ -5,6 +5,7 @@ namespace Microsoft.Psi.Data.Json
 {
     using System;
     using System.Collections.Generic;
+    using System.Runtime.Serialization;
     using System.Threading;
     using Microsoft.Psi;
     using Newtonsoft.Json.Linq;
@@ -125,7 +126,7 @@ public virtual IStreamReader OpenNew()
         }
 
         /// <inheritdoc />
-        public IStreamMetadata OpenStream<T>(string streamName, Action<T, Envelope> target, Func<T> allocator = null)
+        public IStreamMetadata OpenStream<T>(string streamName, Action<T, Envelope> target, Func<T> allocator = null, Action<SerializationException> errorHandler = null)
         {
             if (string.IsNullOrWhiteSpace(streamName))
             {

Sources/Imaging/Microsoft.Psi.Imaging/ImageTransformer.cs

@@ -25,7 +25,7 @@ public class ImageTransformer : ConsumerProducer<Shared<Image>, Shared<Image>>
         /// <summary>
         /// Initializes a new instance of the <see cref="ImageTransformer"/> class.
         /// </summary>
-        /// <param name="pipeline">Pipeline this component is a part of.</param>
+        /// <param name="pipeline">The pipeline to add the component to.</param>
         /// <param name="transformer">Function for transforming the source image.</param>
         /// <param name="pixelFormat">Pixel format for destination image.</param>
         /// <param name="sharedImageAllocator ">Optional image allocator for creating new shared image.</param>

Sources/Imaging/Test.Psi.Imaging.Windows/Properties/AssemblyInfo.cs

@@ -10,6 +10,6 @@
 [assembly: AssemblyCopyright("Copyright (c) Microsoft Corporation. All rights reserved.")]
 [assembly: ComVisible(false)]
 [assembly: Guid("191df615-3d8f-45a3-b763-dd4a604a712a")]
-[assembly: AssemblyVersion("0.13.38.2")]
-[assembly: AssemblyFileVersion("0.13.38.2")]
-[assembly: AssemblyInformationalVersion("0.13.38.2-beta")]
+[assembly: AssemblyVersion("0.14.35.3")]
+[assembly: AssemblyFileVersion("0.14.35.3")]
+[assembly: AssemblyInformationalVersion("0.14.35.3-beta")]

Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/ImageAnalyzer.cs

@@ -238,7 +238,7 @@ private async Task ReceiveAsync(Shared<Image> data, Envelope e)
         {
             var analysisResult = default(ImageAnalysis);
 
-            if (data != null)
+            if (data != null && data.Resource != null)
             {
                 using Stream imageFileStream = new MemoryStream();
 

Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelOutputParser.cs

@@ -0,0 +1,73 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+namespace Microsoft.Psi.Onnx
+{
+    using System;
+    using System.Collections.Generic;
+    using System.IO;
+    using System.Linq;
+    using Microsoft.Psi;
+
+    /// <summary>
+    /// Internal class that parses the outputs from the ImageNet model into
+    /// a set of image classification results.
+    /// </summary>
+    internal class ImageNetModelOutputParser
+    {
+        private readonly string[] labels;
+        private readonly int maxPredictions;
+        private readonly bool applySoftmax;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ImageNetModelOutputParser"/> class.
+        /// </summary>
+        /// <param name="imageClassesFile">The path to the file containing the list of 1000 ImageNet classes.</param>
+        /// <param name="maxPredictions">The maximum number of predictions to return.</param>
+        /// <param name="applySoftmax">Whether the softmax function should be applied to the raw model output.</param>
+        /// <remarks>
+        /// The file referenced by <paramref name="imageClassesFile"/> may be downloaded from the following location:
+        /// https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt.
+        /// </remarks>
+        public ImageNetModelOutputParser(string imageClassesFile, int maxPredictions, bool applySoftmax)
+        {
+            this.labels = File.ReadAllLines(imageClassesFile);
+            if (this.labels.Length != 1000)
+            {
+                throw new ArgumentException($"The file {imageClassesFile} does not appear to be in the correct format. This file should contain exactly 1000 lines representing an ordered list of the 1000 ImageNet classes.");
+            }
+
+            this.maxPredictions = maxPredictions;
+            this.applySoftmax = applySoftmax;
+        }
+
+        /// <summary>
+        /// Gets the predictions from the model output.
+        /// </summary>
+        /// <param name="modelOutput">The model output vector of class probabilities.</param>
+        /// <returns>A list of the top-N predictions, in descending probability order.</returns>
+        public List<LabeledPrediction> GetPredictions(float[] modelOutput)
+        {
+            return GetTopResults(this.applySoftmax ? Softmax(modelOutput) : modelOutput, this.maxPredictions)
+                .Select(c => new LabeledPrediction { Label = this.labels[c.Index], Confidence = c.Value })
+                .ToList();
+        }
+
+        private static IEnumerable<(int Index, float Value)> GetTopResults(IEnumerable<float> predictedClasses, int count)
+        {
+            return predictedClasses
+                .Select((predictedClass, index) => (Index: index, Value: predictedClass))
+                .OrderByDescending(result => result.Value)
+                .Take(count);
+        }
+
+        private static IEnumerable<float> Softmax(IEnumerable<float> values)
+        {
+            var maxVal = values.Max();
+            var exp = values.Select(v => Math.Exp(v - maxVal));
+            var sumExp = exp.Sum();
+
+            return exp.Select(v => (float)(v / sumExp));
+        }
+    }
+}

Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunner.cs

@@ -0,0 +1,141 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+namespace Microsoft.Psi.Onnx
+{
+    using System;
+    using System.Collections.Generic;
+    using Microsoft.Psi;
+    using Microsoft.Psi.Components;
+    using Microsoft.Psi.Imaging;
+
+    /// <summary>
+    /// Component that runs an ImageNet image classification model.
+    /// </summary>
+    /// <remarks>
+    /// This class implements a \psi component that runs an ONNX model trained
+    /// on the ImageNet dataset that operates on 224x224 RGB images and scores
+    /// the image for each of the 1000 ImageNet classes. It takes an input
+    /// stream of \psi images, applies a center-crop, rescales and normalizes
+    /// the pixel values into the input vector expected by the model. It also
+    /// parses the model outputs into a list of <see cref="LabeledPrediction"/>
+    /// values, corresponding to the top N predictions by the model. For
+    /// convenience, a set of pre-defined model runner configurations are
+    /// defined for a number of image classification models available in the
+    /// ONNX Model Zoo (https://github.com/onnx/models/tree/master/vision/classification).
+    /// The ONNX model file for the corresponding configuration will need to be
+    /// downloaded locally and the path to the model file will need to be
+    /// specified when creating the configuration.
+    /// </remarks>
+    public class ImageNetModelRunner : ConsumerProducer<Shared<Image>, List<LabeledPrediction>>
+    {
+        private readonly float[] onnxInputVector = new float[3 * 224 * 224];
+        private readonly OnnxModel onnxModel;
+        private readonly ImageNetModelOutputParser outputParser;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ImageNetModelRunner"/> class.
+        /// </summary>
+        /// <param name="pipeline">The pipeline to add the component to.</param>
+        /// <param name="configuration">The configuration for the compoinent.</param>
+        /// <remarks>
+        /// To run on a GPU, use the Microsoft.Psi.Onnx.ModelRunners.Gpu library instead of Microsoft.Psi.Onnx.ModelRunners.Cpu, and set
+        /// the value of the <pararef name="gpuDeviceId"/> parameter to a valid non-negative integer. Typical device ID values are 0 or 1.
+        /// </remarks>
+        public ImageNetModelRunner(Pipeline pipeline, ImageNetModelRunnerConfiguration configuration)
+            : base(pipeline)
+        {
+            // create an ONNX model based on the supplied ImageNet model runner configuration
+            this.onnxModel = new OnnxModel(new OnnxModelConfiguration()
+            {
+                ModelFileName = configuration.ModelFilePath,
+                InputVectorName = configuration.InputVectorName,
+                InputVectorSize = 3 * 224 * 224,
+                OutputVectorName = configuration.OutputVectorName,
+                GpuDeviceId = configuration.GpuDeviceId,
+            });
+
+            this.outputParser = new ImageNetModelOutputParser(configuration.ImageClassesFilePath, configuration.NumberOfPredictions, configuration.ApplySoftmaxToModelOutput);
+        }
+
+        /// <inheritdoc/>
+        protected override void Receive(Shared<Image> data, Envelope envelope)
+        {
+            // construct the ONNX model input vector (stored in this.onnxInputVector)
+            // based on the incoming image
+            this.ConstructOnnxInputVector(data);
+
+            // run the model over the input vector
+            var outputVector = this.onnxModel.GetPrediction(this.onnxInputVector);
+
+            // parse the model output into an ordered list of the top-N predictions
+            var results = this.outputParser.GetPredictions(outputVector);
+
+            // post the results
+            this.Out.Post(results, envelope.OriginatingTime);
+        }
+
+        /// <summary>
+        /// Constructs the input vector for the ImageNet model for a specified image.
+        /// </summary>
+        /// <param name="sharedImage">The image to construct the input vector for.</param>
+        private void ConstructOnnxInputVector(Shared<Image> sharedImage)
+        {
+            var inputImage = sharedImage.Resource;
+            var inputWidth = sharedImage.Resource.Width;
+            var inputHeight = sharedImage.Resource.Height;
+
+            // crop a center square
+            var squareSize = Math.Min(inputWidth, inputHeight);
+            using var squareImage = ImagePool.GetOrCreate(squareSize, squareSize, sharedImage.Resource.PixelFormat);
+            if (inputWidth > inputHeight)
+            {
+                inputImage.Crop(squareImage.Resource, (inputWidth - squareSize) / 2, 0, squareSize, squareSize);
+            }
+            else
+            {
+                inputImage.Crop(squareImage.Resource, 0, (inputHeight - squareSize) / 2, squareSize, squareSize);
+            }
+
+            // resize the image to 224 x 224
+            using var resizedImage = ImagePool.GetOrCreate(224, 224, sharedImage.Resource.PixelFormat);
+            squareImage.Resource.Resize(resizedImage.Resource, 224, 224, SamplingMode.Bilinear);
+
+            // if the pixel format does not match, do a conversion before extracting the bytes
+            var bytes = default(byte[]);
+            if (sharedImage.Resource.PixelFormat != PixelFormat.BGR_24bpp)
+            {
+                using var reformattedImage = ImagePool.GetOrCreate(224, 224, PixelFormat.BGR_24bpp);
+                resizedImage.Resource.CopyTo(reformattedImage.Resource);
+                bytes = reformattedImage.Resource.ReadBytes(3 * 224 * 224);
+            }
+            else
+            {
+                // get the bytes
+                bytes = resizedImage.Resource.ReadBytes(3 * 224 * 224);
+            }
+
+            // Now populate the onnxInputVector float array / tensor by normalizing
+            // using mean = [0.485, 0.456, 0.406] and std = [0.229, 0.224, 0.225].
+            int fi = 0;
+
+            // first the red bytes
+            for (int i = 2; i < bytes.Length; i += 3)
+            {
+                this.onnxInputVector[fi++] = ((bytes[i] / 255.0f) - 0.485f) / 0.229f;
+            }
+
+            // then the green bytes
+            for (int i = 1; i < bytes.Length; i += 3)
+            {
+                this.onnxInputVector[fi++] = ((bytes[i] / 255.0f) - 0.456f) / 0.224f;
+            }
+
+            // then the blue bytes
+            for (int i = 0; i < bytes.Length; i += 3)
+            {
+                this.onnxInputVector[fi++] = ((bytes[i] / 255.0f) - 0.406f) / 0.225f;
+            }
+        }
+    }
+}