diff --git a/Directory.Build.props b/Directory.Build.props
index f16615819..4ab8566be 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -6,7 +6,7 @@
Microsoft Corporation
microsoft,psi
Microsoft
- 0.13.38.2
+ 0.14.35.3
$(AssemblyVersion)
$(AssemblyVersion)-beta
false
diff --git a/Sources/Audio/Microsoft.Psi.Audio/WaveFileStreamReader.cs b/Sources/Audio/Microsoft.Psi.Audio/WaveFileStreamReader.cs
index 2e00efa61..6804a52e5 100644
--- a/Sources/Audio/Microsoft.Psi.Audio/WaveFileStreamReader.cs
+++ b/Sources/Audio/Microsoft.Psi.Audio/WaveFileStreamReader.cs
@@ -6,6 +6,7 @@ namespace Microsoft.Psi.Audio
using System;
using System.Collections.Generic;
using System.IO;
+ using System.Runtime.Serialization;
using System.Threading;
using Microsoft.Psi;
using Microsoft.Psi.Data;
@@ -160,7 +161,7 @@ public IStreamReader OpenNew()
}
///
- public IStreamMetadata OpenStream(string name, Action target, Func allocator = null)
+ public IStreamMetadata OpenStream(string name, Action target, Func allocator = null, Action errorHandler = null)
{
ValidateStreamName(name);
diff --git a/Sources/Calibration/Microsoft.Psi.Calibration/ProjectTo3D.cs b/Sources/Calibration/Microsoft.Psi.Calibration/ProjectTo3D.cs
index 4f4569e77..8603bcaaf 100644
--- a/Sources/Calibration/Microsoft.Psi.Calibration/ProjectTo3D.cs
+++ b/Sources/Calibration/Microsoft.Psi.Calibration/ProjectTo3D.cs
@@ -20,7 +20,7 @@ public sealed class ProjectTo3D : ConsumerProducer<(Shared, List
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
public ProjectTo3D(Pipeline pipeline)
: base(pipeline)
{
diff --git a/Sources/Data/Microsoft.Psi.Data/Json/JsonGenerator.cs b/Sources/Data/Microsoft.Psi.Data/Json/JsonGenerator.cs
index bc3bed80a..1cb6c04ce 100644
--- a/Sources/Data/Microsoft.Psi.Data/Json/JsonGenerator.cs
+++ b/Sources/Data/Microsoft.Psi.Data/Json/JsonGenerator.cs
@@ -22,7 +22,7 @@ public class JsonGenerator : Generator, IDisposable
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
/// The name of the application that generated the persisted files, or the root name of the files.
/// The directory in which the main persisted file resides.
public JsonGenerator(Pipeline pipeline, string name, string path)
@@ -33,7 +33,7 @@ public JsonGenerator(Pipeline pipeline, string name, string path)
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
/// The underlying store reader.
protected JsonGenerator(Pipeline pipeline, JsonStoreReader reader)
: base(pipeline)
diff --git a/Sources/Data/Microsoft.Psi.Data/Json/JsonStreamReader.cs b/Sources/Data/Microsoft.Psi.Data/Json/JsonStreamReader.cs
index 7fbcfc7b3..27b9f80ba 100644
--- a/Sources/Data/Microsoft.Psi.Data/Json/JsonStreamReader.cs
+++ b/Sources/Data/Microsoft.Psi.Data/Json/JsonStreamReader.cs
@@ -5,6 +5,7 @@ namespace Microsoft.Psi.Data.Json
{
using System;
using System.Collections.Generic;
+ using System.Runtime.Serialization;
using System.Threading;
using Microsoft.Psi;
using Newtonsoft.Json.Linq;
@@ -125,7 +126,7 @@ public virtual IStreamReader OpenNew()
}
///
- public IStreamMetadata OpenStream(string streamName, Action target, Func allocator = null)
+ public IStreamMetadata OpenStream(string streamName, Action target, Func allocator = null, Action errorHandler = null)
{
if (string.IsNullOrWhiteSpace(streamName))
{
diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/ImageTransformer.cs b/Sources/Imaging/Microsoft.Psi.Imaging/ImageTransformer.cs
index 25d84d551..7ab8544b7 100644
--- a/Sources/Imaging/Microsoft.Psi.Imaging/ImageTransformer.cs
+++ b/Sources/Imaging/Microsoft.Psi.Imaging/ImageTransformer.cs
@@ -25,7 +25,7 @@ public class ImageTransformer : ConsumerProducer, Shared>
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
/// Function for transforming the source image.
/// Pixel format for destination image.
/// Optional image allocator for creating new shared image.
diff --git a/Sources/Imaging/Test.Psi.Imaging.Windows/Properties/AssemblyInfo.cs b/Sources/Imaging/Test.Psi.Imaging.Windows/Properties/AssemblyInfo.cs
index 3d256e184..74ac24e7d 100644
--- a/Sources/Imaging/Test.Psi.Imaging.Windows/Properties/AssemblyInfo.cs
+++ b/Sources/Imaging/Test.Psi.Imaging.Windows/Properties/AssemblyInfo.cs
@@ -10,6 +10,6 @@
[assembly: AssemblyCopyright("Copyright (c) Microsoft Corporation. All rights reserved.")]
[assembly: ComVisible(false)]
[assembly: Guid("191df615-3d8f-45a3-b763-dd4a604a712a")]
-[assembly: AssemblyVersion("0.13.38.2")]
-[assembly: AssemblyFileVersion("0.13.38.2")]
-[assembly: AssemblyInformationalVersion("0.13.38.2-beta")]
+[assembly: AssemblyVersion("0.14.35.3")]
+[assembly: AssemblyFileVersion("0.14.35.3")]
+[assembly: AssemblyInformationalVersion("0.14.35.3-beta")]
diff --git a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/ImageAnalyzer.cs b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/ImageAnalyzer.cs
index 34df8fc52..255a65a79 100644
--- a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/ImageAnalyzer.cs
+++ b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/ImageAnalyzer.cs
@@ -238,7 +238,7 @@ private async Task ReceiveAsync(Shared data, Envelope e)
{
var analysisResult = default(ImageAnalysis);
- if (data != null)
+ if (data != null && data.Resource != null)
{
using Stream imageFileStream = new MemoryStream();
diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelOutputParser.cs b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelOutputParser.cs
new file mode 100644
index 000000000..485dc3d81
--- /dev/null
+++ b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelOutputParser.cs
@@ -0,0 +1,73 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+namespace Microsoft.Psi.Onnx
+{
+ using System;
+ using System.Collections.Generic;
+ using System.IO;
+ using System.Linq;
+ using Microsoft.Psi;
+
+ ///
+ /// Internal class that parses the outputs from the ImageNet model into
+ /// a set of image classification results.
+ ///
+ internal class ImageNetModelOutputParser
+ {
+ private readonly string[] labels;
+ private readonly int maxPredictions;
+ private readonly bool applySoftmax;
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The path to the file containing the list of 1000 ImageNet classes.
+ /// The maximum number of predictions to return.
+ /// Whether the softmax function should be applied to the raw model output.
+ ///
+ /// The file referenced by may be downloaded from the following location:
+ /// https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt.
+ ///
+ public ImageNetModelOutputParser(string imageClassesFile, int maxPredictions, bool applySoftmax)
+ {
+ this.labels = File.ReadAllLines(imageClassesFile);
+ if (this.labels.Length != 1000)
+ {
+ throw new ArgumentException($"The file {imageClassesFile} does not appear to be in the correct format. This file should contain exactly 1000 lines representing an ordered list of the 1000 ImageNet classes.");
+ }
+
+ this.maxPredictions = maxPredictions;
+ this.applySoftmax = applySoftmax;
+ }
+
+ ///
+ /// Gets the predictions from the model output.
+ ///
+ /// The model output vector of class probabilities.
+ /// A list of the top-N predictions, in descending probability order.
+ public List GetPredictions(float[] modelOutput)
+ {
+ return GetTopResults(this.applySoftmax ? Softmax(modelOutput) : modelOutput, this.maxPredictions)
+ .Select(c => new LabeledPrediction { Label = this.labels[c.Index], Confidence = c.Value })
+ .ToList();
+ }
+
+ private static IEnumerable<(int Index, float Value)> GetTopResults(IEnumerable predictedClasses, int count)
+ {
+ return predictedClasses
+ .Select((predictedClass, index) => (Index: index, Value: predictedClass))
+ .OrderByDescending(result => result.Value)
+ .Take(count);
+ }
+
+ private static IEnumerable Softmax(IEnumerable values)
+ {
+ var maxVal = values.Max();
+ var exp = values.Select(v => Math.Exp(v - maxVal));
+ var sumExp = exp.Sum();
+
+ return exp.Select(v => (float)(v / sumExp));
+ }
+ }
+}
diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunner.cs b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunner.cs
new file mode 100644
index 000000000..6186a5e9e
--- /dev/null
+++ b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunner.cs
@@ -0,0 +1,141 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+namespace Microsoft.Psi.Onnx
+{
+ using System;
+ using System.Collections.Generic;
+ using Microsoft.Psi;
+ using Microsoft.Psi.Components;
+ using Microsoft.Psi.Imaging;
+
+ ///
+ /// Component that runs an ImageNet image classification model.
+ ///
+ ///
+ /// This class implements a \psi component that runs an ONNX model trained
+ /// on the ImageNet dataset that operates on 224x224 RGB images and scores
+ /// the image for each of the 1000 ImageNet classes. It takes an input
+ /// stream of \psi images, applies a center-crop, rescales and normalizes
+ /// the pixel values into the input vector expected by the model. It also
+ /// parses the model outputs into a list of
+ /// values, corresponding to the top N predictions by the model. For
+ /// convenience, a set of pre-defined model runner configurations are
+ /// defined for a number of image classification models available in the
+ /// ONNX Model Zoo (https://github.com/onnx/models/tree/master/vision/classification).
+ /// The ONNX model file for the corresponding configuration will need to be
+ /// downloaded locally and the path to the model file will need to be
+ /// specified when creating the configuration.
+ ///
+ public class ImageNetModelRunner : ConsumerProducer, List>
+ {
+ private readonly float[] onnxInputVector = new float[3 * 224 * 224];
+ private readonly OnnxModel onnxModel;
+ private readonly ImageNetModelOutputParser outputParser;
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The pipeline to add the component to.
+ /// The configuration for the compoinent.
+ ///
+ /// To run on a GPU, use the Microsoft.Psi.Onnx.ModelRunners.Gpu library instead of Microsoft.Psi.Onnx.ModelRunners.Cpu, and set
+ /// the value of the parameter to a valid non-negative integer. Typical device ID values are 0 or 1.
+ ///
+ public ImageNetModelRunner(Pipeline pipeline, ImageNetModelRunnerConfiguration configuration)
+ : base(pipeline)
+ {
+ // create an ONNX model based on the supplied ImageNet model runner configuration
+ this.onnxModel = new OnnxModel(new OnnxModelConfiguration()
+ {
+ ModelFileName = configuration.ModelFilePath,
+ InputVectorName = configuration.InputVectorName,
+ InputVectorSize = 3 * 224 * 224,
+ OutputVectorName = configuration.OutputVectorName,
+ GpuDeviceId = configuration.GpuDeviceId,
+ });
+
+ this.outputParser = new ImageNetModelOutputParser(configuration.ImageClassesFilePath, configuration.NumberOfPredictions, configuration.ApplySoftmaxToModelOutput);
+ }
+
+ ///
+ protected override void Receive(Shared data, Envelope envelope)
+ {
+ // construct the ONNX model input vector (stored in this.onnxInputVector)
+ // based on the incoming image
+ this.ConstructOnnxInputVector(data);
+
+ // run the model over the input vector
+ var outputVector = this.onnxModel.GetPrediction(this.onnxInputVector);
+
+ // parse the model output into an ordered list of the top-N predictions
+ var results = this.outputParser.GetPredictions(outputVector);
+
+ // post the results
+ this.Out.Post(results, envelope.OriginatingTime);
+ }
+
+ ///
+ /// Constructs the input vector for the ImageNet model for a specified image.
+ ///
+ /// The image to construct the input vector for.
+ private void ConstructOnnxInputVector(Shared sharedImage)
+ {
+ var inputImage = sharedImage.Resource;
+ var inputWidth = sharedImage.Resource.Width;
+ var inputHeight = sharedImage.Resource.Height;
+
+ // crop a center square
+ var squareSize = Math.Min(inputWidth, inputHeight);
+ using var squareImage = ImagePool.GetOrCreate(squareSize, squareSize, sharedImage.Resource.PixelFormat);
+ if (inputWidth > inputHeight)
+ {
+ inputImage.Crop(squareImage.Resource, (inputWidth - squareSize) / 2, 0, squareSize, squareSize);
+ }
+ else
+ {
+ inputImage.Crop(squareImage.Resource, 0, (inputHeight - squareSize) / 2, squareSize, squareSize);
+ }
+
+ // resize the image to 224 x 224
+ using var resizedImage = ImagePool.GetOrCreate(224, 224, sharedImage.Resource.PixelFormat);
+ squareImage.Resource.Resize(resizedImage.Resource, 224, 224, SamplingMode.Bilinear);
+
+ // if the pixel format does not match, do a conversion before extracting the bytes
+ var bytes = default(byte[]);
+ if (sharedImage.Resource.PixelFormat != PixelFormat.BGR_24bpp)
+ {
+ using var reformattedImage = ImagePool.GetOrCreate(224, 224, PixelFormat.BGR_24bpp);
+ resizedImage.Resource.CopyTo(reformattedImage.Resource);
+ bytes = reformattedImage.Resource.ReadBytes(3 * 224 * 224);
+ }
+ else
+ {
+ // get the bytes
+ bytes = resizedImage.Resource.ReadBytes(3 * 224 * 224);
+ }
+
+ // Now populate the onnxInputVector float array / tensor by normalizing
+ // using mean = [0.485, 0.456, 0.406] and std = [0.229, 0.224, 0.225].
+ int fi = 0;
+
+ // first the red bytes
+ for (int i = 2; i < bytes.Length; i += 3)
+ {
+ this.onnxInputVector[fi++] = ((bytes[i] / 255.0f) - 0.485f) / 0.229f;
+ }
+
+ // then the green bytes
+ for (int i = 1; i < bytes.Length; i += 3)
+ {
+ this.onnxInputVector[fi++] = ((bytes[i] / 255.0f) - 0.456f) / 0.224f;
+ }
+
+ // then the blue bytes
+ for (int i = 0; i < bytes.Length; i += 3)
+ {
+ this.onnxInputVector[fi++] = ((bytes[i] / 255.0f) - 0.406f) / 0.225f;
+ }
+ }
+ }
+}
diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunnerConfiguration.cs b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunnerConfiguration.cs
new file mode 100644
index 000000000..56383aec9
--- /dev/null
+++ b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunnerConfiguration.cs
@@ -0,0 +1,204 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+namespace Microsoft.Psi.Onnx
+{
+ ///
+ /// Represents the configuration for the class.
+ ///
+ ///
+ /// For convenience, a set of pre-defined model runner configurations are defined for a few image
+ /// classification models via static creation methods. The model files themselves are available
+ /// in the ONNX Model Zoo (https://github.com/onnx/models/tree/master/vision/classification). They
+ /// will need to be downloaded separately and the path to the model file will need to be supplied
+ /// when creating the configuration.
+ ///
+ public class ImageNetModelRunnerConfiguration
+ {
+ ///
+ /// Gets or sets the path to the model file.
+ ///
+ public string ModelFilePath { get; set; }
+
+ ///
+ /// Gets or sets the path to a text file containing the 1000 ImageNet classes.
+ ///
+ public string ImageClassesFilePath { get; set; }
+
+ ///
+ /// Gets or sets the name of the input vector.
+ ///
+ public string InputVectorName { get; set; }
+
+ ///
+ /// Gets or sets the name of the output vector.
+ ///
+ public string OutputVectorName { get; set; }
+
+ ///
+ /// Gets or sets the number of predictions to include in the output. The
+ /// top-N predictions (where N = NumberOfPredictions) are output by the
+ /// component in order of decreasing probability.
+ ///
+ public int NumberOfPredictions { get; set; }
+
+ ///
+ /// Gets or sets a value indicating whether or not to apply the softmax
+ /// function to the raw model scores to obtain the class probabilities.
+ ///
+ public bool ApplySoftmaxToModelOutput { get; set; }
+
+ ///
+ /// Gets or sets the GPU device ID to run on, or null to run on CPU.
+ ///
+ ///
+ /// To run on a GPU, use the Microsoft.Psi.Onnx.Gpu library instead of Microsoft.Psi.Onnx.Cpu, and set the value of
+ /// the property to a valid non-negative integer. Typical device ID values are 0 or 1.
+ ///
+ public int? GpuDeviceId { get; set; }
+
+ ///
+ /// Creates an configuration for the ResNet50-caffe2 model, available at
+ /// https://github.com/onnx/models/raw/b9a54e89508f101a1611cd64f4ef56b9cb62c7cf/vision/classification/resnet/model/resnet50-caffe2-v1-7.onnx
+ /// and licensed at that commit under the Apache 2.0 License. This file must be downloaded separately
+ /// and the path to it should be specified in the parameter. Additionally,
+ /// the path to a file containing the 1000 ImageNet classes must also be supplied in the
+ /// , similar to the one available at
+ /// https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt.
+ ///
+ /// The path to the model file.
+ /// The path to the ImageNet classes file.
+ /// The number of predictions that the component should output.
+ /// The GPU device ID to run on, or null to run on CPU.
+ ///
+ /// To run on a GPU, use the Microsoft.Psi.Onnx.Gpu library instead of Microsoft.Psi.Onnx.Cpu, and set the value of
+ /// the property to a valid non-negative integer. Typical device ID values are 0 or 1.
+ ///
+ /// The model runner configuration.
+ public static ImageNetModelRunnerConfiguration CreateResNet50Caffe2(
+ string modelFilePath,
+ string imageClassesFilePath,
+ int numberOfPredictions = 5,
+ int? gpuDeviceId = null)
+ {
+ return new ImageNetModelRunnerConfiguration
+ {
+ ModelFilePath = modelFilePath,
+ ImageClassesFilePath = imageClassesFilePath,
+ InputVectorName = "gpu_0/data_0",
+ OutputVectorName = "gpu_0/softmax_1",
+ ApplySoftmaxToModelOutput = false,
+ NumberOfPredictions = numberOfPredictions,
+ GpuDeviceId = gpuDeviceId,
+ };
+ }
+
+ ///
+ /// Creates an configuration for the ResNet50-V2 model, available at
+ /// https://github.com/onnx/models/raw/b9a54e89508f101a1611cd64f4ef56b9cb62c7cf/vision/classification/resnet/model/resnet50-v2-7.onnx
+ /// and licensed at that commit under the Apache 2.0 Licese. This file must be downloaded separately
+ /// and the path to it should be specified in the parameter. Additionally,
+ /// the path to a file containing the 1000 ImageNet classes must also be supplied in the
+ /// , similar to the one available at
+ /// https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt.
+ ///
+ /// The path to the model file.
+ /// The path to the ImageNet classes file.
+ /// The number of predictions that the component should output.
+ /// The GPU device ID to run on, or null to run on CPU.
+ ///
+ /// To run on a GPU, use the Microsoft.Psi.Onnx.Gpu library instead of Microsoft.Psi.Onnx.Cpu, and set the value of
+ /// the property to a valid non-negative integer. Typical device ID values are 0 or 1.
+ ///
+ /// The model runner configuration.
+ public static ImageNetModelRunnerConfiguration CreateResNet50v2(
+ string modelFilePath,
+ string imageClassesFilePath,
+ int numberOfPredictions = 5,
+ int? gpuDeviceId = null)
+ {
+ return new ImageNetModelRunnerConfiguration
+ {
+ ModelFilePath = modelFilePath,
+ ImageClassesFilePath = imageClassesFilePath,
+ InputVectorName = "data",
+ OutputVectorName = "resnetv24_dense0_fwd",
+ ApplySoftmaxToModelOutput = true,
+ NumberOfPredictions = numberOfPredictions,
+ GpuDeviceId = gpuDeviceId,
+ };
+ }
+
+ ///
+ /// Creates an configuration for the VGG 16 model, available at
+ /// https://github.com/onnx/models/raw/f884b33c3e2371952aad7ea091898f418c830fe5/vision/classification/vgg/model/vgg16-7.onnx
+ /// and licensed at that commit under the Apache 2.0 License. This file must be downloaded separately
+ /// and the path to it should be specified in the parameter. Additionally,
+ /// the path to a file containing the 1000 ImageNet classes must also be supplied in the
+ /// , similar to the one available at
+ /// https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt.
+ ///
+ /// The path to the model file.
+ /// The path to the ImageNet classes file.
+ /// The number of predictions that the component should output.
+ /// The GPU device ID to run on, or null to run on CPU.
+ ///
+ /// To run on a GPU, use the Microsoft.Psi.Onnx.Gpu library instead of Microsoft.Psi.Onnx.Cpu, and set the value of
+ /// the property to a valid non-negative integer. Typical device ID values are 0 or 1.
+ ///
+ /// The model runner configuration.
+ public static ImageNetModelRunnerConfiguration CreateVgg16(
+ string modelFilePath,
+ string imageClassesFilePath,
+ int numberOfPredictions = 5,
+ int? gpuDeviceId = null)
+ {
+ return new ImageNetModelRunnerConfiguration
+ {
+ ModelFilePath = modelFilePath,
+ ImageClassesFilePath = imageClassesFilePath,
+ InputVectorName = "data",
+ OutputVectorName = "vgg0_dense2_fwd",
+ ApplySoftmaxToModelOutput = true,
+ NumberOfPredictions = numberOfPredictions,
+ GpuDeviceId = gpuDeviceId,
+ };
+ }
+
+ ///
+ /// Creates an configuration for the ShuffleNetV2 model, available at
+ /// https://github.com/onnx/models/raw/b9a54e89508f101a1611cd64f4ef56b9cb62c7cf/vision/classification/shufflenet/model/shufflenet-v2-10.onnx
+ /// and licensed at that commit under the BSD 3-Clause License. This file must be downloaded separately
+ /// and the path to it should be specified in the parameter. Additionally,
+ /// the path to a file containing the 1000 ImageNet classes must also be supplied in the
+ /// , similar to the one available at
+ /// https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt.
+ ///
+ /// The path to the model file.
+ /// The path to the ImageNet classes file.
+ /// The number of predictions that the component should output.
+ /// The GPU device ID to run on, or null to run on CPU.
+ ///
+ /// To run on a GPU, use the Microsoft.Psi.Onnx.Gpu library instead of Microsoft.Psi.Onnx.Cpu, and set the value of
+ /// the property to a valid non-negative integer. Typical device ID values are 0 or 1.
+ ///
+ /// The model runner configuration.
+ public static ImageNetModelRunnerConfiguration CreateShuffleNet2(
+ string modelFilePath,
+ string imageClassesFilePath,
+ int numberOfPredictions = 5,
+ int? gpuDeviceId = null)
+ {
+ return new ImageNetModelRunnerConfiguration
+ {
+ ModelFilePath = modelFilePath,
+ ImageClassesFilePath = imageClassesFilePath,
+ InputVectorName = "input",
+ OutputVectorName = "output",
+ ApplySoftmaxToModelOutput = true,
+ NumberOfPredictions = numberOfPredictions,
+ GpuDeviceId = gpuDeviceId,
+ };
+ }
+ }
+}
diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/LabeledPrediction.cs b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/LabeledPrediction.cs
new file mode 100644
index 000000000..72998d628
--- /dev/null
+++ b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/LabeledPrediction.cs
@@ -0,0 +1,21 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+namespace Microsoft.Psi.Onnx
+{
+ ///
+ /// Represents a labeled prediction.
+ ///
+ public class LabeledPrediction
+ {
+ ///
+ /// Gets or sets the class label.
+ ///
+ public string Label { get; set; }
+
+ ///
+ /// Gets or sets the confidence level.
+ ///
+ public float Confidence { get; set; }
+ }
+}
diff --git a/Sources/Integrations/Onnx/Test.Psi.Onnx/ImageNetTester.cs b/Sources/Integrations/Onnx/Test.Psi.Onnx/ImageNetTester.cs
new file mode 100644
index 000000000..522a08b1d
--- /dev/null
+++ b/Sources/Integrations/Onnx/Test.Psi.Onnx/ImageNetTester.cs
@@ -0,0 +1,220 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+namespace Test.Psi.Onnx
+{
+ using System.Collections.Generic;
+ using System.IO;
+ using System.Threading;
+ using Microsoft.Psi;
+ using Microsoft.Psi.Imaging;
+ using Microsoft.Psi.Onnx;
+ using Microsoft.VisualStudio.TestTools.UnitTesting;
+ using Test.Psi.Common;
+
+ [TestClass]
+ public class ImageNetTester
+ {
+ [TestMethod]
+ [Timeout(60000)]
+ public void ResNet50Caffe2ImageClassificationTest()
+ {
+ // This test expects a set of resources at the path specified by the PsiTestResources environment
+ // variable. Specifically, it expects a subfolder named ResNet containing the file:
+ // - resnet50-caffe2-v1-7.onnx: this is the model, available from the onnx model zoo:
+ // https://github.com/onnx/models/raw/b9a54e89508f101a1611cd64f4ef56b9cb62c7cf/vision/classification/resnet/model/resnet50-caffe2-v1-7.onnx
+ // (licensed at that commit under the Apache 2.0 License)
+ // a subfolder ImageNet containing the file:
+ // - synset.txt: this is a file containing an ordered list of the 1000 ImageNet classes, available from the onnx model zoo:
+ // https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt
+ // and a subfolder TestImages containing the file:
+ // - image1.jpg: this is the test image, available from the ML.NET samples repo,
+ // https://github.com/dotnet/machinelearning-samples/tree/ffac232a1d599903b8591b68dabd237af48f462f/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx
+ // in the ObjectDetectionConsoleApp/assets/images folder
+
+ if (TestRunner.TestResourcesPath != null)
+ {
+ string modelFile = Path.Combine(TestRunner.TestResourcesPath, "ResNet", "resnet50-caffe2-v1-7.onnx");
+ string imageClassesFile = Path.Combine(TestRunner.TestResourcesPath, "ImageNet", "synset.txt");
+ string testImageFile = Path.Combine(TestRunner.TestResourcesPath, "TestImages", "image1.jpg");
+
+ // create a configuration for the ResNet50Caffe2 model
+ var modelConfig = ImageNetModelRunnerConfiguration.CreateResNet50Caffe2(modelFile, imageClassesFile);
+ var testImage = Shared.Create(Image.FromBitmap(new System.Drawing.Bitmap(testImageFile)));
+
+ this.ImageClassificationTest(
+ modelConfig,
+ testImage,
+ new[] {
+ ("n03100240 convertible", 0.8695966f),
+ ("n03930630 pickup, pickup truck", 0.04933356f),
+ ("n02814533 beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", 0.03959884f),
+ ("n03594945 jeep, landrover", 0.0146017177f),
+ ("n02974003 car wheel", 0.00834592152f),
+ });
+ }
+ else
+ {
+ Assert.Inconclusive($"Test not run because 'PsiTestResources' environment variable not found.");
+ }
+ }
+
+ [TestMethod]
+ [Timeout(60000)]
+ public void ResNet50V2ImageClassificationTest()
+ {
+ // This test expects a set of resources at the path specified by the PsiTestResources environment
+ // variable. Specifically, it expects a subfolder named ResNet containing the file:
+ // - resnet50-v2-7.onnx: this is the model, available from the onnx model zoo:
+ // https://github.com/onnx/models/raw/b9a54e89508f101a1611cd64f4ef56b9cb62c7cf/vision/classification/resnet/model/resnet50-v2-7.onnx
+ // (licensed at that commit under the Apache 2.0 License)
+ // a subfolder ImageNet containing the file:
+ // - synset.txt: this is a file containing an ordered list of the 1000 ImageNet classes, available from the onnx model zoo:
+ // https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt
+ // and a subfolder TestImages containing the file:
+ // - image1.jpg: this is the test image, available from the ML.NET samples repo,
+ // https://github.com/dotnet/machinelearning-samples/tree/ffac232a1d599903b8591b68dabd237af48f462f/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx
+ // in the ObjectDetectionConsoleApp/assets/images folder
+
+ if (TestRunner.TestResourcesPath != null)
+ {
+ string modelFile = Path.Combine(TestRunner.TestResourcesPath, "ResNet", "resnet50-v2-7.onnx");
+ string imageClassesFile = Path.Combine(TestRunner.TestResourcesPath, "ImageNet", "synset.txt");
+ string testImageFile = Path.Combine(TestRunner.TestResourcesPath, "TestImages", "image1.jpg");
+
+ // create a configuration for the ResNet50v2 model
+ var modelConfig = ImageNetModelRunnerConfiguration.CreateResNet50v2(modelFile, imageClassesFile);
+ var testImage = Shared.Create(Image.FromBitmap(new System.Drawing.Bitmap(testImageFile)));
+
+ this.ImageClassificationTest(
+ modelConfig,
+ testImage,
+ new[] {
+ ("n03930630 pickup, pickup truck", 0.6400269f),
+ ("n03100240 convertible", 0.30403322f),
+ ("n04461696 tow truck, tow car, wrecker", 0.0135363257f),
+ ("n02974003 car wheel", 0.0124500105f),
+ ("n03459775 grille, radiator grille", 0.005335613f),
+ });
+ }
+ else
+ {
+ Assert.Inconclusive($"Test not run because 'PsiTestResources' environment variable not found.");
+ }
+ }
+
+ [TestMethod]
+ [Timeout(60000)]
+ public void Vgg16ImageClassificationTest()
+ {
+ // This test expects a set of resources at the path specified by the PsiTestResources environment
+ // variable. Specifically, it expects a subfolder named VGG containing the file:
+ // - vgg16-7.onnx: this is the model, available from the onnx model zoo:
+ // https://github.com/onnx/models/raw/f884b33c3e2371952aad7ea091898f418c830fe5/vision/classification/vgg/model/vgg16-7.onnx
+ // (licensed at that commit under the Apache 2.0 License)
+ // a subfolder ImageNet containing the file:
+ // - synset.txt: this is a file containing an ordered list of the 1000 ImageNet classes, available from the onnx model zoo:
+ // https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt
+ // and a subfolder TestImages containing the file:
+ // - image1.jpg: this is the test image, available from the ML.NET samples repo,
+ // https://github.com/dotnet/machinelearning-samples/tree/ffac232a1d599903b8591b68dabd237af48f462f/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx
+ // in the ObjectDetectionConsoleApp/assets/images folder
+
+ if (TestRunner.TestResourcesPath != null)
+ {
+ string modelFile = Path.Combine(TestRunner.TestResourcesPath, "VGG", "vgg16-7.onnx");
+ string imageClassesFile = Path.Combine(TestRunner.TestResourcesPath, "ImageNet", "synset.txt");
+ string testImageFile = Path.Combine(TestRunner.TestResourcesPath, "TestImages", "image1.jpg");
+
+ // create a configuration for the VGG 16 model
+ var modelConfig = ImageNetModelRunnerConfiguration.CreateVgg16(modelFile, imageClassesFile);
+ var testImage = Shared.Create(Image.FromBitmap(new System.Drawing.Bitmap(testImageFile)));
+
+ this.ImageClassificationTest(
+ modelConfig,
+ testImage,
+ new[] {
+ ("n03100240 convertible", 0.7319748f),
+ ("n03930630 pickup, pickup truck", 0.129529521f),
+ ("n03594945 jeep, landrover", 0.0262848679f),
+ ("n03459775 grille, radiator grille", 0.0258834548f),
+ ("n02974003 car wheel", 0.0252302475f),
+ });
+ }
+ else
+ {
+ Assert.Inconclusive($"Test not run because 'PsiTestResources' environment variable not found.");
+ }
+ }
+
+ [TestMethod]
+ [Timeout(60000)]
+ public void ShuffleNet2ImageClassificationTest()
+ {
+ // This test expects a set of resources at the path specified by the PsiTestResources environment
+ // variable. Specifically, it expects a subfolder named ShuffleNet containing the file:
+ // - shufflenet-v2-10: this is the model, available from the onnx model zoo:
+ // https://github.com/onnx/models/raw/b9a54e89508f101a1611cd64f4ef56b9cb62c7cf/vision/classification/shufflenet/model/shufflenet-v2-10.onnx
+ // (licensed at that commit under the BSD 3-Clause License)
+ // a subfolder ImageNet containing the file:
+ // - synset.txt: this is a file containing an ordered list of the 1000 ImageNet classes, available from the onnx model zoo:
+ // https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt
+ // and a subfolder TestImages containing the file:
+ // - image1.jpg: this is the test image, available from the ML.NET samples repo,
+ // https://github.com/dotnet/machinelearning-samples/tree/ffac232a1d599903b8591b68dabd237af48f462f/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx
+ // in the ObjectDetectionConsoleApp/assets/images folder
+
+ if (TestRunner.TestResourcesPath != null)
+ {
+ string modelFile = Path.Combine(TestRunner.TestResourcesPath, "ShuffleNet", "shufflenet-v2-10.onnx");
+ string imageClassesFile = Path.Combine(TestRunner.TestResourcesPath, "ImageNet", "synset.txt");
+ string testImageFile = Path.Combine(TestRunner.TestResourcesPath, "TestImages", "image1.jpg");
+
+ // create a configuration for the ShuffleNet V2 model
+ var modelConfig = ImageNetModelRunnerConfiguration.CreateShuffleNet2(modelFile, imageClassesFile);
+ var testImage = Shared.Create(Image.FromBitmap(new System.Drawing.Bitmap(testImageFile)));
+
+ this.ImageClassificationTest(
+ modelConfig,
+ testImage,
+ new[] {
+ ("n03100240 convertible", 0.480612665f),
+ ("n02701002 ambulance", 0.155334875f),
+ ("n02974003 car wheel", 0.08949315f),
+ ("n03459775 grille, radiator grille", 0.061813876f),
+ ("n03930630 pickup, pickup truck", 0.05676603f),
+ });
+ }
+ else
+ {
+ Assert.Inconclusive($"Test not run because 'PsiTestResources' environment variable not found.");
+ }
+ }
+
+ private void ImageClassificationTest(
+ ImageNetModelRunnerConfiguration testConfig,
+ Shared testImage,
+ IList<(string Label, float Confidence)> expectedPredictions)
+ {
+ List labeledResults = null;
+
+ // create a pipeline and run the model on the test image
+ using (var p = Pipeline.Create())
+ {
+ var resNet = new ImageNetModelRunner(p, testConfig);
+ Generators.Return(p, testImage)
+ .PipeTo(resNet)
+ .Do(results => labeledResults = results.DeepClone());
+
+ p.Run();
+ }
+
+ // verify the model predictions against the expected predictions
+ for (int i = 0; i < expectedPredictions.Count; i++)
+ {
+ Assert.AreEqual(expectedPredictions[i].Label, labeledResults[i].Label);
+ Assert.AreEqual(expectedPredictions[i].Confidence, labeledResults[i].Confidence, 0.000005);
+ }
+ }
+ }
+}
diff --git a/Sources/Integrations/Onnx/Test.Psi.Onnx/TinyYoloV2Tester.cs b/Sources/Integrations/Onnx/Test.Psi.Onnx/TinyYoloV2Tester.cs
index 402ce9138..8be62a06e 100644
--- a/Sources/Integrations/Onnx/Test.Psi.Onnx/TinyYoloV2Tester.cs
+++ b/Sources/Integrations/Onnx/Test.Psi.Onnx/TinyYoloV2Tester.cs
@@ -21,12 +21,13 @@ public class TinyYoloV2Tester
public void TinyYoloV2ObjectDetectionTest()
{
// This test expects a set of resources at a path specified via the PsiTestResources environment
- // variable. Specifically, it expects a subfolder TinyYoloV2 containg two files:
+ // variable. Specifically, it expects a subfolder TinyYoloV2 containg the file:
// - TinyYolo2_model.onnx: is the model, available from the onnx model zoo:
// https://github.com/onnx/models/raw/3d4b2c28f951064ab35c89d5f5c3ffe74a149e4b/vision/object_detection_segmentation/tiny-yolov2/model/tinyyolov2-8.onnx,
// or in the ML.NET samples repo,
// https://github.com/dotnet/machinelearning-samples/tree/ffac232a1d599903b8591b68dabd237af48f462f/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx
// in the ObjectDetectionConsoleApp/assets/Model folder
+ // and a subfolder TestImages containing the file:
// - image1.jpg: is the test image, available from the ML.NET samples repo,
// https://github.com/dotnet/machinelearning-samples/tree/ffac232a1d599903b8591b68dabd237af48f462f/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx
// in the ObjectDetectionConsoleApp/assets/images folder
@@ -34,7 +35,7 @@ public void TinyYoloV2ObjectDetectionTest()
if (TestRunner.TestResourcesPath != null)
{
var yoloModel = Path.Combine(TestRunner.TestResourcesPath, "TinyYoloV2", "TinyYolo2_model.onnx");
- var testImage = Path.Combine(TestRunner.TestResourcesPath, "TinyYoloV2", "image1.jpg");
+ var testImage = Path.Combine(TestRunner.TestResourcesPath, "TestImages", "image1.jpg");
var labels = new string[] { "car", "car", "person", "car", "car" };
var confidences = new float[] { 0.972779453f, 0.7202784f, 0.512937546f, 0.476578265f, 0.452380836f };
@@ -65,10 +66,13 @@ public void TinyYoloV2ObjectDetectionTest()
p.Run();
- CollectionAssert.AreEqual(labels, labelsResults.ToArray());
- CollectionAssert.AreEqual(confidences, confidencesResults.ToArray());
- CollectionAssert.AreEqual(boxX, boxXResults);
- CollectionAssert.AreEqual(boxY, boxYResults);
+ for (int i = 0; i < confidences.Length; i++)
+ {
+ Assert.AreEqual(labels[i], labelsResults[i]);
+ Assert.AreEqual(confidences[i], confidencesResults[i], 0.000001);
+ Assert.AreEqual(boxX[i], boxXResults[i], 0.001);
+ Assert.AreEqual(boxY[i], boxYResults[i], 0.001);
+ }
}
else
{
diff --git a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectCore.cs b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectCore.cs
index c677d35dc..76c763f72 100644
--- a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectCore.cs
+++ b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectCore.cs
@@ -44,7 +44,7 @@ internal sealed class AzureKinectCore : ISourceComponent, IDisposable
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
/// Configuration to use for the device.
public AzureKinectCore(Pipeline pipeline, AzureKinectSensorConfiguration config = null)
{
diff --git a/Sources/Media/Microsoft.Psi.Media.Linux/MediaCapture.cs b/Sources/Media/Microsoft.Psi.Media.Linux/MediaCapture.cs
index 52a9e4692..1531735b1 100644
--- a/Sources/Media/Microsoft.Psi.Media.Linux/MediaCapture.cs
+++ b/Sources/Media/Microsoft.Psi.Media.Linux/MediaCapture.cs
@@ -26,7 +26,7 @@ public class MediaCapture : IProducer>, ISourceComponent, IDisposa
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
/// Name of file containing media capture device configuration.
public MediaCapture(Pipeline pipeline, string configurationFilename)
: this(pipeline)
@@ -38,7 +38,7 @@ public MediaCapture(Pipeline pipeline, string configurationFilename)
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
/// Describes how to configure the media capture device.
public MediaCapture(Pipeline pipeline, MediaCaptureConfiguration configuration)
: this(pipeline)
@@ -49,7 +49,7 @@ public MediaCapture(Pipeline pipeline, MediaCaptureConfiguration configuration)
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
/// Width of output image in pixels.
/// Height of output image in pixels.
/// Device ID.
diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaCapture.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaCapture.cs
index 86e1e3fe1..b14574d75 100644
--- a/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaCapture.cs
+++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaCapture.cs
@@ -38,7 +38,7 @@ public class MediaCapture : IProducer>, ISourceComponent, IDisposa
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
/// Name of file containing media capture device configuration.
public MediaCapture(Pipeline pipeline, string configurationFilename)
: this(pipeline)
@@ -54,7 +54,7 @@ public MediaCapture(Pipeline pipeline, string configurationFilename)
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
/// Describes how to configure the media capture device.
public MediaCapture(Pipeline pipeline, MediaCaptureConfiguration configuration = null)
: this(pipeline)
@@ -69,7 +69,7 @@ public MediaCapture(Pipeline pipeline, MediaCaptureConfiguration configuration =
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
/// Width of output image in pixels.
/// Height of output image in pixels.
/// Frame rate.
diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaSource.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaSource.cs
index 3d7190280..28afefa15 100644
--- a/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaSource.cs
+++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaSource.cs
@@ -45,7 +45,7 @@ public class MediaSource : Generator, IDisposable
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
/// Name of media file to play.
/// Optional flag specifying whether to drop out of order packets (defaults to false).
public MediaSource(Pipeline pipeline, string filename, bool dropOutOfOrderPackets = false)
@@ -56,7 +56,7 @@ public MediaSource(Pipeline pipeline, string filename, bool dropOutOfOrderPacket
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
/// Source stream of the media to consume.
/// Optional date/time that the media started.
/// Optional flag specifying whether to drop out of order packets (defaults to false).
diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/Microsoft.Psi.Media.Windows.x64.csproj b/Sources/Media/Microsoft.Psi.Media.Windows.x64/Microsoft.Psi.Media.Windows.x64.csproj
index 513984be5..98d5b0540 100644
--- a/Sources/Media/Microsoft.Psi.Media.Windows.x64/Microsoft.Psi.Media.Windows.x64.csproj
+++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/Microsoft.Psi.Media.Windows.x64.csproj
@@ -41,6 +41,7 @@
+
@@ -53,4 +54,9 @@
+
+
+
+
+
\ No newline at end of file
diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/Mpeg4Writer.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/Mpeg4Writer.cs
index 8d2263683..30c37e642 100644
--- a/Sources/Media/Microsoft.Psi.Media.Windows.x64/Mpeg4Writer.cs
+++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/Mpeg4Writer.cs
@@ -22,7 +22,7 @@ public class Mpeg4Writer : IConsumer>, IDisposable
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
/// Name of output file to write to.
/// Name of file containing media capture device configuration.
public Mpeg4Writer(Pipeline pipeline, string filename, string configurationFilename)
@@ -35,7 +35,7 @@ public Mpeg4Writer(Pipeline pipeline, string filename, string configurationFilen
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
/// Name of output file to write to.
/// Describes how to configure the media capture device.
public Mpeg4Writer(Pipeline pipeline, string filename, Mpeg4WriterConfiguration configuration)
@@ -47,7 +47,7 @@ public Mpeg4Writer(Pipeline pipeline, string filename, Mpeg4WriterConfiguration
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
/// Name of output file to write to.
/// Width of output image in pixels.
/// Height of output image in pixels.
diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCapture.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCapture.cs
new file mode 100644
index 000000000..7bdabcd9e
--- /dev/null
+++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCapture.cs
@@ -0,0 +1,76 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+namespace Microsoft.Psi.Media
+{
+ using System;
+ using System.Windows;
+ using System.Windows.Media;
+ using System.Windows.Media.Imaging;
+ using Microsoft.Psi.Components;
+ using Microsoft.Psi.Imaging;
+
+ ///
+ /// Component that streams video from a Windows Media Visual.
+ ///
+ public class VisualCapture : Generator, IProducer>
+ {
+ private readonly Visual visual;
+ private readonly int pixelWidth;
+ private readonly int pixelHeight;
+ private readonly TimeSpan interval;
+ private readonly RenderTargetBitmap renderTargetBitmap;
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The pipeline to add the component to.
+ /// Interval at which to render and emit frames of the rendered visual.
+ /// Windows Media Visual to stream.
+ /// Pixel width at which to render the visual.
+ /// Pixel height at which to render the visual.
+ public VisualCapture(Pipeline pipeline, TimeSpan interval, Visual visual, int pixelWidth, int pixelHeight)
+ : base(pipeline, true)
+ {
+ this.interval = interval;
+ this.visual = visual;
+ this.pixelWidth = pixelWidth;
+ this.pixelHeight = pixelHeight;
+ this.renderTargetBitmap = new RenderTargetBitmap(pixelWidth, pixelHeight, 96, 96, PixelFormats.Pbgra32);
+ this.Out = pipeline.CreateEmitter>(this, nameof(this.Out));
+ }
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The pipeline to add the component to.
+ /// Configuration values for the visual capture component.
+ public VisualCapture(Pipeline pipeline, VisualCaptureConfiguration configuration)
+ : this(pipeline, configuration.Interval, configuration.Visual, configuration.PixelWidth, configuration.PixelHeight)
+ {
+ }
+
+ ///
+ /// Gets the emitter that generates images from the visual.
+ ///
+ public Emitter> Out { get; private set; }
+
+ ///
+ protected override DateTime GenerateNext(DateTime currentTime)
+ {
+ using (var sharedImage = ImagePool.GetOrCreate(this.pixelWidth, this.pixelHeight, Imaging.PixelFormat.BGRA_32bpp))
+ {
+ var resource = sharedImage.Resource;
+ this.visual.Dispatcher.Invoke(() =>
+ {
+ this.renderTargetBitmap.Render(this.visual);
+ this.renderTargetBitmap.CopyPixels(Int32Rect.Empty, resource.ImageData, resource.Size, resource.Stride);
+ });
+
+ this.Out.Post(sharedImage, currentTime);
+ }
+
+ return currentTime + this.interval;
+ }
+ }
+}
diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCaptureConfiguration.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCaptureConfiguration.cs
new file mode 100644
index 000000000..7e2258998
--- /dev/null
+++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCaptureConfiguration.cs
@@ -0,0 +1,41 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+namespace Microsoft.Psi.Media
+{
+ using System;
+ using System.Windows.Media;
+
+ ///
+ /// Encapsulates configuration for VisualCapture component.
+ ///
+ public class VisualCaptureConfiguration
+ {
+ ///
+ /// Default configuration.
+ ///
+ public static readonly VisualCaptureConfiguration Default = new VisualCaptureConfiguration()
+ {
+ };
+
+ ///
+ /// Gets or sets the interval at which to render and emit frames of the rendered visual..
+ ///
+ public TimeSpan Interval { get; set; } = TimeSpan.FromMilliseconds(100);
+
+ ///
+ /// Gets or sets the Windows Media Visual to stream.
+ ///
+ public Visual Visual { get; set; }
+
+ ///
+ /// Gets or sets the pixel width at which to render the visual.
+ ///
+ public int PixelWidth { get; set; }
+
+ ///
+ /// Gets or sets the pixel height at which to render the visual.
+ ///
+ public int PixelHeight { get; set; }
+ }
+}
diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCapture.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCapture.cs
new file mode 100644
index 000000000..9ad099bd5
--- /dev/null
+++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCapture.cs
@@ -0,0 +1,145 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+#pragma warning disable CA1060 // Move pinvokes to native methods class
+#pragma warning disable SA1305 // Field names should not use Hungarian notation
+
+namespace Microsoft.Psi.Media
+{
+ using System;
+ using System.Drawing;
+ using System.Runtime.InteropServices;
+ using Microsoft.Psi.Components;
+ using Microsoft.Psi.Imaging;
+ using PsiImage = Microsoft.Psi.Imaging.Image;
+
+ ///
+ /// Component that streams video from Window handle (default=desktop window/primary screen).
+ ///
+ public class WindowCapture : Generator, IProducer>
+ {
+ // see https://docs.microsoft.com/en-us/windows/win32/api/wingdi/nf-wingdi-bitblt
+ private const int CaptureBlt = 0x40000000; // include layered windows
+ private const int SourceCopy = 0x00CC0020; // copy source rectangle directly to the destination
+
+ private readonly TimeSpan interval;
+ private readonly IntPtr hWnd;
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The pipeline to add the component to.
+ /// Interval at which to render and emit frames of the window.
+ /// Window handle to capture (default=desktop window/primary screen).
+ public WindowCapture(Pipeline pipeline, TimeSpan interval, IntPtr hWnd)
+ : base(pipeline, true)
+ {
+ this.interval = interval;
+ this.hWnd = hWnd == IntPtr.Zero ? GetDesktopWindow() : hWnd;
+ this.Out = pipeline.CreateEmitter>(this, nameof(this.Out));
+ }
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The pipeline to add the component to.
+ /// Interval at which to render and emit frames of the window.
+ public WindowCapture(Pipeline pipeline, TimeSpan interval)
+ : this(pipeline, interval, IntPtr.Zero)
+ {
+ }
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The pipeline to add the component to.
+ /// Configuration values for the window capture component.
+ public WindowCapture(Pipeline pipeline, WindowCaptureConfiguration configuration)
+ : this(pipeline, configuration.Interval, configuration.WindowHandle)
+ {
+ }
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The pipeline to add the component to.
+ public WindowCapture(Pipeline pipeline)
+ : this(pipeline, WindowCaptureConfiguration.Default)
+ {
+ }
+
+ ///
+ /// Gets the emitter that generates images from window.
+ ///
+ public Emitter> Out { get; private set; }
+
+ ///
+ protected override DateTime GenerateNext(DateTime currentTime)
+ {
+ GetWindowRect(this.hWnd, out RECT rect);
+ var width = rect.Right - rect.Left;
+ var height = rect.Bottom - rect.Top;
+ var win = GetWindowDC(this.hWnd);
+ var dest = CreateCompatibleDC(win);
+ var hBmp = CreateCompatibleBitmap(win, width, height);
+ var sel = SelectObject(dest, hBmp);
+ BitBlt(dest, 0, 0, width, height, win, 0, 0, SourceCopy | CaptureBlt);
+ var bitmap = Bitmap.FromHbitmap(hBmp);
+
+ using (var sharedImage = ImagePool.GetOrCreate(width, height, PixelFormat.BGRA_32bpp))
+ {
+ var resource = sharedImage.Resource;
+ resource.CopyFrom(bitmap);
+ this.Out.Post(sharedImage, currentTime);
+ }
+
+ bitmap.Dispose();
+ SelectObject(dest, sel);
+ DeleteObject(hBmp);
+ DeleteDC(dest);
+ ReleaseDC(this.hWnd, win);
+
+ return currentTime + this.interval;
+ }
+
+ [DllImport("user32.dll")]
+ private static extern IntPtr GetDesktopWindow();
+
+ [DllImport("user32.dll")]
+ [return: MarshalAs(UnmanagedType.Bool)]
+ private static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect);
+
+ [DllImport("user32.dll")]
+ private static extern IntPtr GetWindowDC(IntPtr ptr);
+
+ [DllImport("gdi32.dll")]
+ private static extern IntPtr CreateCompatibleDC(IntPtr hdc);
+
+ [DllImport("gdi32.dll")]
+ private static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int nWidth, int nHeight);
+
+ [DllImport("gdi32.dll")]
+ private static extern IntPtr SelectObject(IntPtr hdc, IntPtr bmp);
+
+ [DllImport("gdi32.dll")]
+ private static extern bool BitBlt(IntPtr hdcDest, int xDest, int yDest, int wDest, int hDest, IntPtr hdcSource, int xSrc, int ySrc, uint rop);
+
+ [DllImport("gdi32.dll")]
+ private static extern IntPtr DeleteObject(IntPtr hDc);
+
+ [DllImport("gdi32.dll")]
+ private static extern IntPtr DeleteDC(IntPtr hDc);
+
+ [DllImport("user32.dll")]
+ private static extern bool ReleaseDC(IntPtr hWnd, IntPtr hDc);
+
+ [StructLayout(LayoutKind.Sequential)]
+ private struct RECT
+ {
+ public int Left;
+ public int Top;
+ public int Right;
+ public int Bottom;
+ }
+ }
+}
diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCaptureConfiguration.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCaptureConfiguration.cs
new file mode 100644
index 000000000..a47476ada
--- /dev/null
+++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCaptureConfiguration.cs
@@ -0,0 +1,28 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+namespace Microsoft.Psi.Media
+{
+ using System;
+
+ ///
+ /// Encapsulates configuration for WindowCapture component.
+ ///
+ public class WindowCaptureConfiguration
+ {
+ ///
+ /// Default configuration.
+ ///
+ public static readonly WindowCaptureConfiguration Default = new WindowCaptureConfiguration();
+
+ ///
+ /// Gets or sets the interval at which to render and emit frames of the window..
+ ///
+ public TimeSpan Interval { get; set; } = TimeSpan.FromMilliseconds(100);
+
+ ///
+ /// Gets or sets the Window handle to capture (default=desktop window/primary screen).
+ ///
+ public IntPtr WindowHandle { get; set; } = IntPtr.Zero;
+ }
+}
diff --git a/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.cpp b/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.cpp
index 2f673363d..2b0951c37 100644
--- a/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.cpp
+++ b/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.cpp
@@ -15,6 +15,6 @@ using namespace System::Security::Permissions;
[assembly:AssemblyCopyrightAttribute("Copyright (c) Microsoft Corporation. All rights reserved.")];
[assembly:ComVisible(false)];
[assembly:CLSCompliantAttribute(true)];
-[assembly:AssemblyVersionAttribute("0.13.38.2")];
-[assembly:AssemblyFileVersionAttribute("0.13.38.2")];
-[assembly:AssemblyInformationalVersionAttribute("0.13.38.2-beta")];
+[assembly:AssemblyVersionAttribute("0.14.35.3")];
+[assembly:AssemblyFileVersionAttribute("0.14.35.3")];
+[assembly:AssemblyInformationalVersionAttribute("0.14.35.3-beta")];
diff --git a/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.rc b/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.rc
index 4a44acb8e..7c483c1ac 100644
Binary files a/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.rc and b/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.rc differ
diff --git a/Sources/Media/Shared/FFMPEGMediaSource.cs b/Sources/Media/Shared/FFMPEGMediaSource.cs
index 4fc47d772..764d35ff7 100644
--- a/Sources/Media/Shared/FFMPEGMediaSource.cs
+++ b/Sources/Media/Shared/FFMPEGMediaSource.cs
@@ -37,7 +37,7 @@ public class FFMPEGMediaSource : Generator, IDisposable
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of
+ /// The pipeline to add the component to.
/// Name of media file to play
/// Output format for images
public FFMPEGMediaSource(Pipeline pipeline, string filename, PixelFormat format = PixelFormat.BGRX_32bpp)
diff --git a/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/Properties/AssemblyInfo.cs b/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/Properties/AssemblyInfo.cs
index 7278a41bd..0506a84df 100644
--- a/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/Properties/AssemblyInfo.cs
+++ b/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/Properties/AssemblyInfo.cs
@@ -35,6 +35,6 @@
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("0.13.38.2")]
-[assembly: AssemblyFileVersion("0.13.38.2")]
-[assembly: AssemblyInformationalVersion("0.13.38.2-beta")]
+[assembly: AssemblyVersion("0.14.35.3")]
+[assembly: AssemblyFileVersion("0.14.35.3")]
+[assembly: AssemblyInformationalVersion("0.14.35.3-beta")]
diff --git a/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/RealSenseSensor.cs b/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/RealSenseSensor.cs
index be2f4d903..67ddcf26a 100644
--- a/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/RealSenseSensor.cs
+++ b/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/RealSenseSensor.cs
@@ -22,7 +22,7 @@ public class RealSenseSensor : ISourceComponent, IDisposable
///
/// Initializes a new instance of the class.
///
- /// Pipeline this component is a part of.
+ /// The pipeline to add the component to.
public RealSenseSensor(Pipeline pipeline)
{
this.shutdown = false;
diff --git a/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/AssemblyInfo.cpp b/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/AssemblyInfo.cpp
index 6a39189a6..75a163494 100644
--- a/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/AssemblyInfo.cpp
+++ b/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/AssemblyInfo.cpp
@@ -31,9 +31,9 @@ using namespace System::Security::Permissions;
// You can specify all the value or you can default the Revision and Build Numbers
// by using the '*' as shown below:
-[assembly:AssemblyVersionAttribute("0.13.38.2")];
-[assembly:AssemblyFileVersionAttribute("0.13.38.2")];
-[assembly:AssemblyInformationalVersionAttribute("0.13.38.2-beta")];
+[assembly:AssemblyVersionAttribute("0.14.35.3")];
+[assembly:AssemblyFileVersionAttribute("0.14.35.3")];
+[assembly:AssemblyInformationalVersionAttribute("0.14.35.3-beta")];
[assembly:ComVisible(false)];
diff --git a/Sources/Runtime/Microsoft.Psi/Components/Timer.cs b/Sources/Runtime/Microsoft.Psi/Components/Timer.cs
index 7e687a214..088180b29 100644
--- a/Sources/Runtime/Microsoft.Psi/Components/Timer.cs
+++ b/Sources/Runtime/Microsoft.Psi/Components/Timer.cs
@@ -53,7 +53,7 @@ public abstract class Timer : ISourceComponent, IDisposable
/// Initializes a new instance of the class.
/// The timer fires off messages at the rate specified by timerInterval.
///
- /// The pipeline this component will be part of.
+ /// The pipeline to add the component to.
/// The timer firing interval, in ms.
public Timer(Pipeline pipeline, uint timerInterval)
{
diff --git a/Sources/Runtime/Microsoft.Psi/Data/IStreamReader.cs b/Sources/Runtime/Microsoft.Psi/Data/IStreamReader.cs
index 711f009a8..0ae2fb32a 100644
--- a/Sources/Runtime/Microsoft.Psi/Data/IStreamReader.cs
+++ b/Sources/Runtime/Microsoft.Psi/Data/IStreamReader.cs
@@ -5,6 +5,7 @@ namespace Microsoft.Psi.Data
{
using System;
using System.Collections.Generic;
+ using System.Runtime.Serialization;
using System.Threading;
///
@@ -55,8 +56,9 @@ public interface IStreamReader : IDisposable
/// The name of the stream to open.
/// The function to call for every message in this stream.
/// An optional allocator of messages.
+ /// The function to call if an error occurs when reading the stream.
/// The metadata describing the opened stream.
- IStreamMetadata OpenStream(string name, Action target, Func allocator = null);
+ IStreamMetadata OpenStream(string name, Action target, Func allocator = null, Action errorHandler = null);
///
/// Opens the specified stream for reading, in index form; providing only index entries to the target delegate.
diff --git a/Sources/Runtime/Microsoft.Psi/Data/PsiStore.cs b/Sources/Runtime/Microsoft.Psi/Data/PsiStore.cs
index 0311bbd09..6704db49c 100644
--- a/Sources/Runtime/Microsoft.Psi/Data/PsiStore.cs
+++ b/Sources/Runtime/Microsoft.Psi/Data/PsiStore.cs
@@ -379,6 +379,50 @@ public static void Concatenate(IEnumerable<(string Name, string Path)> storeFile
loggingCallback?.Invoke("Done.");
}
+ ///
+ /// Processes a \psi store, generating a new store.
+ ///
+ /// Predicate function determining whether to process a stream or else should be merely copied.
+ /// Processor action given stream metadata, importer and exporter.
+ /// The name and path of the store to process.
+ /// The name and path of the processed store.
+ /// Indicates whether to create a numbered subdirectory for each store generated by multiple calls to this method.
+ /// An optional progress reporter for progress updates.
+ /// An optional callback to which human-friendly information will be logged.
+ public static void Process(Func predicate, Action processor, (string Name, string Path) input, (string Name, string Path) output, bool createSubdirectory = true, IProgress progress = null, Action loggingCallback = null)
+ {
+ using var pipeline = Pipeline.Create();
+ PsiImporter inputStore = PsiStore.Open(pipeline, input.Name, input.Path);
+ Exporter outputStore = PsiStore.Create(pipeline, output.Name, output.Path, createSubdirectory, inputStore.Serializers);
+
+ // copy all streams not being processed from inputStore to outputStore
+ var maxStreamId = 0;
+ foreach (var streamInfo in inputStore.AvailableStreams)
+ {
+ if (!predicate(streamInfo))
+ {
+ maxStreamId = Math.Max(maxStreamId, streamInfo.Id);
+ inputStore.CopyStream(streamInfo.Name, outputStore);
+ }
+ }
+
+ // process stream
+ Pipeline.SetLastStreamId(maxStreamId); // using IDs beyond those copied above
+ foreach (var streamInfo in inputStore.AvailableStreams)
+ {
+ if (predicate(streamInfo))
+ {
+ processor(streamInfo, inputStore, outputStore);
+ }
+ }
+
+ // run the pipeline to process the store
+ loggingCallback?.Invoke("Processing store ...");
+ pipeline.RunAsync(null, progress);
+ pipeline.WaitAll();
+ loggingCallback?.Invoke("Done.");
+ }
+
///
/// Returns the metadata associated with the specified stream, if the stream is persisted to a \psi store.
///
diff --git a/Sources/Runtime/Microsoft.Psi/Data/PsiStoreStreamReader.cs b/Sources/Runtime/Microsoft.Psi/Data/PsiStoreStreamReader.cs
index 30d55c388..d5a9229ac 100644
--- a/Sources/Runtime/Microsoft.Psi/Data/PsiStoreStreamReader.cs
+++ b/Sources/Runtime/Microsoft.Psi/Data/PsiStoreStreamReader.cs
@@ -5,6 +5,7 @@ namespace Microsoft.Psi.Data
{
using System;
using System.Collections.Generic;
+ using System.Runtime.Serialization;
using System.Threading;
using Microsoft.Psi;
using Microsoft.Psi.Common;
@@ -17,6 +18,7 @@ namespace Microsoft.Psi.Data
public sealed class PsiStoreStreamReader : IStreamReader
{
private readonly Dictionary> targets = new Dictionary>();
+ private readonly Dictionary>> errorHandlers = new Dictionary>>();
private readonly Dictionary> outputs = new Dictionary>();
private readonly Dictionary> indexOutputs = new Dictionary>();
@@ -106,10 +108,10 @@ public bool IsLive()
}
///
- public IStreamMetadata OpenStream(string name, Action target, Func allocator = null)
+ public IStreamMetadata OpenStream(string name, Action target, Func allocator = null, Action errorHandler = null)
{
var meta = this.PsiStoreReader.OpenStream(name); // this checks for duplicates
- this.OpenStream(meta, target, allocator);
+ this.OpenStream(meta, target, allocator, errorHandler);
return meta;
}
@@ -183,11 +185,37 @@ public void ReadAll(ReplayDescriptor descriptor, CancellationToken cancelationTo
var indexEntry = this.PsiStoreReader.ReadIndex();
this.indexOutputs[e.SourceId](indexEntry, e);
}
- else
+ else if (this.outputs.ContainsKey(e.SourceId))
{
int count = this.PsiStoreReader.Read(ref this.buffer);
var bufferReader = new BufferReader(this.buffer, count);
- this.outputs[e.SourceId](bufferReader, e);
+
+ // Deserialize the data and call the listeners. Note that due to polymorphic types, we
+ // may be attempting to create some handlers on the fly which may result in a serialization
+ // exception being thrown due to type mismatch errors.
+ try
+ {
+ this.outputs[e.SourceId](bufferReader, e);
+ }
+ catch (SerializationException ex)
+ {
+ // If any error occurred while processing the message and there are
+ // registered error handlers, stop attempting to process messages
+ // from the stream and notify all registered error handler listeners.
+ // otherwise, rethrow the exception to exit the application.
+ if (this.errorHandlers.ContainsKey(e.SourceId))
+ {
+ this.outputs.Remove(e.SourceId);
+ foreach (Action errorAction in this.errorHandlers[e.SourceId])
+ {
+ errorAction.Invoke(ex);
+ }
+ }
+ else
+ {
+ throw;
+ }
+ }
}
}
}
@@ -229,55 +257,89 @@ private void LoadMetadata(IEnumerable metadata, RuntimeInfo runtimeVer
this.context.Serializers.RegisterMetadata(metadata);
}
- private void OpenStream(IStreamMetadata meta, Action target, Func allocator = null)
+ private void OpenStream(IStreamMetadata meta, Action target, Func allocator = null, Action errorHandler = null)
{
- // Get the deserialization handler for this stream type
- var handler = this.context.Serializers.GetHandler();
+ // If there's no list of targets for this stream, create it now
+ if (!this.targets.ContainsKey(meta.Id))
+ {
+ this.targets[meta.Id] = new List();
+ }
- var isDynamic = typeof(T).FullName == typeof(object).FullName;
- var isRaw = typeof(T).FullName == typeof(Message).FullName;
+ // Add the target to the list to call when this stream has new data
+ this.targets[meta.Id].Add(target);
- if (!isDynamic && !isRaw)
+ // Add the error handler, if any
+ if (errorHandler != null)
{
- // check that the requested type matches the stream type
- var streamType = meta.TypeName;
- var handlerType = handler.Name;
- if (streamType != handlerType)
+ if (!this.errorHandlers.ContainsKey(meta.Id))
{
- // check if the handler is able to handle the stream type
- if (handlerType != streamType)
+ this.errorHandlers[meta.Id] = new List>();
+ }
+
+ this.errorHandlers[meta.Id].Add(errorHandler);
+ }
+
+ // Get the deserialization handler for this stream type
+ SerializationHandler handler = null;
+ try
+ {
+ // A serialization exception may be thrown here if the handler is unable to be initialized due to a
+ // mismatch between the format of the messages in the stream and the current format of T, probably
+ // because a field has been added, removed, or renamed in T since the stream was created.
+ handler = this.context.Serializers.GetHandler();
+
+ var isDynamic = typeof(T).FullName == typeof(object).FullName;
+ var isRaw = typeof(T).FullName == typeof(Message).FullName;
+
+ if (!isDynamic && !isRaw)
+ {
+ // check that the requested type matches the stream type
+ var streamType = meta.TypeName;
+ var handlerType = handler.Name;
+ if (streamType != handlerType)
{
- if (this.context.Serializers.Schemas.TryGetValue(streamType, out var streamTypeSchema) &&
- this.context.Serializers.Schemas.TryGetValue(handlerType, out var handlerTypeSchema))
+ // check if the handler is able to handle the stream type
+ if (handlerType != streamType)
{
- // validate compatibility - will throw if types are incompatible
- handlerTypeSchema.ValidateCompatibleWith(streamTypeSchema);
+ if (this.context.Serializers.Schemas.TryGetValue(streamType, out var streamTypeSchema) &&
+ this.context.Serializers.Schemas.TryGetValue(handlerType, out var handlerTypeSchema))
+ {
+ // validate compatibility - will throw if types are incompatible
+ handlerTypeSchema.ValidateCompatibleWith(streamTypeSchema);
+ }
}
}
}
- }
- // If there's no list of targets for this stream, create it now
- if (!this.targets.ContainsKey(meta.Id))
- {
- this.targets[meta.Id] = new List();
- }
-
- // Add the target to the list to call when this stream has new data
- this.targets[meta.Id].Add(target);
+ // Update the code to execute when this stream receives new data
+ this.outputs[meta.Id] = (br, e) =>
+ {
+ // Deserialize the data
+ var data = this.Deserialize(handler, br, e, isDynamic, isRaw, (allocator == null) ? default(T) : allocator(), meta.TypeName, this.context.Serializers.Schemas);
- // Update the code to execute when this stream receives new data
- this.outputs[meta.Id] = (br, e) =>
+ // Call each of the targets
+ foreach (Delegate action in this.targets[meta.Id])
+ {
+ (action as Action)(data, e);
+ }
+ };
+ }
+ catch (SerializationException ex)
{
- // Deserialize the data
- var data = this.Deserialize(handler, br, e, isDynamic, isRaw, (allocator == null) ? default(T) : allocator(), meta.TypeName, this.context.Serializers.Schemas);
-
- // Call each of the targets
- foreach (Delegate action in this.targets[meta.Id])
+ // If there are any registered error handlers, call the ones registered for
+ // this stream, otherwise rethrow the exception to exit the application.
+ if (this.errorHandlers.ContainsKey(meta.Id))
+ {
+ foreach (Action errorAction in this.errorHandlers[meta.Id])
+ {
+ errorAction.Invoke(ex);
+ }
+ }
+ else
{
- (action as Action)(data, e);
+ throw;
}
- };
+ }
}
private T Deserialize(SerializationHandler handler, BufferReader br, Envelope env, bool isDynamic, bool isRaw, T objectToReuse, string typeName, IDictionary schemas)
diff --git a/Sources/Runtime/Microsoft.Psi/Data/Store.cs b/Sources/Runtime/Microsoft.Psi/Data/Store.cs
deleted file mode 100644
index e86a73daf..000000000
--- a/Sources/Runtime/Microsoft.Psi/Data/Store.cs
+++ /dev/null
@@ -1,221 +0,0 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT license.
-
-namespace Microsoft.Psi
-{
- using System;
- using System.Collections.Generic;
- using Microsoft.Psi.Data;
- using Microsoft.Psi.Serialization;
-
- ///
- /// OBSOLETE. Provides static methods to access multi-stream stores.
- ///
- public static class Store
- {
- ///
- /// OBSOLETE. Creates a new multi-stream store and returns an instance
- /// which can be used to write streams to this store.
- ///
- /// The that owns the .
- /// The name of the store to create.
- /// The path to use. If null, an in-memory store is created.
- /// Indicates whether to create a numbered subdirectory for each execution of the pipeline.
- /// An optional collection of custom serializers to use instead of the default ones.
- /// An instance that can be used to write streams.
- [Obsolete("Store APIs have moved to PsiStore.", true)]
- public static Exporter Create(Pipeline pipeline, string name, string rootPath, bool createSubdirectory = true, KnownSerializers serializers = null)
- {
- throw new NotImplementedException("Store APIs have moved to PsiStore.");
- }
-
- ///
- /// OBSOLETE. Opens a multi-stream store for read and returns an instance
- /// which can be used to inspect the store and open the streams.
- /// The store metadata is available immediately after this call (before the pipeline is running) via the property.
- ///
- /// The that owns the .
- /// The name of the store to open (the same as the catalog file name).
- ///
- /// The path to the store.
- /// This can be one of:
- /// - a full path to a directory containing the store
- /// - a root path containing one or more versions of the store, each in its own subdirectory,
- /// in which case the latest store is opened.
- /// - a null string, in which case an in-memory store is opened.
- ///
- /// Optional stream reader (default PsiStoreStreamReader).
- /// An instance that can be used to open streams and read messages.
- [Obsolete("Store APIs have moved to PsiStore.", true)]
- public static Importer Open(Pipeline pipeline, string name, string rootPath, IStreamReader streamReader = null)
- {
- throw new NotImplementedException("Store APIs have moved to PsiStore.");
- }
-
- ///
- /// OBSOLETE. Indicates whether the specified store file exists.
- ///
- /// The name of the store to check.
- /// The path of the store to check.
- /// Returns true if the store exists.
- [Obsolete("Store APIs have moved to PsiStore.", true)]
- public static bool Exists(string name, string path)
- {
- throw new NotImplementedException("Store APIs have moved to PsiStore.");
- }
-
- ///
- /// OBSOLETE. Indicates whether all streams in a store have been marked as "closed".
- ///
- /// The name of the store to check.
- /// The path of the store to check.
- /// Returns true if all streams in the store are marked as closed.
- [Obsolete("Store APIs have moved to PsiStore.", true)]
- public static bool IsClosed(string name, string path)
- {
- throw new NotImplementedException("Store APIs have moved to PsiStore.");
- }
-
- ///
- /// OBSOLETE. Repairs an invalid store in place.
- ///
- /// The name of the store to check.
- /// The path of the store to check.
- /// Indicates whether the original store should be deleted.
- [Obsolete("Store APIs have moved to PsiStore.", true)]
- public static void Repair(string name, string path, bool deleteOriginalStore = true)
- {
- throw new NotImplementedException("Store APIs have moved to PsiStore.");
- }
-
- ///
- /// OBSOLETE. Crops a store between the extents of a specified interval, generating a new store.
- ///
- /// The name and path of the store to crop.
- /// The name and path of the cropped store.
- /// Start of crop interval relative to beginning of store.
- /// Length of crop interval.
- ///
- /// Indicates whether to create a numbered subdirectory for each cropped store
- /// generated by multiple calls to this method.
- ///
- /// An optional progress reporter for progress updates.
- /// An optional callback to which human-friendly information will be logged.
- [Obsolete("Store APIs have moved to PsiStore.", true)]
- public static void Crop((string Name, string Path) input, (string Name, string Path) output, TimeSpan start, RelativeTimeInterval length, bool createSubdirectory = true, IProgress progress = null, Action loggingCallback = null)
- {
- throw new NotImplementedException("Store APIs have moved to PsiStore.");
- }
-
- ///
- /// OBSOLETE. Crops a store between the extents of a specified originating time interval, generating a new store.
- ///
- /// The name and path of the store to crop.
- /// The name and path of the cropped store.
- /// The originating time interval to which to crop the store.
- ///
- /// Indicates whether to create a numbered subdirectory for each cropped store
- /// generated by multiple calls to this method.
- ///
- /// An optional progress reporter for progress updates.
- /// An optional callback to which human-friendly information will be logged.
- [Obsolete("Store APIs have moved to PsiStore.", true)]
- public static void Crop((string Name, string Path) input, (string Name, string Path) output, TimeInterval cropInterval, bool createSubdirectory = true, IProgress progress = null, Action loggingCallback = null)
- {
- throw new NotImplementedException("Store APIs have moved to PsiStore.");
- }
-
- ///
- /// OBSOLETE. Crops a store in place between the extents of a specified interval.
- ///
- /// The name and path of the store to crop.
- /// Start of crop interval relative to beginning of store.
- /// Length of crop interval.
- /// Indicates whether the original store should be deleted.
- /// An optional progress reporter for progress updates.
- /// An optional callback to which human-friendly information will be logged.
- [Obsolete("Store APIs have moved to PsiStore.", true)]
- public static void CropInPlace((string Name, string Path) input, TimeSpan start, RelativeTimeInterval length, bool deleteOriginalStore = true, IProgress progress = null, Action loggingCallback = null)
- {
- throw new NotImplementedException("Store APIs have moved to PsiStore.");
- }
-
- ///
- /// OBSOLETE. Crops a store in place between the extents of a specified originating time interval.
- ///
- /// The name and path of the store to crop.
- /// The originating time interval to which to crop the store.
- /// Indicates whether the original store should be deleted.
- /// An optional progress reporter for progress updates.
- /// An optional callback to which human-friendly information will be logged.
- [Obsolete("Store APIs have moved to PsiStore.", true)]
- public static void CropInPlace((string Name, string Path) input, TimeInterval cropInterval, bool deleteOriginalStore = true, IProgress progress = null, Action loggingCallback = null)
- {
- throw new NotImplementedException("Store APIs have moved to PsiStore.");
- }
-
- ///
- /// OBSOLETE. Copies a store, or a subset of it.
- ///
- /// The name and path of the store to crop.
- /// The name and path of the cropped store.
- /// An optional function that defines an originating time interval to copy. By default, the extents of the entire store.
- /// An optional predicate that specifies which streams to include. By default, include all streams. By default, all streams are copied.
- ///
- /// Indicates whether to create a numbered subdirectory for each cropped store
- /// generated by multiple calls to this method.
- ///
- /// An optional progress reporter for progress updates.
- /// An optional callback to which human-friendly information will be logged.
- [Obsolete("Store APIs have moved to PsiStore.", true)]
- public static void Copy(
- (string Name, string Path) input,
- (string Name, string Path) output,
- Func cropIntervalFunction = null,
- Predicate includeStreamPredicate = null,
- bool createSubdirectory = true,
- IProgress progress = null,
- Action loggingCallback = null)
- {
- throw new NotImplementedException("Store APIs have moved to PsiStore.");
- }
-
- ///
- /// OBSOLETE. Concatenates a set of stores, generating a new store.
- ///
- /// Streams of the same name across stores must also have the same types as well as non-intersecting originating times.
- /// Set of store files (name, path pairs) to concatenate.
- /// Output store (name, path pair).
- /// An optional progress reporter for progress updates.
- /// An optional callback to which human-friendly information will be logged.
- [Obsolete("Store APIs have moved to PsiStore.", true)]
- public static void Concatenate(IEnumerable<(string Name, string Path)> storeFiles, (string Name, string Path) output, IProgress progress = null, Action loggingCallback = null)
- {
- throw new NotImplementedException("Store APIs have moved to PsiStore.");
- }
-
- ///
- /// OBSOLETE. Returns the metadata associated with the specified stream, if the stream is persisted to a store.
- ///
- /// The type of stream messages.
- /// The stream to retrieve metadata about.
- /// Upon return, this parameter contains the metadata associated with the stream, or null if the stream is not persisted.
- /// True if the stream is persisted to a store, false otherwise.
- public static bool TryGetStreamMetadata(IProducer source, out IStreamMetadata metadata)
- {
- throw new NotImplementedException("Store APIs have moved to PsiStore.");
- }
-
- ///
- /// OBSOLETE. Returns the metadata associated with the specified stream, if the stream is persisted to a store.
- ///
- /// The current pipeline.
- /// The name of the stream to retrieve metadata about.
- /// Upon return, this parameter contains the metadata associated with the stream, or null if the stream is not persisted.
- /// True if the stream is persisted to a store, false otherwise.
- public static bool TryGetStreamMetadata(Pipeline pipeline, string streamName, out IStreamMetadata metadata)
- {
- throw new NotImplementedException("Store APIs have moved to PsiStore.");
- }
- }
-}
\ No newline at end of file
diff --git a/Sources/Runtime/Microsoft.Psi/Serialization/KnownSerializers.cs b/Sources/Runtime/Microsoft.Psi/Serialization/KnownSerializers.cs
index 8119b6322..593fcd7de 100644
--- a/Sources/Runtime/Microsoft.Psi/Serialization/KnownSerializers.cs
+++ b/Sources/Runtime/Microsoft.Psi/Serialization/KnownSerializers.cs
@@ -492,15 +492,43 @@ private SerializationHandler AddHandler()
// initialize the serializer after the handler is registered,
// to make sure all handlers are registered before initialization runs and
// allow the serializer initialization code to find and cache the handlers for the types it needs
- schema = serializer.Initialize(this, schema);
+ try
+ {
+ schema = serializer.Initialize(this, schema);
- // let any subscribers know that we initialized a new serializer that publishes a schema
- if (schema != null)
+ // let any subscribers know that we initialized a new serializer that publishes a schema
+ if (schema != null)
+ {
+ // store the updated schema and override whatever is present already
+ this.schemas[schema.Name] = schema;
+ this.schemasById[schema.Id] = schema;
+ this.SchemaAdded?.Invoke(this, schema);
+ }
+ }
+ catch (SerializationException)
{
- // store the updated schema and override whatever is present already
- this.schemas[schema.Name] = schema;
- this.schemasById[schema.Id] = schema;
- this.SchemaAdded?.Invoke(this, schema);
+ // Even though we're going to rethrow this exception, some callers may wish to
+ // attempt to recover from this error and just mark this one stream type as
+ // unreadable. So we should remove the handler we just registered as it's not
+ // yet properly initialized.
+ oldCount = this.handlers.Length;
+ newHandlers = new SerializationHandler[oldCount - 1];
+ Array.Copy(this.handlers, newHandlers, oldCount - 1);
+ this.handlers = newHandlers;
+
+ newIndex = new Dictionary(this.index);
+ newIndex.Remove(handler);
+ this.index = newIndex;
+
+ newHandlersByType = new Dictionary(this.handlersByType);
+ newHandlersByType.Remove(type);
+ this.handlersByType = newHandlersByType;
+
+ newHandlersById = new Dictionary(this.handlersById);
+ newHandlersById.Remove(handler.Id);
+ this.handlersById = newHandlersById;
+
+ throw;
}
}
diff --git a/Sources/Tools/PsiStoreTool/Program.cs b/Sources/Tools/PsiStoreTool/Program.cs
index 9dd5772bb..474581c3a 100644
--- a/Sources/Tools/PsiStoreTool/Program.cs
+++ b/Sources/Tools/PsiStoreTool/Program.cs
@@ -38,7 +38,7 @@ private static int Main(string[] args)
Console.WriteLine($"Platform for Situated Intelligence Store Tool");
try
{
- return Parser.Default.ParseArguments(args)
+ return Parser.Default.ParseArguments(args)
.MapResult(
(Verbs.ListStreams opts) => Utility.ListStreams(opts.Store, opts.Path, opts.ShowSize),
(Verbs.Info opts) => Utility.DisplayStreamInfo(opts.Stream, opts.Store, opts.Path),
@@ -48,6 +48,7 @@ private static int Main(string[] args)
(Verbs.Send opts) => Utility.SendStreamMessages(opts.Stream, opts.Store, opts.Path, opts.Topic, opts.Address, opts.Format),
(Verbs.Concat opts) => Utility.ConcatenateStores(opts.Store, opts.Path, opts.Output),
(Verbs.Crop opts) => Utility.CropStore(opts.Store, opts.Path, opts.Output, opts.Start, opts.Length),
+ (Verbs.Encode opts) => Utility.EncodeStore(opts.Store, opts.Path, opts.Output, opts.Quality),
(Verbs.ListTasks opts) => Utility.ListTasks(opts.Assemblies),
(Verbs.Exec opts) => Utility.ExecuteTask(opts.Stream, opts.Store, opts.Path, opts.Name, opts.Assemblies, opts.Arguments),
DisplayParseErrors);
diff --git a/Sources/Tools/PsiStoreTool/PsiStoreTool.csproj b/Sources/Tools/PsiStoreTool/PsiStoreTool.csproj
index ccd3e11c3..0c82c92f3 100644
--- a/Sources/Tools/PsiStoreTool/PsiStoreTool.csproj
+++ b/Sources/Tools/PsiStoreTool/PsiStoreTool.csproj
@@ -41,6 +41,7 @@
+
diff --git a/Sources/Tools/PsiStoreTool/Utility.cs b/Sources/Tools/PsiStoreTool/Utility.cs
index 309cacd6c..e7d644644 100644
--- a/Sources/Tools/PsiStoreTool/Utility.cs
+++ b/Sources/Tools/PsiStoreTool/Utility.cs
@@ -16,6 +16,7 @@ namespace PsiStoreTool
using Microsoft.Psi;
using Microsoft.Psi.Common;
using Microsoft.Psi.Data;
+ using Microsoft.Psi.Imaging;
using Microsoft.Psi.Interop.Format;
using Microsoft.Psi.Interop.Transport;
@@ -284,6 +285,37 @@ internal static int CropStore(string store, string path, string output, string s
return 0;
}
+ ///
+ /// Encode image streams, generating a new store.
+ ///
+ /// Store name.
+ /// Store path.
+ /// Output store name.
+ /// Start time relative to beginning.
+ /// Success flag.
+ internal static int EncodeStore(string store, string path, string output, int quality)
+ {
+ Console.WriteLine($"Encoding store (store={store}, path={path}, output={output}, quality={quality}");
+
+ bool IsImageStream(IStreamMetadata streamInfo)
+ {
+ return streamInfo.TypeName.StartsWith("Microsoft.Psi.Shared`1[[Microsoft.Psi.Imaging.Image,");
+ }
+
+ void EncodeImageStreams(IStreamMetadata streamInfo, PsiImporter importer, Exporter exporter)
+ {
+ importer
+ .OpenStream>(streamInfo.Name)
+ .ToPixelFormat(PixelFormat.BGRA_32bpp)
+ .EncodeJpeg(quality)
+ .Write(streamInfo.Name, exporter, true);
+ }
+
+ PsiStore.Process(IsImageStream, EncodeImageStreams, (store, path), (output, path), true, new Progress(p => Console.WriteLine($"Progress: {p * 100.0:F2}%")), Console.WriteLine);
+
+ return 0;
+ }
+
///
/// List tasks discovered in assemblies given in app.config.
///
diff --git a/Sources/Tools/PsiStoreTool/Verbs.cs b/Sources/Tools/PsiStoreTool/Verbs.cs
index ebe77f413..d44fad25f 100644
--- a/Sources/Tools/PsiStoreTool/Verbs.cs
+++ b/Sources/Tools/PsiStoreTool/Verbs.cs
@@ -203,7 +203,7 @@ internal class Exec
internal class Crop : BaseStoreCommand
{
///
- /// Gets or sets name of Psi data store.
+ /// Gets or sets name of output Psi data store.
///
[Option('o', "output", Required = false, Default = "Cropped", HelpText = "Name of output Psi data store (default=Cropped).")]
public string Output { get; set; }
@@ -220,5 +220,24 @@ internal class Crop : BaseStoreCommand
[Option('l', "length", Required = false, HelpText = "Length of interval relative to start (default=unbounded).")]
public string Length { get; set; }
}
+
+ ///
+ /// Encode image streams verb.
+ ///
+ [Verb("encode", HelpText = "Encode image streams to JPEG.")]
+ internal class Encode : BaseStoreCommand
+ {
+ ///
+ /// Gets or sets name of output Psi data store.
+ ///
+ [Option('o', "output", Required = false, Default = "Encoded", HelpText = "Name of output Psi data store (default=Encoded).")]
+ public string Output { get; set; }
+
+ ///
+ /// Gets or sets quality of the JPEG compression.
+ ///
+ [Option('q', "quality", Default = 90, HelpText = "Quality of JPEG compression 0-100 (optional, default 90).")]
+ public int Quality { get; set; }
+ }
}
}
\ No newline at end of file
diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Controls/TimelineScroller.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Controls/TimelineScroller.cs
index f26a2f544..a0aadb9f1 100644
--- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Controls/TimelineScroller.cs
+++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Controls/TimelineScroller.cs
@@ -51,7 +51,11 @@ protected override void OnMouseMove(MouseEventArgs e)
if (snappedTime.HasValue)
{
- this.Navigator.Cursor = snappedTime.Value;
+ // Only snap to the point if it is within the current viewport.
+ if (this.Navigator.ViewRange.AsTimeInterval.PointIsWithin(snappedTime.Value))
+ {
+ this.Navigator.Cursor = snappedTime.Value;
+ }
}
else
{
diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataManager.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataManager.cs
index 495c0cb2b..3d20bee44 100644
--- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataManager.cs
+++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataManager.cs
@@ -76,6 +76,11 @@ private DataManager()
///
public event EventHandler DataStoreStatusChanged;
+ ///
+ /// Event that fires when a stream is unable to be read from.
+ ///
+ public event EventHandler StreamReadError;
+
///
public void Dispose()
{
@@ -171,6 +176,16 @@ public void OnInstantViewRangeChanged(TimeInterval viewRange)
}
}
+ ///
+ /// Checks if a stream is known to be unreadable.
+ ///
+ /// The stream source indicating which stream to read from.
+ /// True if the stream is currently considered readable, otherwise false.
+ public bool IsStreamUnreadable(StreamSource streamSource)
+ {
+ return this.GetOrCreateDataStoreReader(streamSource.StoreName, streamSource.StorePath, streamSource.StreamReaderType).IsStreamUnreadable(streamSource.StreamName);
+ }
+
///
/// Creates a view of the messages identified by the matching start and end times and asynchronously fills it in.
///
@@ -402,12 +417,15 @@ private void RunReadInstantDataTask()
}
catch (Exception ex)
{
- new MessageBoxWindow(
- Application.Current.MainWindow,
- "Instant Data Push Error",
- $"An error occurred while attempting to push instant data to the visualization objects{Environment.NewLine}{Environment.NewLine}{ex.Message}",
- "Close",
- null).ShowDialog();
+ Application.Current.Dispatcher.BeginInvoke((Action)(() =>
+ {
+ new MessageBoxWindow(
+ Application.Current.MainWindow,
+ "Instant Data Push Error",
+ $"An error occurred while attempting to push instant data to the visualization objects{Environment.NewLine}{Environment.NewLine}{ex.Message}",
+ "Close",
+ null).ShowDialog();
+ }));
}
}
else
@@ -493,6 +511,7 @@ private DataStoreReader GetOrCreateDataStoreReader(string storeName, string stor
else
{
DataStoreReader dataStoreReader = new DataStoreReader(storeName, storePath, streamReaderType);
+ dataStoreReader.StreamReadError += this.DataStoreReader_StreamReadError;
this.dataStoreReaders[key] = dataStoreReader;
return dataStoreReader;
}
@@ -517,7 +536,12 @@ private DataStoreReader GetDataStoreReader(string storeName, string storePath)
private void RemoveDataStoreReader(string storeName, string storePath)
{
ValueTuple key = (storeName, storePath);
- this.dataStoreReaders.Remove(key);
+
+ lock (this.dataStoreReaders)
+ {
+ this.dataStoreReaders[key].StreamReadError -= this.DataStoreReader_StreamReadError;
+ this.dataStoreReaders.Remove(key);
+ }
}
private List GetDataStoreReaderList()
@@ -529,6 +553,12 @@ private List GetDataStoreReaderList()
}
}
+ private void DataStoreReader_StreamReadError(object sender, StreamReadErrorEventArgs e)
+ {
+ // Alert the visualization objects that the stream is unreadable.
+ this.StreamReadError?.Invoke(this, e);
+ }
+
private StreamSummaryManager GetStreamSummaryManager(StreamSource streamSource)
{
if (streamSource == null)
diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataStoreReader.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataStoreReader.cs
index 9177e0264..455c93cfa 100644
--- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataStoreReader.cs
+++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataStoreReader.cs
@@ -4,9 +4,9 @@
namespace Microsoft.Psi.Visualization.Data
{
using System;
- using System.Collections;
using System.Collections.Generic;
using System.Linq;
+ using System.Runtime.Serialization;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Psi;
@@ -25,6 +25,12 @@ public class DataStoreReader : IDisposable
private string storeName;
private string storePath;
+ ///
+ /// The list of streams that have been identified as unreadable, probably due to the format of the
+ /// message on disk not matching the current format of the data object they are deserialized from.
+ ///
+ private List unreadableStreams = new List();
+
///
/// Initializes a new instance of the class.
///
@@ -40,6 +46,11 @@ internal DataStoreReader(string storeName, string storePath, Type streamReaderTy
this.streamCaches = new List();
}
+ ///
+ /// Event that fires when a stream is unable to be read from.
+ ///
+ public event EventHandler StreamReadError;
+
///
public void Dispose()
{
@@ -133,6 +144,19 @@ internal void OnInstantViewRangeChanged(TimeInterval viewRange)
}
}
+ ///
+ /// Checks if a stream is known to be unreadable.
+ ///
+ /// The name of the stream.
+ /// True if the stream is currently considered readable, otherwise false.
+ internal bool IsStreamUnreadable(string streamName)
+ {
+ lock (this.unreadableStreams)
+ {
+ return this.unreadableStreams.Contains(streamName);
+ }
+ }
+
///
/// Called to ask the reader to read the data for all instant streams.
///
@@ -143,9 +167,16 @@ internal void ReadInstantData(DateTime cursorTime)
{
foreach (IStreamCache streamCache in this.streamCaches.ToList())
{
- if (streamCache.HasInstantStreamReaders)
+ if (streamCache.HasInstantStreamReaders && (!this.unreadableStreams.Contains(streamCache.StreamName)))
{
- streamCache.ReadInstantData(reader, cursorTime);
+ try
+ {
+ streamCache.ReadInstantData(reader, cursorTime);
+ }
+ catch (SerializationException ex)
+ {
+ this.StreamCache_StreamReadError(this, new StreamReadErrorEventArgs() { StreamName = streamCache.StreamName, Exception = ex });
+ }
}
}
}
@@ -326,11 +357,12 @@ private IStreamCache GetExistingStreamCache(StreamSource streamSource, IStreamAd
private IStreamCache GetOrCreateStreamCacheByStreamSource(StreamSource streamSource, IStreamAdapter streamAdapter)
{
var streamName = streamSource.StreamName;
- var streamCache = this.streamCaches.Find(sr => sr.StreamName == streamName && sr.StreamAdapter == streamAdapter);
+ StreamCache streamCache = this.streamCaches.Find(sr => sr.StreamName == streamName && sr.StreamAdapter == streamAdapter) as StreamCache;
if (streamCache == null)
{
streamCache = new StreamCache(streamSource.StreamName, streamAdapter);
+ streamCache.StreamReadError += this.StreamCache_StreamReadError;
this.streamCaches.Add(streamCache);
}
@@ -339,17 +371,36 @@ private IStreamCache GetOrCreateStreamCacheByStreamSource(StreamSource stream
private IStreamCache GetOrCreateStreamCacheByName(string streamName, IStreamAdapter streamAdapter)
{
- var streamCache = this.streamCaches.Find(sr => sr.StreamName == streamName && sr.StreamAdapter == streamAdapter);
+ StreamCache streamCache = this.streamCaches.Find(sr => sr.StreamName == streamName && sr.StreamAdapter == streamAdapter) as StreamCache;
if (streamCache == null)
{
streamCache = new StreamCache(streamName, streamAdapter);
+ streamCache.StreamReadError += this.StreamCache_StreamReadError;
this.streamCaches.Add(streamCache);
}
return streamCache;
}
+ private void StreamCache_StreamReadError(object sender, StreamReadErrorEventArgs e)
+ {
+ // Add the stream to the list of known unreadable streams if it's not already there.
+ lock (this.unreadableStreams)
+ {
+ if (!this.unreadableStreams.Contains(e.StreamName))
+ {
+ this.unreadableStreams.Add(e.StreamName);
+ }
+ }
+
+ // Add the store name and path and propagate the message to the data manager
+ e.StoreName = this.storeName;
+ e.StorePath = this.storePath;
+
+ this.StreamReadError?.Invoke(this, e);
+ }
+
private struct ExecutionContext
{
public IStreamReader Reader;
diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/EpsilonInstantStreamReader{T}.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/EpsilonInstantStreamReader{T}.cs
index 3f33581b0..a1280902f 100644
--- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/EpsilonInstantStreamReader{T}.cs
+++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/EpsilonInstantStreamReader{T}.cs
@@ -18,6 +18,11 @@ namespace Microsoft.Psi.Visualization.Data
/// The type of messages in the stream.
public class EpsilonInstantStreamReader
{
+ ///
+ /// The stream name.
+ ///
+ private readonly string streamName;
+
///
/// Flag indicating whether type parameter T is Shared{} or not.
///
@@ -31,9 +36,11 @@ public class EpsilonInstantStreamReader
///
/// Initializes a new instance of the class.
///
+ /// The name of the stream.
/// The cursor epsilon to use when searching for messages around a cursor time.
- public EpsilonInstantStreamReader(RelativeTimeInterval cursorEpsilon)
+ public EpsilonInstantStreamReader(string streamName, RelativeTimeInterval cursorEpsilon)
{
+ this.streamName = streamName;
this.CursorEpsilon = cursorEpsilon;
this.dataProviders = new List>();
}
@@ -124,6 +131,17 @@ public void ReadInstantData(IStreamReader streamReader, DateTime cursorTime, Obs
// Read the data
data = cacheEntry.Read(streamReader);
}
+ else
+ {
+ // cache miss, attempt to seek directly while the cache is presumably being populated
+ streamReader.Seek(cursorTime + this.CursorEpsilon, true);
+ streamReader.OpenStream(this.streamName, (m, e) =>
+ {
+ cacheEntry = new StreamCacheEntry(null, e.CreationTime, e.OriginatingTime);
+ data = m;
+ });
+ streamReader.MoveNext(out var envelope);
+ }
// Notify each adapting data provider of the new data
foreach (IAdaptingInstantDataProvider adaptingInstantDataProvider in this.dataProviders.ToList())
diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/StreamCache{T}.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/StreamCache{T}.cs
index 7d09527f9..898226452 100644
--- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/StreamCache{T}.cs
+++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/StreamCache{T}.cs
@@ -9,6 +9,7 @@ namespace Microsoft.Psi.Visualization.Data
using System.Collections.Specialized;
using System.Linq;
using System.Reflection;
+ using System.Runtime.Serialization;
using Microsoft.Psi;
using Microsoft.Psi.Data;
using Microsoft.Psi.Visualization.Collections;
@@ -98,6 +99,11 @@ public StreamCache(string streamName, IStreamAdapter streamAdapter/*, object[] s
}
}
+ ///
+ /// Event that fires when a stream is unable to be read from.
+ ///
+ public event EventHandler StreamReadError;
+
///
/// Gets shared allocator.
///
@@ -318,13 +324,14 @@ public void OpenStream(IStreamReader streamReader, bool readIndicesOnly)
{
if (this.StreamAdapter == null)
{
- streamReader.OpenStream(this.StreamName, this.OnReceiveData, this.Allocator);
+ streamReader.OpenStream(this.StreamName, this.OnReceiveData, this.Allocator, this.OnReadError);
}
else
{
dynamic dynStreamAdapter = this.StreamAdapter;
dynamic dynAdaptedReceiver = dynStreamAdapter.AdaptReceiver(new Action(this.OnReceiveData));
- streamReader.OpenStream(this.StreamName, dynAdaptedReceiver, dynStreamAdapter.Allocator);
+ dynamic dynReadError = new Action(this.OnReadError);
+ streamReader.OpenStream(this.StreamName, dynAdaptedReceiver, dynStreamAdapter.Allocator, dynReadError);
}
}
}
@@ -559,7 +566,7 @@ private EpsilonInstantStreamReader GetInstantStreamReader(RelativeTimeInterva
if (createIfNecessary)
{
// Create the instant stream reader
- instantStreamReader = new EpsilonInstantStreamReader(cursorEpsilon);
+ instantStreamReader = new EpsilonInstantStreamReader(this.StreamName, cursorEpsilon);
this.instantStreamReaders.Add(instantStreamReader);
}
else
@@ -635,5 +642,11 @@ private void OnReceiveIndex(Func indexThunk, Envelope env)
}
}
}
+
+ private void OnReadError(SerializationException ex)
+ {
+ // Notify the data store reader
+ this.StreamReadError?.Invoke(this, new StreamReadErrorEventArgs() { StreamName = this.StreamName, Exception = ex });
+ }
}
}
diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/StreamReadErrorEventArgs.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/StreamReadErrorEventArgs.cs
new file mode 100644
index 000000000..ea72629a0
--- /dev/null
+++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/StreamReadErrorEventArgs.cs
@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+namespace Microsoft.Psi.Visualization.Data
+{
+ using System;
+ using System.Runtime.Serialization;
+
+ ///
+ /// Represents the event arguments passed by the stream read error event of .
+ ///
+ public class StreamReadErrorEventArgs : EventArgs
+ {
+ ///
+ /// Gets or sets the name of the store containing the stream that was unable to be read.
+ ///
+ public string StoreName { get; set; }
+
+ ///
+ /// Gets or sets the path to the store containing the stream that was unable to be read.
+ ///
+ public string StorePath { get; set; }
+
+ ///
+ /// Gets or sets the name of the stream that was unable to be read.
+ ///
+ public string StreamName { get; set; }
+
+ ///
+ /// Gets or sets the serialization exception that occurred.
+ ///
+ public SerializationException Exception { get; set; }
+ }
+}
diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/PluginMap.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/PluginMap.cs
index 4342798e1..ee6f3dab5 100644
--- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/PluginMap.cs
+++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/PluginMap.cs
@@ -272,14 +272,15 @@ public Type GetStreamReaderType(string extension)
// We force-add the latency visualizer b/c it's not detectable by data type
// (the adapter to make it work will be added automatically later in
- // CustomizeVisualizerMetadata.
+ // CustomizeVisualizerMetadata). Latency visualizer is only compatible with
+ // timeline visualization panels.
if (isUniversal.HasValue && isUniversal.Value)
{
if (isInNewPanel.HasValue && isInNewPanel.Value)
{
results.Add(this.visualizers.FirstOrDefault(v => v.CommandText == ContextMenuName.VisualizeLatencyInNewPanel));
}
- else
+ else if (visualizationPanel is TimelineVisualizationPanel)
{
results.Add(this.visualizers.FirstOrDefault(v => v.CommandText == ContextMenuName.VisualizeLatency));
}
diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/PartitionViewModel.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/PartitionViewModel.cs
index b72343481..043bc9c0c 100644
--- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/PartitionViewModel.cs
+++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/PartitionViewModel.cs
@@ -290,21 +290,29 @@ public bool IsPsiPartition
/// A stream source if a source was found to bind to, otherwise returns null.
public StreamSource GetStreamSource(StreamBinding streamBinding)
{
+ StreamSource streamSource = null;
+
// Check if the partition contains the required stream
IStreamMetadata streamMetadata = this.Partition.AvailableStreams.FirstOrDefault(s => s.Name == streamBinding.StreamName);
if (streamMetadata != default)
{
- // We have found a source to bind to
- return new StreamSource(
+ // Create the stream source
+ streamSource = new StreamSource(
this,
TypeResolutionHelper.GetVerifiedType(this.StreamReaderTypeName),
streamBinding.StreamName,
streamMetadata,
streamBinding.StreamAdapter,
streamBinding.Summarizer);
+
+ // Check with data manager as to whether the stream is already known to be unreadable
+ if (DataManager.Instance.IsStreamUnreadable(streamSource))
+ {
+ streamSource = null;
+ }
}
- return null;
+ return streamSource;
}
///
diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/TimelineVisualizationPanelView.xaml.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/TimelineVisualizationPanelView.xaml.cs
index 1fdbe83d5..a49199410 100644
--- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/TimelineVisualizationPanelView.xaml.cs
+++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/TimelineVisualizationPanelView.xaml.cs
@@ -9,8 +9,6 @@ namespace Microsoft.Psi.Visualization.Views
using System.Windows.Input;
using System.Windows.Media;
using System.Windows.Media.Imaging;
- using Microsoft.Psi.PsiStudio;
- using Microsoft.Psi.PsiStudio.Common;
using Microsoft.Psi.Visualization;
using Microsoft.Psi.Visualization.Helpers;
using Microsoft.Psi.Visualization.Navigation;
@@ -52,6 +50,16 @@ public void FinishDragDrop()
this.currentDragOperation = DragOperation.None;
}
+ ///
+ /// Notifies of a change in mouse position while a context menu is being displayed.
+ ///
+ /// The sender of the event.
+ /// The mouse event args.
+ public void ContextMenuMouseMove(object sender, MouseEventArgs e)
+ {
+ this.lastMousePosition = e.GetPosition(this);
+ }
+
///
protected override void OnRenderSizeChanged(SizeChangedInfo sizeInfo)
{
@@ -74,7 +82,15 @@ private void Root_ContextMenuOpening(object sender, ContextMenuEventArgs e)
}
else
{
- this.VisualizationPanel.OnContextMenuOpening(sender);
+ // Create the context menu if it doesn't yet exist
+ FrameworkElement senderElement = sender as FrameworkElement;
+ if (senderElement.ContextMenu == null)
+ {
+ senderElement.ContextMenu = new ContextMenu();
+ senderElement.ContextMenu.AddHandler(MouseMoveEvent, new MouseEventHandler(this.ContextMenuMouseMove), true);
+ }
+
+ this.VisualizationPanel.OnContextMenuOpening(senderElement.ContextMenu);
}
}
@@ -82,8 +98,8 @@ private void Root_MouseMove(object sender, MouseEventArgs e)
{
Point mousePosition = e.GetPosition(this);
- // If the user has the Left Mouse button pressed, initiate a Drag & Drop reorder operation
- if (e.LeftButton == MouseButtonState.Pressed)
+ // If the user has the Left Mouse button pressed, initiate a Drag & Drop reorder operation.
+ if ((e.LeftButton == MouseButtonState.Pressed) && (mousePosition != this.lastMousePosition))
{
switch (this.currentDragOperation)
{
@@ -115,7 +131,6 @@ private void BeginDragOperation(Point mousePosition)
if (VisualizationContext.Instance.VisualizationContainer.Navigator.CursorMode == CursorMode.Manual)
{
this.currentDragOperation = DragOperation.TimelineScroll;
- this.DoDragTimeline(mousePosition);
this.Cursor = Cursors.Hand;
}
}
diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/VisualizationContainerView.xaml.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/VisualizationContainerView.xaml.cs
index 3a52618a2..95a6ea147 100644
--- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/VisualizationContainerView.xaml.cs
+++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/VisualizationContainerView.xaml.cs
@@ -4,7 +4,6 @@
namespace Microsoft.Psi.Visualization.Views
{
using System;
- using System.Collections.Generic;
using System.Linq;
using System.Windows;
using System.Windows.Controls;
diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml
index 9afaa9970..4078451a7 100644
--- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml
+++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml
@@ -33,6 +33,9 @@
+
+
+
@@ -40,6 +43,21 @@
+
+
+
+
+
+
+
diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml.cs
index e2551a1c7..c664058b7 100644
--- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml.cs
+++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml.cs
@@ -3,10 +3,17 @@
namespace Microsoft.Psi.Visualization.Views.Visuals2D
{
+ using System;
+ using System.Collections;
using System.Collections.Generic;
using System.ComponentModel;
+ using System.Reflection;
+ using System.Windows;
+ using System.Windows.Controls;
+ using System.Windows.Input;
using System.Windows.Media;
using Microsoft.Psi.Data.Annotations;
+ using Microsoft.Psi.Visualization.Helpers;
using Microsoft.Psi.Visualization.VisualizationObjects;
///
@@ -47,6 +54,22 @@ internal Brush GetBrush(System.Drawing.Color systemDrawingColor)
return this.brushes[color];
}
+ ///
+ protected override void OnDataContextChanged(object sender, DependencyPropertyChangedEventArgs e)
+ {
+ if (e.OldValue is TimeIntervalAnnotationVisualizationObject oldVisualizationObject)
+ {
+ oldVisualizationObject.TimeIntervalAnnotationEdit -= this.VisualizationObject_TimeIntervalAnnotationEdit;
+ }
+
+ if (e.NewValue is TimeIntervalAnnotationVisualizationObject newVisualizationObject)
+ {
+ newVisualizationObject.TimeIntervalAnnotationEdit += this.VisualizationObject_TimeIntervalAnnotationEdit;
+ }
+
+ base.OnDataContextChanged(sender, e);
+ }
+
///
protected override void OnVisualizationObjectPropertyChanged(object sender, PropertyChangedEventArgs e)
{
@@ -74,6 +97,139 @@ protected override void OnTransformsChanged()
this.Rerender();
}
+ private static Color ToMediaColor(System.Drawing.Color color)
+ {
+ return Color.FromArgb(color.A, color.R, color.G, color.B);
+ }
+
+ private void VisualizationObject_TimeIntervalAnnotationEdit(object sender, TimeIntervalAnnotationEditEventArgs e)
+ {
+ // If we were already editing an unrestricted annotation, stop.
+ this.EditUnrestrictedAnnotationTextBox.Visibility = Visibility.Collapsed;
+
+ // Check if there's an annotation to edit.
+ if (e.DisplayData != null)
+ {
+ // Check if a finite or an unrestricted annotation value is being edited
+ if (e.DisplayData.Definition.SchemaDefinitions[e.TrackId].Schema.IsFiniteAnnotationSchema)
+ {
+ this.EditFiniteAnnotationValue(e.DisplayData, e.TrackId);
+ }
+ else
+ {
+ this.EditUnrestrictedAnnotationValue(e.DisplayData, e.TrackId);
+ }
+ }
+ }
+
+ private void EditFiniteAnnotationValue(TimeIntervalAnnotationDisplayData displayData, int trackId)
+ {
+ // Get the schema definition
+ AnnotationSchemaDefinition schemaDefinition = displayData.Definition.SchemaDefinitions[trackId];
+
+ // Get the collection of possible values
+ Type schemaType = schemaDefinition.Schema.GetType();
+ MethodInfo valuesProperty = schemaType.GetProperty("Values").GetGetMethod();
+ IEnumerable values = (IEnumerable)valuesProperty.Invoke(schemaDefinition.Schema, new object[] { });
+
+ // Create a new context menu
+ ContextMenu contextMenu = new ContextMenu();
+
+ // Create a menuitem for each value, with a command to update the value on the annotation.
+ foreach (object value in values)
+ {
+ var metadata = this.GetAnnotationValueMetadata(value, schemaDefinition.Schema);
+ contextMenu.Items.Add(MenuItemHelper.CreateAnnotationMenuItem(
+ value.ToString(),
+ metadata.BorderColor,
+ metadata.FillColor,
+ new PsiCommand(() => this.VisualizationObject.SetAnnotationValue(displayData.Annotation, schemaDefinition.Name, value))));
+ }
+
+ // Add a handler so that the timeline visualization panel continues to receive mouse move messages
+ // while the context menu is displayed, and remove the handler once the context menu closes.
+ MouseEventHandler mouseMoveHandler = new MouseEventHandler(this.FindTimelineVisualizationPanelView().ContextMenuMouseMove);
+ contextMenu.AddHandler(MouseMoveEvent, mouseMoveHandler, true);
+ contextMenu.Closed += (sender, e) => contextMenu.RemoveHandler(MouseMoveEvent, mouseMoveHandler);
+
+ // Show the context menu
+ contextMenu.IsOpen = true;
+ }
+
+ private void EditUnrestrictedAnnotationValue(TimeIntervalAnnotationDisplayData displayData, int trackId)
+ {
+ // Get the schema definition
+ AnnotationSchemaDefinition schemaDefinition = displayData.Definition.SchemaDefinitions[trackId];
+
+ // Get the current value
+ object value = displayData.Annotation.Data.Values[schemaDefinition.Name];
+
+ // Get the associated metadata
+ AnnotationSchemaValueMetadata schemaMetadata = this.GetAnnotationValueMetadata(value, schemaDefinition.Schema);
+
+ // Style the text box to match the schema of the annotation value
+ this.EditUnrestrictedAnnotationTextBox.Foreground = new SolidColorBrush(ToMediaColor(schemaMetadata.TextColor));
+ this.EditUnrestrictedAnnotationTextBox.Background = new SolidColorBrush(ToMediaColor(schemaMetadata.FillColor));
+ this.EditUnrestrictedAnnotationTextBox.BorderBrush = new SolidColorBrush(ToMediaColor(schemaMetadata.BorderColor));
+ this.EditUnrestrictedAnnotationTextBox.BorderThickness = new Thickness(schemaMetadata.BorderWidth);
+
+ // The textbox's tag holds context information to allow us to update the value in the display object when
+ // the text in the textbox changes. Note that we must set the correct tag before we set the text, otherwise
+ // setting the text will cause it to be copied to the last annotation that we edited.
+ this.EditUnrestrictedAnnotationTextBox.Tag = new UnrestrictedAnnotationValueContext(displayData, schemaDefinition.Name);
+ this.EditUnrestrictedAnnotationTextBox.Text = value.ToString();
+
+ // Set the textbox position to exactly cover the annotation value
+ var navigatorViewDuration = this.Navigator.ViewRange.Duration.TotalSeconds;
+ var labelStart = Math.Min(navigatorViewDuration, Math.Max((displayData.StartTime - this.Navigator.ViewRange.StartTime).TotalSeconds, 0));
+ var labelEnd = Math.Max(0, Math.Min((displayData.EndTime - this.Navigator.ViewRange.StartTime).TotalSeconds, navigatorViewDuration));
+
+ var verticalSpace = this.VisualizationObject.Padding / this.ScaleTransform.ScaleY;
+ var lo = (double)(trackId + verticalSpace) / this.VisualizationObject.TrackCount;
+ var hi = (double)(trackId + 1 - verticalSpace) / this.VisualizationObject.TrackCount;
+
+ this.EditUnrestrictedAnnotationTextBox.Width = (labelEnd - labelStart) * this.Canvas.ActualWidth / this.Navigator.ViewRange.Duration.TotalSeconds;
+ this.EditUnrestrictedAnnotationTextBox.Height = (hi - lo) * this.Canvas.ActualHeight;
+ (this.EditUnrestrictedAnnotationTextBox.RenderTransform as TranslateTransform).X = labelStart * this.Canvas.ActualWidth / this.Navigator.ViewRange.Duration.TotalSeconds;
+ (this.EditUnrestrictedAnnotationTextBox.RenderTransform as TranslateTransform).Y = lo * this.Canvas.ActualHeight;
+
+ // Initially select the entire text of the textbox, then show the textbox and set keyboard focus to it.
+ this.EditUnrestrictedAnnotationTextBox.SelectAll();
+ this.EditUnrestrictedAnnotationTextBox.Visibility = Visibility.Visible;
+ this.EditUnrestrictedAnnotationTextBox.Focus();
+ }
+
+ private void EditUnrestrictedAnnotationTextBox_TextChanged(object sender, TextChangedEventArgs e)
+ {
+ UnrestrictedAnnotationValueContext context = this.EditUnrestrictedAnnotationTextBox.Tag as UnrestrictedAnnotationValueContext;
+ if (context != null)
+ {
+ context.DisplayData.SetValue(context.ValueName, this.EditUnrestrictedAnnotationTextBox.Text);
+ }
+ }
+
+ private void TextBox_LostFocus(object sender, RoutedEventArgs e)
+ {
+ this.EditUnrestrictedAnnotationTextBox.Visibility = Visibility.Collapsed;
+ }
+
+ private AnnotationSchemaValueMetadata GetAnnotationValueMetadata(object value, IAnnotationSchema annotationSchema)
+ {
+ MethodInfo getMetadataProperty = annotationSchema.GetType().GetMethod("GetMetadata");
+ return (AnnotationSchemaValueMetadata)getMetadataProperty.Invoke(annotationSchema, new[] { value });
+ }
+
+ private TimelineVisualizationPanelView FindTimelineVisualizationPanelView()
+ {
+ DependencyObject timelinePanelView = this.VisualParent;
+ while (!(timelinePanelView is TimelineVisualizationPanelView))
+ {
+ timelinePanelView = VisualTreeHelper.GetParent(timelinePanelView);
+ }
+
+ return timelinePanelView as TimelineVisualizationPanelView;
+ }
+
private void Rerender()
{
for (int i = 0; i < this.VisualizationObject.DisplayData.Count; i++)
@@ -101,5 +257,18 @@ private void Rerender()
this.items.Remove(item);
}
}
+
+ private class UnrestrictedAnnotationValueContext
+ {
+ public UnrestrictedAnnotationValueContext(TimeIntervalAnnotationDisplayData displayData, string valueName)
+ {
+ this.DisplayData = displayData;
+ this.ValueName = valueName;
+ }
+
+ public TimeIntervalAnnotationDisplayData DisplayData { get; private set; }
+
+ public string ValueName { get; private set; }
+ }
}
}
\ No newline at end of file
diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationEditEventArgs.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationEditEventArgs.cs
new file mode 100644
index 000000000..779bb0576
--- /dev/null
+++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationEditEventArgs.cs
@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+namespace Microsoft.Psi.Visualization.VisualizationObjects
+{
+ using System;
+
+ ///
+ /// Provides data for editing a time interval annotation value.
+ ///
+ public class TimeIntervalAnnotationEditEventArgs : EventArgs
+ {
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The annotation display object to edit.
+ /// The id of the track in the display data to edit.
+ public TimeIntervalAnnotationEditEventArgs(TimeIntervalAnnotationDisplayData displayData, int trackId)
+ {
+ this.DisplayData = displayData;
+ this.TrackId = trackId;
+ }
+
+ ///
+ /// Gets the annotation to edit.
+ ///
+ public TimeIntervalAnnotationDisplayData DisplayData { get; private set; }
+
+ ///
+ /// Gets the ID of the track in the annotation to edit.
+ ///
+ public int TrackId { get; private set; }
+ }
+}
diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationVisualizationObject.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationVisualizationObject.cs
index 504ce5df7..d170483ad 100644
--- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationVisualizationObject.cs
+++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationVisualizationObject.cs
@@ -4,7 +4,6 @@
namespace Microsoft.Psi.Visualization.VisualizationObjects
{
using System;
- using System.Collections;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.ComponentModel;
@@ -23,6 +22,8 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects
using Microsoft.Psi.Visualization.Data;
using Microsoft.Psi.Visualization.Helpers;
using Microsoft.Psi.Visualization.Views.Visuals2D;
+ using Microsoft.Psi.Visualization.VisualizationPanels;
+ using Microsoft.Psi.Visualization.Windows;
///
/// Class implements a .
@@ -30,6 +31,11 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects
[VisualizationObject("Time Interval Annotations")]
public class TimeIntervalAnnotationVisualizationObject : TimelineVisualizationObject
{
+ private const string ErrorStreamNotBound = "The visualization object is not currently bound to a stream.";
+ private const string ErrorEditingDisabled = "Annotation add/delete is currently disabled in the visualization object properties.";
+ private const string ErrorSelectionMarkersUnset = "Both the start and end selection markers must be set.\r\n\r\nYou can set the start and end selection markers with SHIFT + Left mouse button and SHIFT + Right mouse button.";
+ private const string ErrorOverlappingAnnotations = "Time interval annotations may not overlap.";
+
private double padding = 0;
private double lineWidth = 2;
private double fontSize = 10;
@@ -38,6 +44,7 @@ public class TimeIntervalAnnotationVisualizationObject : TimelineVisualizationOb
private RelayCommand nextAnnotationCommand;
private RelayCommand previousAnnotationCommand;
private RelayCommand mouseLeftButtonDownCommand;
+ private RelayCommand mouseRightButtonDownCommand;
private RelayCommand mouseMoveCommand;
private RelayCommand mouseLeftButtonUpCommand;
private RelayCommand mouseDoubleClickCommand;
@@ -46,6 +53,11 @@ public class TimeIntervalAnnotationVisualizationObject : TimelineVisualizationOb
private TimeIntervalAnnotationDragInfo annotationDragInfo = null;
+ ///
+ /// Event that fires when an annotation value should be edited in the view.
+ ///
+ public event EventHandler TimeIntervalAnnotationEdit;
+
private enum AnnotationEdge
{
None,
@@ -144,7 +156,7 @@ public override string IconSource
}
///
- /// Gets the mouse move command.
+ /// Gets the mouse left button down command.
///
[Browsable(false)]
public RelayCommand MouseLeftButtonDownCommand
@@ -153,7 +165,6 @@ public RelayCommand MouseLeftButtonDownCommand
{
if (this.mouseLeftButtonDownCommand == null)
{
- // Ensure playback is stopped before exiting
this.mouseLeftButtonDownCommand = new RelayCommand(
(e) =>
{
@@ -165,6 +176,27 @@ public RelayCommand MouseLeftButtonDownCommand
}
}
+ ///
+ /// Gets the mouse right button down command.
+ ///
+ [Browsable(false)]
+ public RelayCommand MouseRightButtonDownCommand
+ {
+ get
+ {
+ if (this.mouseRightButtonDownCommand == null)
+ {
+ this.mouseRightButtonDownCommand = new RelayCommand(
+ (e) =>
+ {
+ this.DoMouseRightButtonDown(e);
+ });
+ }
+
+ return this.mouseRightButtonDownCommand;
+ }
+ }
+
///
/// Gets the mouse move command.
///
@@ -175,7 +207,6 @@ public RelayCommand MouseMoveCommand
{
if (this.mouseMoveCommand == null)
{
- // Ensure playback is stopped before exiting
this.mouseMoveCommand = new RelayCommand(
(e) =>
{
@@ -197,7 +228,6 @@ public RelayCommand MouseLeftButtonUpCommand
{
if (this.mouseLeftButtonUpCommand == null)
{
- // Ensure playback is stopped before exiting
this.mouseLeftButtonUpCommand = new RelayCommand(
(e) =>
{
@@ -219,7 +249,6 @@ public RelayCommand MouseDoubleClickCommand
{
if (this.mouseDoubleClickCommand == null)
{
- // Ensure playback is stopped before exiting
this.mouseDoubleClickCommand = new RelayCommand(
(e) =>
{
@@ -303,12 +332,15 @@ public override IEnumerable