From eee896b9926846a4876e9745e87483c6ed86f38a Mon Sep 17 00:00:00 2001 From: Stuart Dent Date: Mon, 7 Dec 2020 23:17:13 +0000 Subject: [PATCH] Merged PR 48123: Release 0.14.35.3 Release 0.14.35.1 --- Directory.Build.props | 2 +- .../WaveFileStreamReader.cs | 3 +- .../Microsoft.Psi.Calibration/ProjectTo3D.cs | 2 +- .../Microsoft.Psi.Data/Json/JsonGenerator.cs | 4 +- .../Json/JsonStreamReader.cs | 3 +- .../Microsoft.Psi.Imaging/ImageTransformer.cs | 2 +- .../Properties/AssemblyInfo.cs | 6 +- .../ImageAnalyzer.cs | 2 +- .../ImageNet/ImageNetModelOutputParser.cs | 73 ++++ .../ImageNet/ImageNetModelRunner.cs | 141 ++++++++ .../ImageNetModelRunnerConfiguration.cs | 204 +++++++++++ .../ImageNet/LabeledPrediction.cs | 21 ++ .../Onnx/Test.Psi.Onnx/ImageNetTester.cs | 220 +++++++++++ .../Onnx/Test.Psi.Onnx/TinyYoloV2Tester.cs | 16 +- .../AzureKinectCore.cs | 2 +- .../Microsoft.Psi.Media.Linux/MediaCapture.cs | 6 +- .../MediaCapture.cs | 6 +- .../MediaSource.cs | 4 +- .../Microsoft.Psi.Media.Windows.x64.csproj | 6 + .../Mpeg4Writer.cs | 6 +- .../VisualCapture.cs | 76 ++++ .../VisualCaptureConfiguration.cs | 41 +++ .../WindowCapture.cs | 145 ++++++++ .../WindowCaptureConfiguration.cs | 28 ++ .../AssemblyInfo.cpp | 6 +- .../AssemblyInfo.rc | Bin 5104 -> 5104 bytes Sources/Media/Shared/FFMPEGMediaSource.cs | 2 +- .../Properties/AssemblyInfo.cs | 6 +- .../RealSenseSensor.cs | 2 +- .../AssemblyInfo.cpp | 6 +- .../Runtime/Microsoft.Psi/Components/Timer.cs | 2 +- .../Microsoft.Psi/Data/IStreamReader.cs | 4 +- .../Runtime/Microsoft.Psi/Data/PsiStore.cs | 44 +++ .../Data/PsiStoreStreamReader.cs | 138 +++++-- Sources/Runtime/Microsoft.Psi/Data/Store.cs | 221 ------------ .../Serialization/KnownSerializers.cs | 42 ++- Sources/Tools/PsiStoreTool/Program.cs | 3 +- .../Tools/PsiStoreTool/PsiStoreTool.csproj | 1 + Sources/Tools/PsiStoreTool/Utility.cs | 32 ++ Sources/Tools/PsiStoreTool/Verbs.cs | 21 +- .../Controls/TimelineScroller.cs | 6 +- .../Data/DataManager.cs | 44 ++- .../Data/DataStoreReader.cs | 61 +++- .../Data/EpsilonInstantStreamReader{T}.cs | 20 +- .../Data/StreamCache{T}.cs | 19 +- .../Data/StreamReadErrorEventArgs.cs | 34 ++ .../PluginMap.cs | 5 +- .../ViewModels/PartitionViewModel.cs | 14 +- .../TimelineVisualizationPanelView.xaml.cs | 27 +- .../Views/VisualizationContainerView.xaml.cs | 1 - ...rvalAnnotationVisualizationObjectView.xaml | 18 + ...lAnnotationVisualizationObjectView.xaml.cs | 169 +++++++++ .../TimeIntervalAnnotationEditEventArgs.cs | 34 ++ ...meIntervalAnnotationVisualizationObject.cs | 341 ++++++++++-------- .../ImageWithIntrinsicsVisualizationObject.cs | 4 +- .../InstantVisualizationObject{TData}.cs | 1 + .../StreamVisualizationObject{TData}.cs | 34 ++ .../InstantVisualizationContainer.cs | 7 + .../InstantVisualizationPlaceholderPanel.cs | 8 + .../TimelineVisualizationPanel.cs | 115 +++--- .../Properties/AssemblyInfo.cs | 6 +- 61 files changed, 1963 insertions(+), 554 deletions(-) create mode 100644 Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelOutputParser.cs create mode 100644 Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunner.cs create mode 100644 Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunnerConfiguration.cs create mode 100644 Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/LabeledPrediction.cs create mode 100644 Sources/Integrations/Onnx/Test.Psi.Onnx/ImageNetTester.cs create mode 100644 Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCapture.cs create mode 100644 Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCaptureConfiguration.cs create mode 100644 Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCapture.cs create mode 100644 Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCaptureConfiguration.cs delete mode 100644 Sources/Runtime/Microsoft.Psi/Data/Store.cs create mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/StreamReadErrorEventArgs.cs create mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationEditEventArgs.cs diff --git a/Directory.Build.props b/Directory.Build.props index f16615819..4ab8566be 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -6,7 +6,7 @@ Microsoft Corporation microsoft,psi Microsoft - 0.13.38.2 + 0.14.35.3 $(AssemblyVersion) $(AssemblyVersion)-beta false diff --git a/Sources/Audio/Microsoft.Psi.Audio/WaveFileStreamReader.cs b/Sources/Audio/Microsoft.Psi.Audio/WaveFileStreamReader.cs index 2e00efa61..6804a52e5 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/WaveFileStreamReader.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/WaveFileStreamReader.cs @@ -6,6 +6,7 @@ namespace Microsoft.Psi.Audio using System; using System.Collections.Generic; using System.IO; + using System.Runtime.Serialization; using System.Threading; using Microsoft.Psi; using Microsoft.Psi.Data; @@ -160,7 +161,7 @@ public IStreamReader OpenNew() } /// - public IStreamMetadata OpenStream(string name, Action target, Func allocator = null) + public IStreamMetadata OpenStream(string name, Action target, Func allocator = null, Action errorHandler = null) { ValidateStreamName(name); diff --git a/Sources/Calibration/Microsoft.Psi.Calibration/ProjectTo3D.cs b/Sources/Calibration/Microsoft.Psi.Calibration/ProjectTo3D.cs index 4f4569e77..8603bcaaf 100644 --- a/Sources/Calibration/Microsoft.Psi.Calibration/ProjectTo3D.cs +++ b/Sources/Calibration/Microsoft.Psi.Calibration/ProjectTo3D.cs @@ -20,7 +20,7 @@ public sealed class ProjectTo3D : ConsumerProducer<(Shared, List /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. public ProjectTo3D(Pipeline pipeline) : base(pipeline) { diff --git a/Sources/Data/Microsoft.Psi.Data/Json/JsonGenerator.cs b/Sources/Data/Microsoft.Psi.Data/Json/JsonGenerator.cs index bc3bed80a..1cb6c04ce 100644 --- a/Sources/Data/Microsoft.Psi.Data/Json/JsonGenerator.cs +++ b/Sources/Data/Microsoft.Psi.Data/Json/JsonGenerator.cs @@ -22,7 +22,7 @@ public class JsonGenerator : Generator, IDisposable /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. /// The name of the application that generated the persisted files, or the root name of the files. /// The directory in which the main persisted file resides. public JsonGenerator(Pipeline pipeline, string name, string path) @@ -33,7 +33,7 @@ public JsonGenerator(Pipeline pipeline, string name, string path) /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. /// The underlying store reader. protected JsonGenerator(Pipeline pipeline, JsonStoreReader reader) : base(pipeline) diff --git a/Sources/Data/Microsoft.Psi.Data/Json/JsonStreamReader.cs b/Sources/Data/Microsoft.Psi.Data/Json/JsonStreamReader.cs index 7fbcfc7b3..27b9f80ba 100644 --- a/Sources/Data/Microsoft.Psi.Data/Json/JsonStreamReader.cs +++ b/Sources/Data/Microsoft.Psi.Data/Json/JsonStreamReader.cs @@ -5,6 +5,7 @@ namespace Microsoft.Psi.Data.Json { using System; using System.Collections.Generic; + using System.Runtime.Serialization; using System.Threading; using Microsoft.Psi; using Newtonsoft.Json.Linq; @@ -125,7 +126,7 @@ public virtual IStreamReader OpenNew() } /// - public IStreamMetadata OpenStream(string streamName, Action target, Func allocator = null) + public IStreamMetadata OpenStream(string streamName, Action target, Func allocator = null, Action errorHandler = null) { if (string.IsNullOrWhiteSpace(streamName)) { diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/ImageTransformer.cs b/Sources/Imaging/Microsoft.Psi.Imaging/ImageTransformer.cs index 25d84d551..7ab8544b7 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/ImageTransformer.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/ImageTransformer.cs @@ -25,7 +25,7 @@ public class ImageTransformer : ConsumerProducer, Shared> /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. /// Function for transforming the source image. /// Pixel format for destination image. /// Optional image allocator for creating new shared image. diff --git a/Sources/Imaging/Test.Psi.Imaging.Windows/Properties/AssemblyInfo.cs b/Sources/Imaging/Test.Psi.Imaging.Windows/Properties/AssemblyInfo.cs index 3d256e184..74ac24e7d 100644 --- a/Sources/Imaging/Test.Psi.Imaging.Windows/Properties/AssemblyInfo.cs +++ b/Sources/Imaging/Test.Psi.Imaging.Windows/Properties/AssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyCopyright("Copyright (c) Microsoft Corporation. All rights reserved.")] [assembly: ComVisible(false)] [assembly: Guid("191df615-3d8f-45a3-b763-dd4a604a712a")] -[assembly: AssemblyVersion("0.13.38.2")] -[assembly: AssemblyFileVersion("0.13.38.2")] -[assembly: AssemblyInformationalVersion("0.13.38.2-beta")] +[assembly: AssemblyVersion("0.14.35.3")] +[assembly: AssemblyFileVersion("0.14.35.3")] +[assembly: AssemblyInformationalVersion("0.14.35.3-beta")] diff --git a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/ImageAnalyzer.cs b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/ImageAnalyzer.cs index 34df8fc52..255a65a79 100644 --- a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/ImageAnalyzer.cs +++ b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/ImageAnalyzer.cs @@ -238,7 +238,7 @@ private async Task ReceiveAsync(Shared data, Envelope e) { var analysisResult = default(ImageAnalysis); - if (data != null) + if (data != null && data.Resource != null) { using Stream imageFileStream = new MemoryStream(); diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelOutputParser.cs b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelOutputParser.cs new file mode 100644 index 000000000..485dc3d81 --- /dev/null +++ b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelOutputParser.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Onnx +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using Microsoft.Psi; + + /// + /// Internal class that parses the outputs from the ImageNet model into + /// a set of image classification results. + /// + internal class ImageNetModelOutputParser + { + private readonly string[] labels; + private readonly int maxPredictions; + private readonly bool applySoftmax; + + /// + /// Initializes a new instance of the class. + /// + /// The path to the file containing the list of 1000 ImageNet classes. + /// The maximum number of predictions to return. + /// Whether the softmax function should be applied to the raw model output. + /// + /// The file referenced by may be downloaded from the following location: + /// https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt. + /// + public ImageNetModelOutputParser(string imageClassesFile, int maxPredictions, bool applySoftmax) + { + this.labels = File.ReadAllLines(imageClassesFile); + if (this.labels.Length != 1000) + { + throw new ArgumentException($"The file {imageClassesFile} does not appear to be in the correct format. This file should contain exactly 1000 lines representing an ordered list of the 1000 ImageNet classes."); + } + + this.maxPredictions = maxPredictions; + this.applySoftmax = applySoftmax; + } + + /// + /// Gets the predictions from the model output. + /// + /// The model output vector of class probabilities. + /// A list of the top-N predictions, in descending probability order. + public List GetPredictions(float[] modelOutput) + { + return GetTopResults(this.applySoftmax ? Softmax(modelOutput) : modelOutput, this.maxPredictions) + .Select(c => new LabeledPrediction { Label = this.labels[c.Index], Confidence = c.Value }) + .ToList(); + } + + private static IEnumerable<(int Index, float Value)> GetTopResults(IEnumerable predictedClasses, int count) + { + return predictedClasses + .Select((predictedClass, index) => (Index: index, Value: predictedClass)) + .OrderByDescending(result => result.Value) + .Take(count); + } + + private static IEnumerable Softmax(IEnumerable values) + { + var maxVal = values.Max(); + var exp = values.Select(v => Math.Exp(v - maxVal)); + var sumExp = exp.Sum(); + + return exp.Select(v => (float)(v / sumExp)); + } + } +} diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunner.cs b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunner.cs new file mode 100644 index 000000000..6186a5e9e --- /dev/null +++ b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunner.cs @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Onnx +{ + using System; + using System.Collections.Generic; + using Microsoft.Psi; + using Microsoft.Psi.Components; + using Microsoft.Psi.Imaging; + + /// + /// Component that runs an ImageNet image classification model. + /// + /// + /// This class implements a \psi component that runs an ONNX model trained + /// on the ImageNet dataset that operates on 224x224 RGB images and scores + /// the image for each of the 1000 ImageNet classes. It takes an input + /// stream of \psi images, applies a center-crop, rescales and normalizes + /// the pixel values into the input vector expected by the model. It also + /// parses the model outputs into a list of + /// values, corresponding to the top N predictions by the model. For + /// convenience, a set of pre-defined model runner configurations are + /// defined for a number of image classification models available in the + /// ONNX Model Zoo (https://github.com/onnx/models/tree/master/vision/classification). + /// The ONNX model file for the corresponding configuration will need to be + /// downloaded locally and the path to the model file will need to be + /// specified when creating the configuration. + /// + public class ImageNetModelRunner : ConsumerProducer, List> + { + private readonly float[] onnxInputVector = new float[3 * 224 * 224]; + private readonly OnnxModel onnxModel; + private readonly ImageNetModelOutputParser outputParser; + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// The configuration for the compoinent. + /// + /// To run on a GPU, use the Microsoft.Psi.Onnx.ModelRunners.Gpu library instead of Microsoft.Psi.Onnx.ModelRunners.Cpu, and set + /// the value of the parameter to a valid non-negative integer. Typical device ID values are 0 or 1. + /// + public ImageNetModelRunner(Pipeline pipeline, ImageNetModelRunnerConfiguration configuration) + : base(pipeline) + { + // create an ONNX model based on the supplied ImageNet model runner configuration + this.onnxModel = new OnnxModel(new OnnxModelConfiguration() + { + ModelFileName = configuration.ModelFilePath, + InputVectorName = configuration.InputVectorName, + InputVectorSize = 3 * 224 * 224, + OutputVectorName = configuration.OutputVectorName, + GpuDeviceId = configuration.GpuDeviceId, + }); + + this.outputParser = new ImageNetModelOutputParser(configuration.ImageClassesFilePath, configuration.NumberOfPredictions, configuration.ApplySoftmaxToModelOutput); + } + + /// + protected override void Receive(Shared data, Envelope envelope) + { + // construct the ONNX model input vector (stored in this.onnxInputVector) + // based on the incoming image + this.ConstructOnnxInputVector(data); + + // run the model over the input vector + var outputVector = this.onnxModel.GetPrediction(this.onnxInputVector); + + // parse the model output into an ordered list of the top-N predictions + var results = this.outputParser.GetPredictions(outputVector); + + // post the results + this.Out.Post(results, envelope.OriginatingTime); + } + + /// + /// Constructs the input vector for the ImageNet model for a specified image. + /// + /// The image to construct the input vector for. + private void ConstructOnnxInputVector(Shared sharedImage) + { + var inputImage = sharedImage.Resource; + var inputWidth = sharedImage.Resource.Width; + var inputHeight = sharedImage.Resource.Height; + + // crop a center square + var squareSize = Math.Min(inputWidth, inputHeight); + using var squareImage = ImagePool.GetOrCreate(squareSize, squareSize, sharedImage.Resource.PixelFormat); + if (inputWidth > inputHeight) + { + inputImage.Crop(squareImage.Resource, (inputWidth - squareSize) / 2, 0, squareSize, squareSize); + } + else + { + inputImage.Crop(squareImage.Resource, 0, (inputHeight - squareSize) / 2, squareSize, squareSize); + } + + // resize the image to 224 x 224 + using var resizedImage = ImagePool.GetOrCreate(224, 224, sharedImage.Resource.PixelFormat); + squareImage.Resource.Resize(resizedImage.Resource, 224, 224, SamplingMode.Bilinear); + + // if the pixel format does not match, do a conversion before extracting the bytes + var bytes = default(byte[]); + if (sharedImage.Resource.PixelFormat != PixelFormat.BGR_24bpp) + { + using var reformattedImage = ImagePool.GetOrCreate(224, 224, PixelFormat.BGR_24bpp); + resizedImage.Resource.CopyTo(reformattedImage.Resource); + bytes = reformattedImage.Resource.ReadBytes(3 * 224 * 224); + } + else + { + // get the bytes + bytes = resizedImage.Resource.ReadBytes(3 * 224 * 224); + } + + // Now populate the onnxInputVector float array / tensor by normalizing + // using mean = [0.485, 0.456, 0.406] and std = [0.229, 0.224, 0.225]. + int fi = 0; + + // first the red bytes + for (int i = 2; i < bytes.Length; i += 3) + { + this.onnxInputVector[fi++] = ((bytes[i] / 255.0f) - 0.485f) / 0.229f; + } + + // then the green bytes + for (int i = 1; i < bytes.Length; i += 3) + { + this.onnxInputVector[fi++] = ((bytes[i] / 255.0f) - 0.456f) / 0.224f; + } + + // then the blue bytes + for (int i = 0; i < bytes.Length; i += 3) + { + this.onnxInputVector[fi++] = ((bytes[i] / 255.0f) - 0.406f) / 0.225f; + } + } + } +} diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunnerConfiguration.cs b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunnerConfiguration.cs new file mode 100644 index 000000000..56383aec9 --- /dev/null +++ b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunnerConfiguration.cs @@ -0,0 +1,204 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Onnx +{ + /// + /// Represents the configuration for the class. + /// + /// + /// For convenience, a set of pre-defined model runner configurations are defined for a few image + /// classification models via static creation methods. The model files themselves are available + /// in the ONNX Model Zoo (https://github.com/onnx/models/tree/master/vision/classification). They + /// will need to be downloaded separately and the path to the model file will need to be supplied + /// when creating the configuration. + /// + public class ImageNetModelRunnerConfiguration + { + /// + /// Gets or sets the path to the model file. + /// + public string ModelFilePath { get; set; } + + /// + /// Gets or sets the path to a text file containing the 1000 ImageNet classes. + /// + public string ImageClassesFilePath { get; set; } + + /// + /// Gets or sets the name of the input vector. + /// + public string InputVectorName { get; set; } + + /// + /// Gets or sets the name of the output vector. + /// + public string OutputVectorName { get; set; } + + /// + /// Gets or sets the number of predictions to include in the output. The + /// top-N predictions (where N = NumberOfPredictions) are output by the + /// component in order of decreasing probability. + /// + public int NumberOfPredictions { get; set; } + + /// + /// Gets or sets a value indicating whether or not to apply the softmax + /// function to the raw model scores to obtain the class probabilities. + /// + public bool ApplySoftmaxToModelOutput { get; set; } + + /// + /// Gets or sets the GPU device ID to run on, or null to run on CPU. + /// + /// + /// To run on a GPU, use the Microsoft.Psi.Onnx.Gpu library instead of Microsoft.Psi.Onnx.Cpu, and set the value of + /// the property to a valid non-negative integer. Typical device ID values are 0 or 1. + /// + public int? GpuDeviceId { get; set; } + + /// + /// Creates an configuration for the ResNet50-caffe2 model, available at + /// https://github.com/onnx/models/raw/b9a54e89508f101a1611cd64f4ef56b9cb62c7cf/vision/classification/resnet/model/resnet50-caffe2-v1-7.onnx + /// and licensed at that commit under the Apache 2.0 License. This file must be downloaded separately + /// and the path to it should be specified in the parameter. Additionally, + /// the path to a file containing the 1000 ImageNet classes must also be supplied in the + /// , similar to the one available at + /// https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt. + /// + /// The path to the model file. + /// The path to the ImageNet classes file. + /// The number of predictions that the component should output. + /// The GPU device ID to run on, or null to run on CPU. + /// + /// To run on a GPU, use the Microsoft.Psi.Onnx.Gpu library instead of Microsoft.Psi.Onnx.Cpu, and set the value of + /// the property to a valid non-negative integer. Typical device ID values are 0 or 1. + /// + /// The model runner configuration. + public static ImageNetModelRunnerConfiguration CreateResNet50Caffe2( + string modelFilePath, + string imageClassesFilePath, + int numberOfPredictions = 5, + int? gpuDeviceId = null) + { + return new ImageNetModelRunnerConfiguration + { + ModelFilePath = modelFilePath, + ImageClassesFilePath = imageClassesFilePath, + InputVectorName = "gpu_0/data_0", + OutputVectorName = "gpu_0/softmax_1", + ApplySoftmaxToModelOutput = false, + NumberOfPredictions = numberOfPredictions, + GpuDeviceId = gpuDeviceId, + }; + } + + /// + /// Creates an configuration for the ResNet50-V2 model, available at + /// https://github.com/onnx/models/raw/b9a54e89508f101a1611cd64f4ef56b9cb62c7cf/vision/classification/resnet/model/resnet50-v2-7.onnx + /// and licensed at that commit under the Apache 2.0 Licese. This file must be downloaded separately + /// and the path to it should be specified in the parameter. Additionally, + /// the path to a file containing the 1000 ImageNet classes must also be supplied in the + /// , similar to the one available at + /// https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt. + /// + /// The path to the model file. + /// The path to the ImageNet classes file. + /// The number of predictions that the component should output. + /// The GPU device ID to run on, or null to run on CPU. + /// + /// To run on a GPU, use the Microsoft.Psi.Onnx.Gpu library instead of Microsoft.Psi.Onnx.Cpu, and set the value of + /// the property to a valid non-negative integer. Typical device ID values are 0 or 1. + /// + /// The model runner configuration. + public static ImageNetModelRunnerConfiguration CreateResNet50v2( + string modelFilePath, + string imageClassesFilePath, + int numberOfPredictions = 5, + int? gpuDeviceId = null) + { + return new ImageNetModelRunnerConfiguration + { + ModelFilePath = modelFilePath, + ImageClassesFilePath = imageClassesFilePath, + InputVectorName = "data", + OutputVectorName = "resnetv24_dense0_fwd", + ApplySoftmaxToModelOutput = true, + NumberOfPredictions = numberOfPredictions, + GpuDeviceId = gpuDeviceId, + }; + } + + /// + /// Creates an configuration for the VGG 16 model, available at + /// https://github.com/onnx/models/raw/f884b33c3e2371952aad7ea091898f418c830fe5/vision/classification/vgg/model/vgg16-7.onnx + /// and licensed at that commit under the Apache 2.0 License. This file must be downloaded separately + /// and the path to it should be specified in the parameter. Additionally, + /// the path to a file containing the 1000 ImageNet classes must also be supplied in the + /// , similar to the one available at + /// https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt. + /// + /// The path to the model file. + /// The path to the ImageNet classes file. + /// The number of predictions that the component should output. + /// The GPU device ID to run on, or null to run on CPU. + /// + /// To run on a GPU, use the Microsoft.Psi.Onnx.Gpu library instead of Microsoft.Psi.Onnx.Cpu, and set the value of + /// the property to a valid non-negative integer. Typical device ID values are 0 or 1. + /// + /// The model runner configuration. + public static ImageNetModelRunnerConfiguration CreateVgg16( + string modelFilePath, + string imageClassesFilePath, + int numberOfPredictions = 5, + int? gpuDeviceId = null) + { + return new ImageNetModelRunnerConfiguration + { + ModelFilePath = modelFilePath, + ImageClassesFilePath = imageClassesFilePath, + InputVectorName = "data", + OutputVectorName = "vgg0_dense2_fwd", + ApplySoftmaxToModelOutput = true, + NumberOfPredictions = numberOfPredictions, + GpuDeviceId = gpuDeviceId, + }; + } + + /// + /// Creates an configuration for the ShuffleNetV2 model, available at + /// https://github.com/onnx/models/raw/b9a54e89508f101a1611cd64f4ef56b9cb62c7cf/vision/classification/shufflenet/model/shufflenet-v2-10.onnx + /// and licensed at that commit under the BSD 3-Clause License. This file must be downloaded separately + /// and the path to it should be specified in the parameter. Additionally, + /// the path to a file containing the 1000 ImageNet classes must also be supplied in the + /// , similar to the one available at + /// https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt. + /// + /// The path to the model file. + /// The path to the ImageNet classes file. + /// The number of predictions that the component should output. + /// The GPU device ID to run on, or null to run on CPU. + /// + /// To run on a GPU, use the Microsoft.Psi.Onnx.Gpu library instead of Microsoft.Psi.Onnx.Cpu, and set the value of + /// the property to a valid non-negative integer. Typical device ID values are 0 or 1. + /// + /// The model runner configuration. + public static ImageNetModelRunnerConfiguration CreateShuffleNet2( + string modelFilePath, + string imageClassesFilePath, + int numberOfPredictions = 5, + int? gpuDeviceId = null) + { + return new ImageNetModelRunnerConfiguration + { + ModelFilePath = modelFilePath, + ImageClassesFilePath = imageClassesFilePath, + InputVectorName = "input", + OutputVectorName = "output", + ApplySoftmaxToModelOutput = true, + NumberOfPredictions = numberOfPredictions, + GpuDeviceId = gpuDeviceId, + }; + } + } +} diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/LabeledPrediction.cs b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/LabeledPrediction.cs new file mode 100644 index 000000000..72998d628 --- /dev/null +++ b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/LabeledPrediction.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Onnx +{ + /// + /// Represents a labeled prediction. + /// + public class LabeledPrediction + { + /// + /// Gets or sets the class label. + /// + public string Label { get; set; } + + /// + /// Gets or sets the confidence level. + /// + public float Confidence { get; set; } + } +} diff --git a/Sources/Integrations/Onnx/Test.Psi.Onnx/ImageNetTester.cs b/Sources/Integrations/Onnx/Test.Psi.Onnx/ImageNetTester.cs new file mode 100644 index 000000000..522a08b1d --- /dev/null +++ b/Sources/Integrations/Onnx/Test.Psi.Onnx/ImageNetTester.cs @@ -0,0 +1,220 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Test.Psi.Onnx +{ + using System.Collections.Generic; + using System.IO; + using System.Threading; + using Microsoft.Psi; + using Microsoft.Psi.Imaging; + using Microsoft.Psi.Onnx; + using Microsoft.VisualStudio.TestTools.UnitTesting; + using Test.Psi.Common; + + [TestClass] + public class ImageNetTester + { + [TestMethod] + [Timeout(60000)] + public void ResNet50Caffe2ImageClassificationTest() + { + // This test expects a set of resources at the path specified by the PsiTestResources environment + // variable. Specifically, it expects a subfolder named ResNet containing the file: + // - resnet50-caffe2-v1-7.onnx: this is the model, available from the onnx model zoo: + // https://github.com/onnx/models/raw/b9a54e89508f101a1611cd64f4ef56b9cb62c7cf/vision/classification/resnet/model/resnet50-caffe2-v1-7.onnx + // (licensed at that commit under the Apache 2.0 License) + // a subfolder ImageNet containing the file: + // - synset.txt: this is a file containing an ordered list of the 1000 ImageNet classes, available from the onnx model zoo: + // https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt + // and a subfolder TestImages containing the file: + // - image1.jpg: this is the test image, available from the ML.NET samples repo, + // https://github.com/dotnet/machinelearning-samples/tree/ffac232a1d599903b8591b68dabd237af48f462f/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx + // in the ObjectDetectionConsoleApp/assets/images folder + + if (TestRunner.TestResourcesPath != null) + { + string modelFile = Path.Combine(TestRunner.TestResourcesPath, "ResNet", "resnet50-caffe2-v1-7.onnx"); + string imageClassesFile = Path.Combine(TestRunner.TestResourcesPath, "ImageNet", "synset.txt"); + string testImageFile = Path.Combine(TestRunner.TestResourcesPath, "TestImages", "image1.jpg"); + + // create a configuration for the ResNet50Caffe2 model + var modelConfig = ImageNetModelRunnerConfiguration.CreateResNet50Caffe2(modelFile, imageClassesFile); + var testImage = Shared.Create(Image.FromBitmap(new System.Drawing.Bitmap(testImageFile))); + + this.ImageClassificationTest( + modelConfig, + testImage, + new[] { + ("n03100240 convertible", 0.8695966f), + ("n03930630 pickup, pickup truck", 0.04933356f), + ("n02814533 beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", 0.03959884f), + ("n03594945 jeep, landrover", 0.0146017177f), + ("n02974003 car wheel", 0.00834592152f), + }); + } + else + { + Assert.Inconclusive($"Test not run because 'PsiTestResources' environment variable not found."); + } + } + + [TestMethod] + [Timeout(60000)] + public void ResNet50V2ImageClassificationTest() + { + // This test expects a set of resources at the path specified by the PsiTestResources environment + // variable. Specifically, it expects a subfolder named ResNet containing the file: + // - resnet50-v2-7.onnx: this is the model, available from the onnx model zoo: + // https://github.com/onnx/models/raw/b9a54e89508f101a1611cd64f4ef56b9cb62c7cf/vision/classification/resnet/model/resnet50-v2-7.onnx + // (licensed at that commit under the Apache 2.0 License) + // a subfolder ImageNet containing the file: + // - synset.txt: this is a file containing an ordered list of the 1000 ImageNet classes, available from the onnx model zoo: + // https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt + // and a subfolder TestImages containing the file: + // - image1.jpg: this is the test image, available from the ML.NET samples repo, + // https://github.com/dotnet/machinelearning-samples/tree/ffac232a1d599903b8591b68dabd237af48f462f/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx + // in the ObjectDetectionConsoleApp/assets/images folder + + if (TestRunner.TestResourcesPath != null) + { + string modelFile = Path.Combine(TestRunner.TestResourcesPath, "ResNet", "resnet50-v2-7.onnx"); + string imageClassesFile = Path.Combine(TestRunner.TestResourcesPath, "ImageNet", "synset.txt"); + string testImageFile = Path.Combine(TestRunner.TestResourcesPath, "TestImages", "image1.jpg"); + + // create a configuration for the ResNet50v2 model + var modelConfig = ImageNetModelRunnerConfiguration.CreateResNet50v2(modelFile, imageClassesFile); + var testImage = Shared.Create(Image.FromBitmap(new System.Drawing.Bitmap(testImageFile))); + + this.ImageClassificationTest( + modelConfig, + testImage, + new[] { + ("n03930630 pickup, pickup truck", 0.6400269f), + ("n03100240 convertible", 0.30403322f), + ("n04461696 tow truck, tow car, wrecker", 0.0135363257f), + ("n02974003 car wheel", 0.0124500105f), + ("n03459775 grille, radiator grille", 0.005335613f), + }); + } + else + { + Assert.Inconclusive($"Test not run because 'PsiTestResources' environment variable not found."); + } + } + + [TestMethod] + [Timeout(60000)] + public void Vgg16ImageClassificationTest() + { + // This test expects a set of resources at the path specified by the PsiTestResources environment + // variable. Specifically, it expects a subfolder named VGG containing the file: + // - vgg16-7.onnx: this is the model, available from the onnx model zoo: + // https://github.com/onnx/models/raw/f884b33c3e2371952aad7ea091898f418c830fe5/vision/classification/vgg/model/vgg16-7.onnx + // (licensed at that commit under the Apache 2.0 License) + // a subfolder ImageNet containing the file: + // - synset.txt: this is a file containing an ordered list of the 1000 ImageNet classes, available from the onnx model zoo: + // https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt + // and a subfolder TestImages containing the file: + // - image1.jpg: this is the test image, available from the ML.NET samples repo, + // https://github.com/dotnet/machinelearning-samples/tree/ffac232a1d599903b8591b68dabd237af48f462f/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx + // in the ObjectDetectionConsoleApp/assets/images folder + + if (TestRunner.TestResourcesPath != null) + { + string modelFile = Path.Combine(TestRunner.TestResourcesPath, "VGG", "vgg16-7.onnx"); + string imageClassesFile = Path.Combine(TestRunner.TestResourcesPath, "ImageNet", "synset.txt"); + string testImageFile = Path.Combine(TestRunner.TestResourcesPath, "TestImages", "image1.jpg"); + + // create a configuration for the VGG 16 model + var modelConfig = ImageNetModelRunnerConfiguration.CreateVgg16(modelFile, imageClassesFile); + var testImage = Shared.Create(Image.FromBitmap(new System.Drawing.Bitmap(testImageFile))); + + this.ImageClassificationTest( + modelConfig, + testImage, + new[] { + ("n03100240 convertible", 0.7319748f), + ("n03930630 pickup, pickup truck", 0.129529521f), + ("n03594945 jeep, landrover", 0.0262848679f), + ("n03459775 grille, radiator grille", 0.0258834548f), + ("n02974003 car wheel", 0.0252302475f), + }); + } + else + { + Assert.Inconclusive($"Test not run because 'PsiTestResources' environment variable not found."); + } + } + + [TestMethod] + [Timeout(60000)] + public void ShuffleNet2ImageClassificationTest() + { + // This test expects a set of resources at the path specified by the PsiTestResources environment + // variable. Specifically, it expects a subfolder named ShuffleNet containing the file: + // - shufflenet-v2-10: this is the model, available from the onnx model zoo: + // https://github.com/onnx/models/raw/b9a54e89508f101a1611cd64f4ef56b9cb62c7cf/vision/classification/shufflenet/model/shufflenet-v2-10.onnx + // (licensed at that commit under the BSD 3-Clause License) + // a subfolder ImageNet containing the file: + // - synset.txt: this is a file containing an ordered list of the 1000 ImageNet classes, available from the onnx model zoo: + // https://github.com/onnx/models/raw/8d50e3f598e6d5c67c7c7253e5a203a26e731a1b/vision/classification/synset.txt + // and a subfolder TestImages containing the file: + // - image1.jpg: this is the test image, available from the ML.NET samples repo, + // https://github.com/dotnet/machinelearning-samples/tree/ffac232a1d599903b8591b68dabd237af48f462f/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx + // in the ObjectDetectionConsoleApp/assets/images folder + + if (TestRunner.TestResourcesPath != null) + { + string modelFile = Path.Combine(TestRunner.TestResourcesPath, "ShuffleNet", "shufflenet-v2-10.onnx"); + string imageClassesFile = Path.Combine(TestRunner.TestResourcesPath, "ImageNet", "synset.txt"); + string testImageFile = Path.Combine(TestRunner.TestResourcesPath, "TestImages", "image1.jpg"); + + // create a configuration for the ShuffleNet V2 model + var modelConfig = ImageNetModelRunnerConfiguration.CreateShuffleNet2(modelFile, imageClassesFile); + var testImage = Shared.Create(Image.FromBitmap(new System.Drawing.Bitmap(testImageFile))); + + this.ImageClassificationTest( + modelConfig, + testImage, + new[] { + ("n03100240 convertible", 0.480612665f), + ("n02701002 ambulance", 0.155334875f), + ("n02974003 car wheel", 0.08949315f), + ("n03459775 grille, radiator grille", 0.061813876f), + ("n03930630 pickup, pickup truck", 0.05676603f), + }); + } + else + { + Assert.Inconclusive($"Test not run because 'PsiTestResources' environment variable not found."); + } + } + + private void ImageClassificationTest( + ImageNetModelRunnerConfiguration testConfig, + Shared testImage, + IList<(string Label, float Confidence)> expectedPredictions) + { + List labeledResults = null; + + // create a pipeline and run the model on the test image + using (var p = Pipeline.Create()) + { + var resNet = new ImageNetModelRunner(p, testConfig); + Generators.Return(p, testImage) + .PipeTo(resNet) + .Do(results => labeledResults = results.DeepClone()); + + p.Run(); + } + + // verify the model predictions against the expected predictions + for (int i = 0; i < expectedPredictions.Count; i++) + { + Assert.AreEqual(expectedPredictions[i].Label, labeledResults[i].Label); + Assert.AreEqual(expectedPredictions[i].Confidence, labeledResults[i].Confidence, 0.000005); + } + } + } +} diff --git a/Sources/Integrations/Onnx/Test.Psi.Onnx/TinyYoloV2Tester.cs b/Sources/Integrations/Onnx/Test.Psi.Onnx/TinyYoloV2Tester.cs index 402ce9138..8be62a06e 100644 --- a/Sources/Integrations/Onnx/Test.Psi.Onnx/TinyYoloV2Tester.cs +++ b/Sources/Integrations/Onnx/Test.Psi.Onnx/TinyYoloV2Tester.cs @@ -21,12 +21,13 @@ public class TinyYoloV2Tester public void TinyYoloV2ObjectDetectionTest() { // This test expects a set of resources at a path specified via the PsiTestResources environment - // variable. Specifically, it expects a subfolder TinyYoloV2 containg two files: + // variable. Specifically, it expects a subfolder TinyYoloV2 containg the file: // - TinyYolo2_model.onnx: is the model, available from the onnx model zoo: // https://github.com/onnx/models/raw/3d4b2c28f951064ab35c89d5f5c3ffe74a149e4b/vision/object_detection_segmentation/tiny-yolov2/model/tinyyolov2-8.onnx, // or in the ML.NET samples repo, // https://github.com/dotnet/machinelearning-samples/tree/ffac232a1d599903b8591b68dabd237af48f462f/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx // in the ObjectDetectionConsoleApp/assets/Model folder + // and a subfolder TestImages containing the file: // - image1.jpg: is the test image, available from the ML.NET samples repo, // https://github.com/dotnet/machinelearning-samples/tree/ffac232a1d599903b8591b68dabd237af48f462f/samples/csharp/getting-started/DeepLearning_ObjectDetection_Onnx // in the ObjectDetectionConsoleApp/assets/images folder @@ -34,7 +35,7 @@ public void TinyYoloV2ObjectDetectionTest() if (TestRunner.TestResourcesPath != null) { var yoloModel = Path.Combine(TestRunner.TestResourcesPath, "TinyYoloV2", "TinyYolo2_model.onnx"); - var testImage = Path.Combine(TestRunner.TestResourcesPath, "TinyYoloV2", "image1.jpg"); + var testImage = Path.Combine(TestRunner.TestResourcesPath, "TestImages", "image1.jpg"); var labels = new string[] { "car", "car", "person", "car", "car" }; var confidences = new float[] { 0.972779453f, 0.7202784f, 0.512937546f, 0.476578265f, 0.452380836f }; @@ -65,10 +66,13 @@ public void TinyYoloV2ObjectDetectionTest() p.Run(); - CollectionAssert.AreEqual(labels, labelsResults.ToArray()); - CollectionAssert.AreEqual(confidences, confidencesResults.ToArray()); - CollectionAssert.AreEqual(boxX, boxXResults); - CollectionAssert.AreEqual(boxY, boxYResults); + for (int i = 0; i < confidences.Length; i++) + { + Assert.AreEqual(labels[i], labelsResults[i]); + Assert.AreEqual(confidences[i], confidencesResults[i], 0.000001); + Assert.AreEqual(boxX[i], boxXResults[i], 0.001); + Assert.AreEqual(boxY[i], boxYResults[i], 0.001); + } } else { diff --git a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectCore.cs b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectCore.cs index c677d35dc..76c763f72 100644 --- a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectCore.cs +++ b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectCore.cs @@ -44,7 +44,7 @@ internal sealed class AzureKinectCore : ISourceComponent, IDisposable /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. /// Configuration to use for the device. public AzureKinectCore(Pipeline pipeline, AzureKinectSensorConfiguration config = null) { diff --git a/Sources/Media/Microsoft.Psi.Media.Linux/MediaCapture.cs b/Sources/Media/Microsoft.Psi.Media.Linux/MediaCapture.cs index 52a9e4692..1531735b1 100644 --- a/Sources/Media/Microsoft.Psi.Media.Linux/MediaCapture.cs +++ b/Sources/Media/Microsoft.Psi.Media.Linux/MediaCapture.cs @@ -26,7 +26,7 @@ public class MediaCapture : IProducer>, ISourceComponent, IDisposa /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. /// Name of file containing media capture device configuration. public MediaCapture(Pipeline pipeline, string configurationFilename) : this(pipeline) @@ -38,7 +38,7 @@ public MediaCapture(Pipeline pipeline, string configurationFilename) /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. /// Describes how to configure the media capture device. public MediaCapture(Pipeline pipeline, MediaCaptureConfiguration configuration) : this(pipeline) @@ -49,7 +49,7 @@ public MediaCapture(Pipeline pipeline, MediaCaptureConfiguration configuration) /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. /// Width of output image in pixels. /// Height of output image in pixels. /// Device ID. diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaCapture.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaCapture.cs index 86e1e3fe1..b14574d75 100644 --- a/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaCapture.cs +++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaCapture.cs @@ -38,7 +38,7 @@ public class MediaCapture : IProducer>, ISourceComponent, IDisposa /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. /// Name of file containing media capture device configuration. public MediaCapture(Pipeline pipeline, string configurationFilename) : this(pipeline) @@ -54,7 +54,7 @@ public MediaCapture(Pipeline pipeline, string configurationFilename) /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. /// Describes how to configure the media capture device. public MediaCapture(Pipeline pipeline, MediaCaptureConfiguration configuration = null) : this(pipeline) @@ -69,7 +69,7 @@ public MediaCapture(Pipeline pipeline, MediaCaptureConfiguration configuration = /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. /// Width of output image in pixels. /// Height of output image in pixels. /// Frame rate. diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaSource.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaSource.cs index 3d7190280..28afefa15 100644 --- a/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaSource.cs +++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaSource.cs @@ -45,7 +45,7 @@ public class MediaSource : Generator, IDisposable /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. /// Name of media file to play. /// Optional flag specifying whether to drop out of order packets (defaults to false). public MediaSource(Pipeline pipeline, string filename, bool dropOutOfOrderPackets = false) @@ -56,7 +56,7 @@ public MediaSource(Pipeline pipeline, string filename, bool dropOutOfOrderPacket /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. /// Source stream of the media to consume. /// Optional date/time that the media started. /// Optional flag specifying whether to drop out of order packets (defaults to false). diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/Microsoft.Psi.Media.Windows.x64.csproj b/Sources/Media/Microsoft.Psi.Media.Windows.x64/Microsoft.Psi.Media.Windows.x64.csproj index 513984be5..98d5b0540 100644 --- a/Sources/Media/Microsoft.Psi.Media.Windows.x64/Microsoft.Psi.Media.Windows.x64.csproj +++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/Microsoft.Psi.Media.Windows.x64.csproj @@ -41,6 +41,7 @@ + @@ -53,4 +54,9 @@ + + + + + \ No newline at end of file diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/Mpeg4Writer.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/Mpeg4Writer.cs index 8d2263683..30c37e642 100644 --- a/Sources/Media/Microsoft.Psi.Media.Windows.x64/Mpeg4Writer.cs +++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/Mpeg4Writer.cs @@ -22,7 +22,7 @@ public class Mpeg4Writer : IConsumer>, IDisposable /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. /// Name of output file to write to. /// Name of file containing media capture device configuration. public Mpeg4Writer(Pipeline pipeline, string filename, string configurationFilename) @@ -35,7 +35,7 @@ public Mpeg4Writer(Pipeline pipeline, string filename, string configurationFilen /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. /// Name of output file to write to. /// Describes how to configure the media capture device. public Mpeg4Writer(Pipeline pipeline, string filename, Mpeg4WriterConfiguration configuration) @@ -47,7 +47,7 @@ public Mpeg4Writer(Pipeline pipeline, string filename, Mpeg4WriterConfiguration /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. /// Name of output file to write to. /// Width of output image in pixels. /// Height of output image in pixels. diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCapture.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCapture.cs new file mode 100644 index 000000000..7bdabcd9e --- /dev/null +++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCapture.cs @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Media +{ + using System; + using System.Windows; + using System.Windows.Media; + using System.Windows.Media.Imaging; + using Microsoft.Psi.Components; + using Microsoft.Psi.Imaging; + + /// + /// Component that streams video from a Windows Media Visual. + /// + public class VisualCapture : Generator, IProducer> + { + private readonly Visual visual; + private readonly int pixelWidth; + private readonly int pixelHeight; + private readonly TimeSpan interval; + private readonly RenderTargetBitmap renderTargetBitmap; + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Interval at which to render and emit frames of the rendered visual. + /// Windows Media Visual to stream. + /// Pixel width at which to render the visual. + /// Pixel height at which to render the visual. + public VisualCapture(Pipeline pipeline, TimeSpan interval, Visual visual, int pixelWidth, int pixelHeight) + : base(pipeline, true) + { + this.interval = interval; + this.visual = visual; + this.pixelWidth = pixelWidth; + this.pixelHeight = pixelHeight; + this.renderTargetBitmap = new RenderTargetBitmap(pixelWidth, pixelHeight, 96, 96, PixelFormats.Pbgra32); + this.Out = pipeline.CreateEmitter>(this, nameof(this.Out)); + } + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Configuration values for the visual capture component. + public VisualCapture(Pipeline pipeline, VisualCaptureConfiguration configuration) + : this(pipeline, configuration.Interval, configuration.Visual, configuration.PixelWidth, configuration.PixelHeight) + { + } + + /// + /// Gets the emitter that generates images from the visual. + /// + public Emitter> Out { get; private set; } + + /// + protected override DateTime GenerateNext(DateTime currentTime) + { + using (var sharedImage = ImagePool.GetOrCreate(this.pixelWidth, this.pixelHeight, Imaging.PixelFormat.BGRA_32bpp)) + { + var resource = sharedImage.Resource; + this.visual.Dispatcher.Invoke(() => + { + this.renderTargetBitmap.Render(this.visual); + this.renderTargetBitmap.CopyPixels(Int32Rect.Empty, resource.ImageData, resource.Size, resource.Stride); + }); + + this.Out.Post(sharedImage, currentTime); + } + + return currentTime + this.interval; + } + } +} diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCaptureConfiguration.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCaptureConfiguration.cs new file mode 100644 index 000000000..7e2258998 --- /dev/null +++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCaptureConfiguration.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Media +{ + using System; + using System.Windows.Media; + + /// + /// Encapsulates configuration for VisualCapture component. + /// + public class VisualCaptureConfiguration + { + /// + /// Default configuration. + /// + public static readonly VisualCaptureConfiguration Default = new VisualCaptureConfiguration() + { + }; + + /// + /// Gets or sets the interval at which to render and emit frames of the rendered visual.. + /// + public TimeSpan Interval { get; set; } = TimeSpan.FromMilliseconds(100); + + /// + /// Gets or sets the Windows Media Visual to stream. + /// + public Visual Visual { get; set; } + + /// + /// Gets or sets the pixel width at which to render the visual. + /// + public int PixelWidth { get; set; } + + /// + /// Gets or sets the pixel height at which to render the visual. + /// + public int PixelHeight { get; set; } + } +} diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCapture.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCapture.cs new file mode 100644 index 000000000..9ad099bd5 --- /dev/null +++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCapture.cs @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#pragma warning disable CA1060 // Move pinvokes to native methods class +#pragma warning disable SA1305 // Field names should not use Hungarian notation + +namespace Microsoft.Psi.Media +{ + using System; + using System.Drawing; + using System.Runtime.InteropServices; + using Microsoft.Psi.Components; + using Microsoft.Psi.Imaging; + using PsiImage = Microsoft.Psi.Imaging.Image; + + /// + /// Component that streams video from Window handle (default=desktop window/primary screen). + /// + public class WindowCapture : Generator, IProducer> + { + // see https://docs.microsoft.com/en-us/windows/win32/api/wingdi/nf-wingdi-bitblt + private const int CaptureBlt = 0x40000000; // include layered windows + private const int SourceCopy = 0x00CC0020; // copy source rectangle directly to the destination + + private readonly TimeSpan interval; + private readonly IntPtr hWnd; + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Interval at which to render and emit frames of the window. + /// Window handle to capture (default=desktop window/primary screen). + public WindowCapture(Pipeline pipeline, TimeSpan interval, IntPtr hWnd) + : base(pipeline, true) + { + this.interval = interval; + this.hWnd = hWnd == IntPtr.Zero ? GetDesktopWindow() : hWnd; + this.Out = pipeline.CreateEmitter>(this, nameof(this.Out)); + } + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Interval at which to render and emit frames of the window. + public WindowCapture(Pipeline pipeline, TimeSpan interval) + : this(pipeline, interval, IntPtr.Zero) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Configuration values for the window capture component. + public WindowCapture(Pipeline pipeline, WindowCaptureConfiguration configuration) + : this(pipeline, configuration.Interval, configuration.WindowHandle) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + public WindowCapture(Pipeline pipeline) + : this(pipeline, WindowCaptureConfiguration.Default) + { + } + + /// + /// Gets the emitter that generates images from window. + /// + public Emitter> Out { get; private set; } + + /// + protected override DateTime GenerateNext(DateTime currentTime) + { + GetWindowRect(this.hWnd, out RECT rect); + var width = rect.Right - rect.Left; + var height = rect.Bottom - rect.Top; + var win = GetWindowDC(this.hWnd); + var dest = CreateCompatibleDC(win); + var hBmp = CreateCompatibleBitmap(win, width, height); + var sel = SelectObject(dest, hBmp); + BitBlt(dest, 0, 0, width, height, win, 0, 0, SourceCopy | CaptureBlt); + var bitmap = Bitmap.FromHbitmap(hBmp); + + using (var sharedImage = ImagePool.GetOrCreate(width, height, PixelFormat.BGRA_32bpp)) + { + var resource = sharedImage.Resource; + resource.CopyFrom(bitmap); + this.Out.Post(sharedImage, currentTime); + } + + bitmap.Dispose(); + SelectObject(dest, sel); + DeleteObject(hBmp); + DeleteDC(dest); + ReleaseDC(this.hWnd, win); + + return currentTime + this.interval; + } + + [DllImport("user32.dll")] + private static extern IntPtr GetDesktopWindow(); + + [DllImport("user32.dll")] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect); + + [DllImport("user32.dll")] + private static extern IntPtr GetWindowDC(IntPtr ptr); + + [DllImport("gdi32.dll")] + private static extern IntPtr CreateCompatibleDC(IntPtr hdc); + + [DllImport("gdi32.dll")] + private static extern IntPtr CreateCompatibleBitmap(IntPtr hdc, int nWidth, int nHeight); + + [DllImport("gdi32.dll")] + private static extern IntPtr SelectObject(IntPtr hdc, IntPtr bmp); + + [DllImport("gdi32.dll")] + private static extern bool BitBlt(IntPtr hdcDest, int xDest, int yDest, int wDest, int hDest, IntPtr hdcSource, int xSrc, int ySrc, uint rop); + + [DllImport("gdi32.dll")] + private static extern IntPtr DeleteObject(IntPtr hDc); + + [DllImport("gdi32.dll")] + private static extern IntPtr DeleteDC(IntPtr hDc); + + [DllImport("user32.dll")] + private static extern bool ReleaseDC(IntPtr hWnd, IntPtr hDc); + + [StructLayout(LayoutKind.Sequential)] + private struct RECT + { + public int Left; + public int Top; + public int Right; + public int Bottom; + } + } +} diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCaptureConfiguration.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCaptureConfiguration.cs new file mode 100644 index 000000000..a47476ada --- /dev/null +++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCaptureConfiguration.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Media +{ + using System; + + /// + /// Encapsulates configuration for WindowCapture component. + /// + public class WindowCaptureConfiguration + { + /// + /// Default configuration. + /// + public static readonly WindowCaptureConfiguration Default = new WindowCaptureConfiguration(); + + /// + /// Gets or sets the interval at which to render and emit frames of the window.. + /// + public TimeSpan Interval { get; set; } = TimeSpan.FromMilliseconds(100); + + /// + /// Gets or sets the Window handle to capture (default=desktop window/primary screen). + /// + public IntPtr WindowHandle { get; set; } = IntPtr.Zero; + } +} diff --git a/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.cpp b/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.cpp index 2f673363d..2b0951c37 100644 --- a/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.cpp +++ b/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.cpp @@ -15,6 +15,6 @@ using namespace System::Security::Permissions; [assembly:AssemblyCopyrightAttribute("Copyright (c) Microsoft Corporation. All rights reserved.")]; [assembly:ComVisible(false)]; [assembly:CLSCompliantAttribute(true)]; -[assembly:AssemblyVersionAttribute("0.13.38.2")]; -[assembly:AssemblyFileVersionAttribute("0.13.38.2")]; -[assembly:AssemblyInformationalVersionAttribute("0.13.38.2-beta")]; +[assembly:AssemblyVersionAttribute("0.14.35.3")]; +[assembly:AssemblyFileVersionAttribute("0.14.35.3")]; +[assembly:AssemblyInformationalVersionAttribute("0.14.35.3-beta")]; diff --git a/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.rc b/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.rc index 4a44acb8ee48bd3e6287e41057112cccfff2deb0..7c483c1ac65f20b841fc5b009684c5021c3f4f60 100644 GIT binary patch delta 44 scmeyM{y}|19tXDxgARi+gDHrdY{+3ec@2jUP)rXj1|&D1 /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of + /// The pipeline to add the component to. /// Name of media file to play /// Output format for images public FFMPEGMediaSource(Pipeline pipeline, string filename, PixelFormat format = PixelFormat.BGRX_32bpp) diff --git a/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/Properties/AssemblyInfo.cs b/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/Properties/AssemblyInfo.cs index 7278a41bd..0506a84df 100644 --- a/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/Properties/AssemblyInfo.cs +++ b/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/Properties/AssemblyInfo.cs @@ -35,6 +35,6 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("0.13.38.2")] -[assembly: AssemblyFileVersion("0.13.38.2")] -[assembly: AssemblyInformationalVersion("0.13.38.2-beta")] +[assembly: AssemblyVersion("0.14.35.3")] +[assembly: AssemblyFileVersion("0.14.35.3")] +[assembly: AssemblyInformationalVersion("0.14.35.3-beta")] diff --git a/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/RealSenseSensor.cs b/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/RealSenseSensor.cs index be2f4d903..67ddcf26a 100644 --- a/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/RealSenseSensor.cs +++ b/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/RealSenseSensor.cs @@ -22,7 +22,7 @@ public class RealSenseSensor : ISourceComponent, IDisposable /// /// Initializes a new instance of the class. /// - /// Pipeline this component is a part of. + /// The pipeline to add the component to. public RealSenseSensor(Pipeline pipeline) { this.shutdown = false; diff --git a/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/AssemblyInfo.cpp b/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/AssemblyInfo.cpp index 6a39189a6..75a163494 100644 --- a/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/AssemblyInfo.cpp +++ b/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/AssemblyInfo.cpp @@ -31,9 +31,9 @@ using namespace System::Security::Permissions; // You can specify all the value or you can default the Revision and Build Numbers // by using the '*' as shown below: -[assembly:AssemblyVersionAttribute("0.13.38.2")]; -[assembly:AssemblyFileVersionAttribute("0.13.38.2")]; -[assembly:AssemblyInformationalVersionAttribute("0.13.38.2-beta")]; +[assembly:AssemblyVersionAttribute("0.14.35.3")]; +[assembly:AssemblyFileVersionAttribute("0.14.35.3")]; +[assembly:AssemblyInformationalVersionAttribute("0.14.35.3-beta")]; [assembly:ComVisible(false)]; diff --git a/Sources/Runtime/Microsoft.Psi/Components/Timer.cs b/Sources/Runtime/Microsoft.Psi/Components/Timer.cs index 7e687a214..088180b29 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/Timer.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/Timer.cs @@ -53,7 +53,7 @@ public abstract class Timer : ISourceComponent, IDisposable /// Initializes a new instance of the class. /// The timer fires off messages at the rate specified by timerInterval. /// - /// The pipeline this component will be part of. + /// The pipeline to add the component to. /// The timer firing interval, in ms. public Timer(Pipeline pipeline, uint timerInterval) { diff --git a/Sources/Runtime/Microsoft.Psi/Data/IStreamReader.cs b/Sources/Runtime/Microsoft.Psi/Data/IStreamReader.cs index 711f009a8..0ae2fb32a 100644 --- a/Sources/Runtime/Microsoft.Psi/Data/IStreamReader.cs +++ b/Sources/Runtime/Microsoft.Psi/Data/IStreamReader.cs @@ -5,6 +5,7 @@ namespace Microsoft.Psi.Data { using System; using System.Collections.Generic; + using System.Runtime.Serialization; using System.Threading; /// @@ -55,8 +56,9 @@ public interface IStreamReader : IDisposable /// The name of the stream to open. /// The function to call for every message in this stream. /// An optional allocator of messages. + /// The function to call if an error occurs when reading the stream. /// The metadata describing the opened stream. - IStreamMetadata OpenStream(string name, Action target, Func allocator = null); + IStreamMetadata OpenStream(string name, Action target, Func allocator = null, Action errorHandler = null); /// /// Opens the specified stream for reading, in index form; providing only index entries to the target delegate. diff --git a/Sources/Runtime/Microsoft.Psi/Data/PsiStore.cs b/Sources/Runtime/Microsoft.Psi/Data/PsiStore.cs index 0311bbd09..6704db49c 100644 --- a/Sources/Runtime/Microsoft.Psi/Data/PsiStore.cs +++ b/Sources/Runtime/Microsoft.Psi/Data/PsiStore.cs @@ -379,6 +379,50 @@ public static void Concatenate(IEnumerable<(string Name, string Path)> storeFile loggingCallback?.Invoke("Done."); } + /// + /// Processes a \psi store, generating a new store. + /// + /// Predicate function determining whether to process a stream or else should be merely copied. + /// Processor action given stream metadata, importer and exporter. + /// The name and path of the store to process. + /// The name and path of the processed store. + /// Indicates whether to create a numbered subdirectory for each store generated by multiple calls to this method. + /// An optional progress reporter for progress updates. + /// An optional callback to which human-friendly information will be logged. + public static void Process(Func predicate, Action processor, (string Name, string Path) input, (string Name, string Path) output, bool createSubdirectory = true, IProgress progress = null, Action loggingCallback = null) + { + using var pipeline = Pipeline.Create(); + PsiImporter inputStore = PsiStore.Open(pipeline, input.Name, input.Path); + Exporter outputStore = PsiStore.Create(pipeline, output.Name, output.Path, createSubdirectory, inputStore.Serializers); + + // copy all streams not being processed from inputStore to outputStore + var maxStreamId = 0; + foreach (var streamInfo in inputStore.AvailableStreams) + { + if (!predicate(streamInfo)) + { + maxStreamId = Math.Max(maxStreamId, streamInfo.Id); + inputStore.CopyStream(streamInfo.Name, outputStore); + } + } + + // process stream + Pipeline.SetLastStreamId(maxStreamId); // using IDs beyond those copied above + foreach (var streamInfo in inputStore.AvailableStreams) + { + if (predicate(streamInfo)) + { + processor(streamInfo, inputStore, outputStore); + } + } + + // run the pipeline to process the store + loggingCallback?.Invoke("Processing store ..."); + pipeline.RunAsync(null, progress); + pipeline.WaitAll(); + loggingCallback?.Invoke("Done."); + } + /// /// Returns the metadata associated with the specified stream, if the stream is persisted to a \psi store. /// diff --git a/Sources/Runtime/Microsoft.Psi/Data/PsiStoreStreamReader.cs b/Sources/Runtime/Microsoft.Psi/Data/PsiStoreStreamReader.cs index 30d55c388..d5a9229ac 100644 --- a/Sources/Runtime/Microsoft.Psi/Data/PsiStoreStreamReader.cs +++ b/Sources/Runtime/Microsoft.Psi/Data/PsiStoreStreamReader.cs @@ -5,6 +5,7 @@ namespace Microsoft.Psi.Data { using System; using System.Collections.Generic; + using System.Runtime.Serialization; using System.Threading; using Microsoft.Psi; using Microsoft.Psi.Common; @@ -17,6 +18,7 @@ namespace Microsoft.Psi.Data public sealed class PsiStoreStreamReader : IStreamReader { private readonly Dictionary> targets = new Dictionary>(); + private readonly Dictionary>> errorHandlers = new Dictionary>>(); private readonly Dictionary> outputs = new Dictionary>(); private readonly Dictionary> indexOutputs = new Dictionary>(); @@ -106,10 +108,10 @@ public bool IsLive() } /// - public IStreamMetadata OpenStream(string name, Action target, Func allocator = null) + public IStreamMetadata OpenStream(string name, Action target, Func allocator = null, Action errorHandler = null) { var meta = this.PsiStoreReader.OpenStream(name); // this checks for duplicates - this.OpenStream(meta, target, allocator); + this.OpenStream(meta, target, allocator, errorHandler); return meta; } @@ -183,11 +185,37 @@ public void ReadAll(ReplayDescriptor descriptor, CancellationToken cancelationTo var indexEntry = this.PsiStoreReader.ReadIndex(); this.indexOutputs[e.SourceId](indexEntry, e); } - else + else if (this.outputs.ContainsKey(e.SourceId)) { int count = this.PsiStoreReader.Read(ref this.buffer); var bufferReader = new BufferReader(this.buffer, count); - this.outputs[e.SourceId](bufferReader, e); + + // Deserialize the data and call the listeners. Note that due to polymorphic types, we + // may be attempting to create some handlers on the fly which may result in a serialization + // exception being thrown due to type mismatch errors. + try + { + this.outputs[e.SourceId](bufferReader, e); + } + catch (SerializationException ex) + { + // If any error occurred while processing the message and there are + // registered error handlers, stop attempting to process messages + // from the stream and notify all registered error handler listeners. + // otherwise, rethrow the exception to exit the application. + if (this.errorHandlers.ContainsKey(e.SourceId)) + { + this.outputs.Remove(e.SourceId); + foreach (Action errorAction in this.errorHandlers[e.SourceId]) + { + errorAction.Invoke(ex); + } + } + else + { + throw; + } + } } } } @@ -229,55 +257,89 @@ private void LoadMetadata(IEnumerable metadata, RuntimeInfo runtimeVer this.context.Serializers.RegisterMetadata(metadata); } - private void OpenStream(IStreamMetadata meta, Action target, Func allocator = null) + private void OpenStream(IStreamMetadata meta, Action target, Func allocator = null, Action errorHandler = null) { - // Get the deserialization handler for this stream type - var handler = this.context.Serializers.GetHandler(); + // If there's no list of targets for this stream, create it now + if (!this.targets.ContainsKey(meta.Id)) + { + this.targets[meta.Id] = new List(); + } - var isDynamic = typeof(T).FullName == typeof(object).FullName; - var isRaw = typeof(T).FullName == typeof(Message).FullName; + // Add the target to the list to call when this stream has new data + this.targets[meta.Id].Add(target); - if (!isDynamic && !isRaw) + // Add the error handler, if any + if (errorHandler != null) { - // check that the requested type matches the stream type - var streamType = meta.TypeName; - var handlerType = handler.Name; - if (streamType != handlerType) + if (!this.errorHandlers.ContainsKey(meta.Id)) { - // check if the handler is able to handle the stream type - if (handlerType != streamType) + this.errorHandlers[meta.Id] = new List>(); + } + + this.errorHandlers[meta.Id].Add(errorHandler); + } + + // Get the deserialization handler for this stream type + SerializationHandler handler = null; + try + { + // A serialization exception may be thrown here if the handler is unable to be initialized due to a + // mismatch between the format of the messages in the stream and the current format of T, probably + // because a field has been added, removed, or renamed in T since the stream was created. + handler = this.context.Serializers.GetHandler(); + + var isDynamic = typeof(T).FullName == typeof(object).FullName; + var isRaw = typeof(T).FullName == typeof(Message).FullName; + + if (!isDynamic && !isRaw) + { + // check that the requested type matches the stream type + var streamType = meta.TypeName; + var handlerType = handler.Name; + if (streamType != handlerType) { - if (this.context.Serializers.Schemas.TryGetValue(streamType, out var streamTypeSchema) && - this.context.Serializers.Schemas.TryGetValue(handlerType, out var handlerTypeSchema)) + // check if the handler is able to handle the stream type + if (handlerType != streamType) { - // validate compatibility - will throw if types are incompatible - handlerTypeSchema.ValidateCompatibleWith(streamTypeSchema); + if (this.context.Serializers.Schemas.TryGetValue(streamType, out var streamTypeSchema) && + this.context.Serializers.Schemas.TryGetValue(handlerType, out var handlerTypeSchema)) + { + // validate compatibility - will throw if types are incompatible + handlerTypeSchema.ValidateCompatibleWith(streamTypeSchema); + } } } } - } - // If there's no list of targets for this stream, create it now - if (!this.targets.ContainsKey(meta.Id)) - { - this.targets[meta.Id] = new List(); - } - - // Add the target to the list to call when this stream has new data - this.targets[meta.Id].Add(target); + // Update the code to execute when this stream receives new data + this.outputs[meta.Id] = (br, e) => + { + // Deserialize the data + var data = this.Deserialize(handler, br, e, isDynamic, isRaw, (allocator == null) ? default(T) : allocator(), meta.TypeName, this.context.Serializers.Schemas); - // Update the code to execute when this stream receives new data - this.outputs[meta.Id] = (br, e) => + // Call each of the targets + foreach (Delegate action in this.targets[meta.Id]) + { + (action as Action)(data, e); + } + }; + } + catch (SerializationException ex) { - // Deserialize the data - var data = this.Deserialize(handler, br, e, isDynamic, isRaw, (allocator == null) ? default(T) : allocator(), meta.TypeName, this.context.Serializers.Schemas); - - // Call each of the targets - foreach (Delegate action in this.targets[meta.Id]) + // If there are any registered error handlers, call the ones registered for + // this stream, otherwise rethrow the exception to exit the application. + if (this.errorHandlers.ContainsKey(meta.Id)) + { + foreach (Action errorAction in this.errorHandlers[meta.Id]) + { + errorAction.Invoke(ex); + } + } + else { - (action as Action)(data, e); + throw; } - }; + } } private T Deserialize(SerializationHandler handler, BufferReader br, Envelope env, bool isDynamic, bool isRaw, T objectToReuse, string typeName, IDictionary schemas) diff --git a/Sources/Runtime/Microsoft.Psi/Data/Store.cs b/Sources/Runtime/Microsoft.Psi/Data/Store.cs deleted file mode 100644 index e86a73daf..000000000 --- a/Sources/Runtime/Microsoft.Psi/Data/Store.cs +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace Microsoft.Psi -{ - using System; - using System.Collections.Generic; - using Microsoft.Psi.Data; - using Microsoft.Psi.Serialization; - - /// - /// OBSOLETE. Provides static methods to access multi-stream stores. - /// - public static class Store - { - /// - /// OBSOLETE. Creates a new multi-stream store and returns an instance - /// which can be used to write streams to this store. - /// - /// The that owns the . - /// The name of the store to create. - /// The path to use. If null, an in-memory store is created. - /// Indicates whether to create a numbered subdirectory for each execution of the pipeline. - /// An optional collection of custom serializers to use instead of the default ones. - /// An instance that can be used to write streams. - [Obsolete("Store APIs have moved to PsiStore.", true)] - public static Exporter Create(Pipeline pipeline, string name, string rootPath, bool createSubdirectory = true, KnownSerializers serializers = null) - { - throw new NotImplementedException("Store APIs have moved to PsiStore."); - } - - /// - /// OBSOLETE. Opens a multi-stream store for read and returns an instance - /// which can be used to inspect the store and open the streams. - /// The store metadata is available immediately after this call (before the pipeline is running) via the property. - /// - /// The that owns the . - /// The name of the store to open (the same as the catalog file name). - /// - /// The path to the store. - /// This can be one of: - /// - a full path to a directory containing the store - /// - a root path containing one or more versions of the store, each in its own subdirectory, - /// in which case the latest store is opened. - /// - a null string, in which case an in-memory store is opened. - /// - /// Optional stream reader (default PsiStoreStreamReader). - /// An instance that can be used to open streams and read messages. - [Obsolete("Store APIs have moved to PsiStore.", true)] - public static Importer Open(Pipeline pipeline, string name, string rootPath, IStreamReader streamReader = null) - { - throw new NotImplementedException("Store APIs have moved to PsiStore."); - } - - /// - /// OBSOLETE. Indicates whether the specified store file exists. - /// - /// The name of the store to check. - /// The path of the store to check. - /// Returns true if the store exists. - [Obsolete("Store APIs have moved to PsiStore.", true)] - public static bool Exists(string name, string path) - { - throw new NotImplementedException("Store APIs have moved to PsiStore."); - } - - /// - /// OBSOLETE. Indicates whether all streams in a store have been marked as "closed". - /// - /// The name of the store to check. - /// The path of the store to check. - /// Returns true if all streams in the store are marked as closed. - [Obsolete("Store APIs have moved to PsiStore.", true)] - public static bool IsClosed(string name, string path) - { - throw new NotImplementedException("Store APIs have moved to PsiStore."); - } - - /// - /// OBSOLETE. Repairs an invalid store in place. - /// - /// The name of the store to check. - /// The path of the store to check. - /// Indicates whether the original store should be deleted. - [Obsolete("Store APIs have moved to PsiStore.", true)] - public static void Repair(string name, string path, bool deleteOriginalStore = true) - { - throw new NotImplementedException("Store APIs have moved to PsiStore."); - } - - /// - /// OBSOLETE. Crops a store between the extents of a specified interval, generating a new store. - /// - /// The name and path of the store to crop. - /// The name and path of the cropped store. - /// Start of crop interval relative to beginning of store. - /// Length of crop interval. - /// - /// Indicates whether to create a numbered subdirectory for each cropped store - /// generated by multiple calls to this method. - /// - /// An optional progress reporter for progress updates. - /// An optional callback to which human-friendly information will be logged. - [Obsolete("Store APIs have moved to PsiStore.", true)] - public static void Crop((string Name, string Path) input, (string Name, string Path) output, TimeSpan start, RelativeTimeInterval length, bool createSubdirectory = true, IProgress progress = null, Action loggingCallback = null) - { - throw new NotImplementedException("Store APIs have moved to PsiStore."); - } - - /// - /// OBSOLETE. Crops a store between the extents of a specified originating time interval, generating a new store. - /// - /// The name and path of the store to crop. - /// The name and path of the cropped store. - /// The originating time interval to which to crop the store. - /// - /// Indicates whether to create a numbered subdirectory for each cropped store - /// generated by multiple calls to this method. - /// - /// An optional progress reporter for progress updates. - /// An optional callback to which human-friendly information will be logged. - [Obsolete("Store APIs have moved to PsiStore.", true)] - public static void Crop((string Name, string Path) input, (string Name, string Path) output, TimeInterval cropInterval, bool createSubdirectory = true, IProgress progress = null, Action loggingCallback = null) - { - throw new NotImplementedException("Store APIs have moved to PsiStore."); - } - - /// - /// OBSOLETE. Crops a store in place between the extents of a specified interval. - /// - /// The name and path of the store to crop. - /// Start of crop interval relative to beginning of store. - /// Length of crop interval. - /// Indicates whether the original store should be deleted. - /// An optional progress reporter for progress updates. - /// An optional callback to which human-friendly information will be logged. - [Obsolete("Store APIs have moved to PsiStore.", true)] - public static void CropInPlace((string Name, string Path) input, TimeSpan start, RelativeTimeInterval length, bool deleteOriginalStore = true, IProgress progress = null, Action loggingCallback = null) - { - throw new NotImplementedException("Store APIs have moved to PsiStore."); - } - - /// - /// OBSOLETE. Crops a store in place between the extents of a specified originating time interval. - /// - /// The name and path of the store to crop. - /// The originating time interval to which to crop the store. - /// Indicates whether the original store should be deleted. - /// An optional progress reporter for progress updates. - /// An optional callback to which human-friendly information will be logged. - [Obsolete("Store APIs have moved to PsiStore.", true)] - public static void CropInPlace((string Name, string Path) input, TimeInterval cropInterval, bool deleteOriginalStore = true, IProgress progress = null, Action loggingCallback = null) - { - throw new NotImplementedException("Store APIs have moved to PsiStore."); - } - - /// - /// OBSOLETE. Copies a store, or a subset of it. - /// - /// The name and path of the store to crop. - /// The name and path of the cropped store. - /// An optional function that defines an originating time interval to copy. By default, the extents of the entire store. - /// An optional predicate that specifies which streams to include. By default, include all streams. By default, all streams are copied. - /// - /// Indicates whether to create a numbered subdirectory for each cropped store - /// generated by multiple calls to this method. - /// - /// An optional progress reporter for progress updates. - /// An optional callback to which human-friendly information will be logged. - [Obsolete("Store APIs have moved to PsiStore.", true)] - public static void Copy( - (string Name, string Path) input, - (string Name, string Path) output, - Func cropIntervalFunction = null, - Predicate includeStreamPredicate = null, - bool createSubdirectory = true, - IProgress progress = null, - Action loggingCallback = null) - { - throw new NotImplementedException("Store APIs have moved to PsiStore."); - } - - /// - /// OBSOLETE. Concatenates a set of stores, generating a new store. - /// - /// Streams of the same name across stores must also have the same types as well as non-intersecting originating times. - /// Set of store files (name, path pairs) to concatenate. - /// Output store (name, path pair). - /// An optional progress reporter for progress updates. - /// An optional callback to which human-friendly information will be logged. - [Obsolete("Store APIs have moved to PsiStore.", true)] - public static void Concatenate(IEnumerable<(string Name, string Path)> storeFiles, (string Name, string Path) output, IProgress progress = null, Action loggingCallback = null) - { - throw new NotImplementedException("Store APIs have moved to PsiStore."); - } - - /// - /// OBSOLETE. Returns the metadata associated with the specified stream, if the stream is persisted to a store. - /// - /// The type of stream messages. - /// The stream to retrieve metadata about. - /// Upon return, this parameter contains the metadata associated with the stream, or null if the stream is not persisted. - /// True if the stream is persisted to a store, false otherwise. - public static bool TryGetStreamMetadata(IProducer source, out IStreamMetadata metadata) - { - throw new NotImplementedException("Store APIs have moved to PsiStore."); - } - - /// - /// OBSOLETE. Returns the metadata associated with the specified stream, if the stream is persisted to a store. - /// - /// The current pipeline. - /// The name of the stream to retrieve metadata about. - /// Upon return, this parameter contains the metadata associated with the stream, or null if the stream is not persisted. - /// True if the stream is persisted to a store, false otherwise. - public static bool TryGetStreamMetadata(Pipeline pipeline, string streamName, out IStreamMetadata metadata) - { - throw new NotImplementedException("Store APIs have moved to PsiStore."); - } - } -} \ No newline at end of file diff --git a/Sources/Runtime/Microsoft.Psi/Serialization/KnownSerializers.cs b/Sources/Runtime/Microsoft.Psi/Serialization/KnownSerializers.cs index 8119b6322..593fcd7de 100644 --- a/Sources/Runtime/Microsoft.Psi/Serialization/KnownSerializers.cs +++ b/Sources/Runtime/Microsoft.Psi/Serialization/KnownSerializers.cs @@ -492,15 +492,43 @@ private SerializationHandler AddHandler() // initialize the serializer after the handler is registered, // to make sure all handlers are registered before initialization runs and // allow the serializer initialization code to find and cache the handlers for the types it needs - schema = serializer.Initialize(this, schema); + try + { + schema = serializer.Initialize(this, schema); - // let any subscribers know that we initialized a new serializer that publishes a schema - if (schema != null) + // let any subscribers know that we initialized a new serializer that publishes a schema + if (schema != null) + { + // store the updated schema and override whatever is present already + this.schemas[schema.Name] = schema; + this.schemasById[schema.Id] = schema; + this.SchemaAdded?.Invoke(this, schema); + } + } + catch (SerializationException) { - // store the updated schema and override whatever is present already - this.schemas[schema.Name] = schema; - this.schemasById[schema.Id] = schema; - this.SchemaAdded?.Invoke(this, schema); + // Even though we're going to rethrow this exception, some callers may wish to + // attempt to recover from this error and just mark this one stream type as + // unreadable. So we should remove the handler we just registered as it's not + // yet properly initialized. + oldCount = this.handlers.Length; + newHandlers = new SerializationHandler[oldCount - 1]; + Array.Copy(this.handlers, newHandlers, oldCount - 1); + this.handlers = newHandlers; + + newIndex = new Dictionary(this.index); + newIndex.Remove(handler); + this.index = newIndex; + + newHandlersByType = new Dictionary(this.handlersByType); + newHandlersByType.Remove(type); + this.handlersByType = newHandlersByType; + + newHandlersById = new Dictionary(this.handlersById); + newHandlersById.Remove(handler.Id); + this.handlersById = newHandlersById; + + throw; } } diff --git a/Sources/Tools/PsiStoreTool/Program.cs b/Sources/Tools/PsiStoreTool/Program.cs index 9dd5772bb..474581c3a 100644 --- a/Sources/Tools/PsiStoreTool/Program.cs +++ b/Sources/Tools/PsiStoreTool/Program.cs @@ -38,7 +38,7 @@ private static int Main(string[] args) Console.WriteLine($"Platform for Situated Intelligence Store Tool"); try { - return Parser.Default.ParseArguments(args) + return Parser.Default.ParseArguments(args) .MapResult( (Verbs.ListStreams opts) => Utility.ListStreams(opts.Store, opts.Path, opts.ShowSize), (Verbs.Info opts) => Utility.DisplayStreamInfo(opts.Stream, opts.Store, opts.Path), @@ -48,6 +48,7 @@ private static int Main(string[] args) (Verbs.Send opts) => Utility.SendStreamMessages(opts.Stream, opts.Store, opts.Path, opts.Topic, opts.Address, opts.Format), (Verbs.Concat opts) => Utility.ConcatenateStores(opts.Store, opts.Path, opts.Output), (Verbs.Crop opts) => Utility.CropStore(opts.Store, opts.Path, opts.Output, opts.Start, opts.Length), + (Verbs.Encode opts) => Utility.EncodeStore(opts.Store, opts.Path, opts.Output, opts.Quality), (Verbs.ListTasks opts) => Utility.ListTasks(opts.Assemblies), (Verbs.Exec opts) => Utility.ExecuteTask(opts.Stream, opts.Store, opts.Path, opts.Name, opts.Assemblies, opts.Arguments), DisplayParseErrors); diff --git a/Sources/Tools/PsiStoreTool/PsiStoreTool.csproj b/Sources/Tools/PsiStoreTool/PsiStoreTool.csproj index ccd3e11c3..0c82c92f3 100644 --- a/Sources/Tools/PsiStoreTool/PsiStoreTool.csproj +++ b/Sources/Tools/PsiStoreTool/PsiStoreTool.csproj @@ -41,6 +41,7 @@ + diff --git a/Sources/Tools/PsiStoreTool/Utility.cs b/Sources/Tools/PsiStoreTool/Utility.cs index 309cacd6c..e7d644644 100644 --- a/Sources/Tools/PsiStoreTool/Utility.cs +++ b/Sources/Tools/PsiStoreTool/Utility.cs @@ -16,6 +16,7 @@ namespace PsiStoreTool using Microsoft.Psi; using Microsoft.Psi.Common; using Microsoft.Psi.Data; + using Microsoft.Psi.Imaging; using Microsoft.Psi.Interop.Format; using Microsoft.Psi.Interop.Transport; @@ -284,6 +285,37 @@ internal static int CropStore(string store, string path, string output, string s return 0; } + /// + /// Encode image streams, generating a new store. + /// + /// Store name. + /// Store path. + /// Output store name. + /// Start time relative to beginning. + /// Success flag. + internal static int EncodeStore(string store, string path, string output, int quality) + { + Console.WriteLine($"Encoding store (store={store}, path={path}, output={output}, quality={quality}"); + + bool IsImageStream(IStreamMetadata streamInfo) + { + return streamInfo.TypeName.StartsWith("Microsoft.Psi.Shared`1[[Microsoft.Psi.Imaging.Image,"); + } + + void EncodeImageStreams(IStreamMetadata streamInfo, PsiImporter importer, Exporter exporter) + { + importer + .OpenStream>(streamInfo.Name) + .ToPixelFormat(PixelFormat.BGRA_32bpp) + .EncodeJpeg(quality) + .Write(streamInfo.Name, exporter, true); + } + + PsiStore.Process(IsImageStream, EncodeImageStreams, (store, path), (output, path), true, new Progress(p => Console.WriteLine($"Progress: {p * 100.0:F2}%")), Console.WriteLine); + + return 0; + } + /// /// List tasks discovered in assemblies given in app.config. /// diff --git a/Sources/Tools/PsiStoreTool/Verbs.cs b/Sources/Tools/PsiStoreTool/Verbs.cs index ebe77f413..d44fad25f 100644 --- a/Sources/Tools/PsiStoreTool/Verbs.cs +++ b/Sources/Tools/PsiStoreTool/Verbs.cs @@ -203,7 +203,7 @@ internal class Exec internal class Crop : BaseStoreCommand { /// - /// Gets or sets name of Psi data store. + /// Gets or sets name of output Psi data store. /// [Option('o', "output", Required = false, Default = "Cropped", HelpText = "Name of output Psi data store (default=Cropped).")] public string Output { get; set; } @@ -220,5 +220,24 @@ internal class Crop : BaseStoreCommand [Option('l', "length", Required = false, HelpText = "Length of interval relative to start (default=unbounded).")] public string Length { get; set; } } + + /// + /// Encode image streams verb. + /// + [Verb("encode", HelpText = "Encode image streams to JPEG.")] + internal class Encode : BaseStoreCommand + { + /// + /// Gets or sets name of output Psi data store. + /// + [Option('o', "output", Required = false, Default = "Encoded", HelpText = "Name of output Psi data store (default=Encoded).")] + public string Output { get; set; } + + /// + /// Gets or sets quality of the JPEG compression. + /// + [Option('q', "quality", Default = 90, HelpText = "Quality of JPEG compression 0-100 (optional, default 90).")] + public int Quality { get; set; } + } } } \ No newline at end of file diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Controls/TimelineScroller.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Controls/TimelineScroller.cs index f26a2f544..a0aadb9f1 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Controls/TimelineScroller.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Controls/TimelineScroller.cs @@ -51,7 +51,11 @@ protected override void OnMouseMove(MouseEventArgs e) if (snappedTime.HasValue) { - this.Navigator.Cursor = snappedTime.Value; + // Only snap to the point if it is within the current viewport. + if (this.Navigator.ViewRange.AsTimeInterval.PointIsWithin(snappedTime.Value)) + { + this.Navigator.Cursor = snappedTime.Value; + } } else { diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataManager.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataManager.cs index 495c0cb2b..3d20bee44 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataManager.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataManager.cs @@ -76,6 +76,11 @@ private DataManager() /// public event EventHandler DataStoreStatusChanged; + /// + /// Event that fires when a stream is unable to be read from. + /// + public event EventHandler StreamReadError; + /// public void Dispose() { @@ -171,6 +176,16 @@ public void OnInstantViewRangeChanged(TimeInterval viewRange) } } + /// + /// Checks if a stream is known to be unreadable. + /// + /// The stream source indicating which stream to read from. + /// True if the stream is currently considered readable, otherwise false. + public bool IsStreamUnreadable(StreamSource streamSource) + { + return this.GetOrCreateDataStoreReader(streamSource.StoreName, streamSource.StorePath, streamSource.StreamReaderType).IsStreamUnreadable(streamSource.StreamName); + } + /// /// Creates a view of the messages identified by the matching start and end times and asynchronously fills it in. /// @@ -402,12 +417,15 @@ private void RunReadInstantDataTask() } catch (Exception ex) { - new MessageBoxWindow( - Application.Current.MainWindow, - "Instant Data Push Error", - $"An error occurred while attempting to push instant data to the visualization objects{Environment.NewLine}{Environment.NewLine}{ex.Message}", - "Close", - null).ShowDialog(); + Application.Current.Dispatcher.BeginInvoke((Action)(() => + { + new MessageBoxWindow( + Application.Current.MainWindow, + "Instant Data Push Error", + $"An error occurred while attempting to push instant data to the visualization objects{Environment.NewLine}{Environment.NewLine}{ex.Message}", + "Close", + null).ShowDialog(); + })); } } else @@ -493,6 +511,7 @@ private DataStoreReader GetOrCreateDataStoreReader(string storeName, string stor else { DataStoreReader dataStoreReader = new DataStoreReader(storeName, storePath, streamReaderType); + dataStoreReader.StreamReadError += this.DataStoreReader_StreamReadError; this.dataStoreReaders[key] = dataStoreReader; return dataStoreReader; } @@ -517,7 +536,12 @@ private DataStoreReader GetDataStoreReader(string storeName, string storePath) private void RemoveDataStoreReader(string storeName, string storePath) { ValueTuple key = (storeName, storePath); - this.dataStoreReaders.Remove(key); + + lock (this.dataStoreReaders) + { + this.dataStoreReaders[key].StreamReadError -= this.DataStoreReader_StreamReadError; + this.dataStoreReaders.Remove(key); + } } private List GetDataStoreReaderList() @@ -529,6 +553,12 @@ private List GetDataStoreReaderList() } } + private void DataStoreReader_StreamReadError(object sender, StreamReadErrorEventArgs e) + { + // Alert the visualization objects that the stream is unreadable. + this.StreamReadError?.Invoke(this, e); + } + private StreamSummaryManager GetStreamSummaryManager(StreamSource streamSource) { if (streamSource == null) diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataStoreReader.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataStoreReader.cs index 9177e0264..455c93cfa 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataStoreReader.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/DataStoreReader.cs @@ -4,9 +4,9 @@ namespace Microsoft.Psi.Visualization.Data { using System; - using System.Collections; using System.Collections.Generic; using System.Linq; + using System.Runtime.Serialization; using System.Threading; using System.Threading.Tasks; using Microsoft.Psi; @@ -25,6 +25,12 @@ public class DataStoreReader : IDisposable private string storeName; private string storePath; + /// + /// The list of streams that have been identified as unreadable, probably due to the format of the + /// message on disk not matching the current format of the data object they are deserialized from. + /// + private List unreadableStreams = new List(); + /// /// Initializes a new instance of the class. /// @@ -40,6 +46,11 @@ internal DataStoreReader(string storeName, string storePath, Type streamReaderTy this.streamCaches = new List(); } + /// + /// Event that fires when a stream is unable to be read from. + /// + public event EventHandler StreamReadError; + /// public void Dispose() { @@ -133,6 +144,19 @@ internal void OnInstantViewRangeChanged(TimeInterval viewRange) } } + /// + /// Checks if a stream is known to be unreadable. + /// + /// The name of the stream. + /// True if the stream is currently considered readable, otherwise false. + internal bool IsStreamUnreadable(string streamName) + { + lock (this.unreadableStreams) + { + return this.unreadableStreams.Contains(streamName); + } + } + /// /// Called to ask the reader to read the data for all instant streams. /// @@ -143,9 +167,16 @@ internal void ReadInstantData(DateTime cursorTime) { foreach (IStreamCache streamCache in this.streamCaches.ToList()) { - if (streamCache.HasInstantStreamReaders) + if (streamCache.HasInstantStreamReaders && (!this.unreadableStreams.Contains(streamCache.StreamName))) { - streamCache.ReadInstantData(reader, cursorTime); + try + { + streamCache.ReadInstantData(reader, cursorTime); + } + catch (SerializationException ex) + { + this.StreamCache_StreamReadError(this, new StreamReadErrorEventArgs() { StreamName = streamCache.StreamName, Exception = ex }); + } } } } @@ -326,11 +357,12 @@ private IStreamCache GetExistingStreamCache(StreamSource streamSource, IStreamAd private IStreamCache GetOrCreateStreamCacheByStreamSource(StreamSource streamSource, IStreamAdapter streamAdapter) { var streamName = streamSource.StreamName; - var streamCache = this.streamCaches.Find(sr => sr.StreamName == streamName && sr.StreamAdapter == streamAdapter); + StreamCache streamCache = this.streamCaches.Find(sr => sr.StreamName == streamName && sr.StreamAdapter == streamAdapter) as StreamCache; if (streamCache == null) { streamCache = new StreamCache(streamSource.StreamName, streamAdapter); + streamCache.StreamReadError += this.StreamCache_StreamReadError; this.streamCaches.Add(streamCache); } @@ -339,17 +371,36 @@ private IStreamCache GetOrCreateStreamCacheByStreamSource(StreamSource stream private IStreamCache GetOrCreateStreamCacheByName(string streamName, IStreamAdapter streamAdapter) { - var streamCache = this.streamCaches.Find(sr => sr.StreamName == streamName && sr.StreamAdapter == streamAdapter); + StreamCache streamCache = this.streamCaches.Find(sr => sr.StreamName == streamName && sr.StreamAdapter == streamAdapter) as StreamCache; if (streamCache == null) { streamCache = new StreamCache(streamName, streamAdapter); + streamCache.StreamReadError += this.StreamCache_StreamReadError; this.streamCaches.Add(streamCache); } return streamCache; } + private void StreamCache_StreamReadError(object sender, StreamReadErrorEventArgs e) + { + // Add the stream to the list of known unreadable streams if it's not already there. + lock (this.unreadableStreams) + { + if (!this.unreadableStreams.Contains(e.StreamName)) + { + this.unreadableStreams.Add(e.StreamName); + } + } + + // Add the store name and path and propagate the message to the data manager + e.StoreName = this.storeName; + e.StorePath = this.storePath; + + this.StreamReadError?.Invoke(this, e); + } + private struct ExecutionContext { public IStreamReader Reader; diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/EpsilonInstantStreamReader{T}.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/EpsilonInstantStreamReader{T}.cs index 3f33581b0..a1280902f 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/EpsilonInstantStreamReader{T}.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/EpsilonInstantStreamReader{T}.cs @@ -18,6 +18,11 @@ namespace Microsoft.Psi.Visualization.Data /// The type of messages in the stream. public class EpsilonInstantStreamReader { + /// + /// The stream name. + /// + private readonly string streamName; + /// /// Flag indicating whether type parameter T is Shared{} or not. /// @@ -31,9 +36,11 @@ public class EpsilonInstantStreamReader /// /// Initializes a new instance of the class. /// + /// The name of the stream. /// The cursor epsilon to use when searching for messages around a cursor time. - public EpsilonInstantStreamReader(RelativeTimeInterval cursorEpsilon) + public EpsilonInstantStreamReader(string streamName, RelativeTimeInterval cursorEpsilon) { + this.streamName = streamName; this.CursorEpsilon = cursorEpsilon; this.dataProviders = new List>(); } @@ -124,6 +131,17 @@ public void ReadInstantData(IStreamReader streamReader, DateTime cursorTime, Obs // Read the data data = cacheEntry.Read(streamReader); } + else + { + // cache miss, attempt to seek directly while the cache is presumably being populated + streamReader.Seek(cursorTime + this.CursorEpsilon, true); + streamReader.OpenStream(this.streamName, (m, e) => + { + cacheEntry = new StreamCacheEntry(null, e.CreationTime, e.OriginatingTime); + data = m; + }); + streamReader.MoveNext(out var envelope); + } // Notify each adapting data provider of the new data foreach (IAdaptingInstantDataProvider adaptingInstantDataProvider in this.dataProviders.ToList()) diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/StreamCache{T}.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/StreamCache{T}.cs index 7d09527f9..898226452 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/StreamCache{T}.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/StreamCache{T}.cs @@ -9,6 +9,7 @@ namespace Microsoft.Psi.Visualization.Data using System.Collections.Specialized; using System.Linq; using System.Reflection; + using System.Runtime.Serialization; using Microsoft.Psi; using Microsoft.Psi.Data; using Microsoft.Psi.Visualization.Collections; @@ -98,6 +99,11 @@ public StreamCache(string streamName, IStreamAdapter streamAdapter/*, object[] s } } + /// + /// Event that fires when a stream is unable to be read from. + /// + public event EventHandler StreamReadError; + /// /// Gets shared allocator. /// @@ -318,13 +324,14 @@ public void OpenStream(IStreamReader streamReader, bool readIndicesOnly) { if (this.StreamAdapter == null) { - streamReader.OpenStream(this.StreamName, this.OnReceiveData, this.Allocator); + streamReader.OpenStream(this.StreamName, this.OnReceiveData, this.Allocator, this.OnReadError); } else { dynamic dynStreamAdapter = this.StreamAdapter; dynamic dynAdaptedReceiver = dynStreamAdapter.AdaptReceiver(new Action(this.OnReceiveData)); - streamReader.OpenStream(this.StreamName, dynAdaptedReceiver, dynStreamAdapter.Allocator); + dynamic dynReadError = new Action(this.OnReadError); + streamReader.OpenStream(this.StreamName, dynAdaptedReceiver, dynStreamAdapter.Allocator, dynReadError); } } } @@ -559,7 +566,7 @@ private EpsilonInstantStreamReader GetInstantStreamReader(RelativeTimeInterva if (createIfNecessary) { // Create the instant stream reader - instantStreamReader = new EpsilonInstantStreamReader(cursorEpsilon); + instantStreamReader = new EpsilonInstantStreamReader(this.StreamName, cursorEpsilon); this.instantStreamReaders.Add(instantStreamReader); } else @@ -635,5 +642,11 @@ private void OnReceiveIndex(Func indexThunk, Envelope env) } } } + + private void OnReadError(SerializationException ex) + { + // Notify the data store reader + this.StreamReadError?.Invoke(this, new StreamReadErrorEventArgs() { StreamName = this.StreamName, Exception = ex }); + } } } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/StreamReadErrorEventArgs.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/StreamReadErrorEventArgs.cs new file mode 100644 index 000000000..ea72629a0 --- /dev/null +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Data/StreamReadErrorEventArgs.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Visualization.Data +{ + using System; + using System.Runtime.Serialization; + + /// + /// Represents the event arguments passed by the stream read error event of . + /// + public class StreamReadErrorEventArgs : EventArgs + { + /// + /// Gets or sets the name of the store containing the stream that was unable to be read. + /// + public string StoreName { get; set; } + + /// + /// Gets or sets the path to the store containing the stream that was unable to be read. + /// + public string StorePath { get; set; } + + /// + /// Gets or sets the name of the stream that was unable to be read. + /// + public string StreamName { get; set; } + + /// + /// Gets or sets the serialization exception that occurred. + /// + public SerializationException Exception { get; set; } + } +} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/PluginMap.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/PluginMap.cs index 4342798e1..ee6f3dab5 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/PluginMap.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/PluginMap.cs @@ -272,14 +272,15 @@ public Type GetStreamReaderType(string extension) // We force-add the latency visualizer b/c it's not detectable by data type // (the adapter to make it work will be added automatically later in - // CustomizeVisualizerMetadata. + // CustomizeVisualizerMetadata). Latency visualizer is only compatible with + // timeline visualization panels. if (isUniversal.HasValue && isUniversal.Value) { if (isInNewPanel.HasValue && isInNewPanel.Value) { results.Add(this.visualizers.FirstOrDefault(v => v.CommandText == ContextMenuName.VisualizeLatencyInNewPanel)); } - else + else if (visualizationPanel is TimelineVisualizationPanel) { results.Add(this.visualizers.FirstOrDefault(v => v.CommandText == ContextMenuName.VisualizeLatency)); } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/PartitionViewModel.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/PartitionViewModel.cs index b72343481..043bc9c0c 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/PartitionViewModel.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/PartitionViewModel.cs @@ -290,21 +290,29 @@ public bool IsPsiPartition /// A stream source if a source was found to bind to, otherwise returns null. public StreamSource GetStreamSource(StreamBinding streamBinding) { + StreamSource streamSource = null; + // Check if the partition contains the required stream IStreamMetadata streamMetadata = this.Partition.AvailableStreams.FirstOrDefault(s => s.Name == streamBinding.StreamName); if (streamMetadata != default) { - // We have found a source to bind to - return new StreamSource( + // Create the stream source + streamSource = new StreamSource( this, TypeResolutionHelper.GetVerifiedType(this.StreamReaderTypeName), streamBinding.StreamName, streamMetadata, streamBinding.StreamAdapter, streamBinding.Summarizer); + + // Check with data manager as to whether the stream is already known to be unreadable + if (DataManager.Instance.IsStreamUnreadable(streamSource)) + { + streamSource = null; + } } - return null; + return streamSource; } /// diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/TimelineVisualizationPanelView.xaml.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/TimelineVisualizationPanelView.xaml.cs index 1fdbe83d5..a49199410 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/TimelineVisualizationPanelView.xaml.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/TimelineVisualizationPanelView.xaml.cs @@ -9,8 +9,6 @@ namespace Microsoft.Psi.Visualization.Views using System.Windows.Input; using System.Windows.Media; using System.Windows.Media.Imaging; - using Microsoft.Psi.PsiStudio; - using Microsoft.Psi.PsiStudio.Common; using Microsoft.Psi.Visualization; using Microsoft.Psi.Visualization.Helpers; using Microsoft.Psi.Visualization.Navigation; @@ -52,6 +50,16 @@ public void FinishDragDrop() this.currentDragOperation = DragOperation.None; } + /// + /// Notifies of a change in mouse position while a context menu is being displayed. + /// + /// The sender of the event. + /// The mouse event args. + public void ContextMenuMouseMove(object sender, MouseEventArgs e) + { + this.lastMousePosition = e.GetPosition(this); + } + /// protected override void OnRenderSizeChanged(SizeChangedInfo sizeInfo) { @@ -74,7 +82,15 @@ private void Root_ContextMenuOpening(object sender, ContextMenuEventArgs e) } else { - this.VisualizationPanel.OnContextMenuOpening(sender); + // Create the context menu if it doesn't yet exist + FrameworkElement senderElement = sender as FrameworkElement; + if (senderElement.ContextMenu == null) + { + senderElement.ContextMenu = new ContextMenu(); + senderElement.ContextMenu.AddHandler(MouseMoveEvent, new MouseEventHandler(this.ContextMenuMouseMove), true); + } + + this.VisualizationPanel.OnContextMenuOpening(senderElement.ContextMenu); } } @@ -82,8 +98,8 @@ private void Root_MouseMove(object sender, MouseEventArgs e) { Point mousePosition = e.GetPosition(this); - // If the user has the Left Mouse button pressed, initiate a Drag & Drop reorder operation - if (e.LeftButton == MouseButtonState.Pressed) + // If the user has the Left Mouse button pressed, initiate a Drag & Drop reorder operation. + if ((e.LeftButton == MouseButtonState.Pressed) && (mousePosition != this.lastMousePosition)) { switch (this.currentDragOperation) { @@ -115,7 +131,6 @@ private void BeginDragOperation(Point mousePosition) if (VisualizationContext.Instance.VisualizationContainer.Navigator.CursorMode == CursorMode.Manual) { this.currentDragOperation = DragOperation.TimelineScroll; - this.DoDragTimeline(mousePosition); this.Cursor = Cursors.Hand; } } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/VisualizationContainerView.xaml.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/VisualizationContainerView.xaml.cs index 3a52618a2..95a6ea147 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/VisualizationContainerView.xaml.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/VisualizationContainerView.xaml.cs @@ -4,7 +4,6 @@ namespace Microsoft.Psi.Visualization.Views { using System; - using System.Collections.Generic; using System.Linq; using System.Windows; using System.Windows.Controls; diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml index 9afaa9970..4078451a7 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml @@ -33,6 +33,9 @@ + + + @@ -40,6 +43,21 @@ + + + + + + + diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml.cs index e2551a1c7..c664058b7 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/TimeIntervalAnnotationVisualizationObjectView.xaml.cs @@ -3,10 +3,17 @@ namespace Microsoft.Psi.Visualization.Views.Visuals2D { + using System; + using System.Collections; using System.Collections.Generic; using System.ComponentModel; + using System.Reflection; + using System.Windows; + using System.Windows.Controls; + using System.Windows.Input; using System.Windows.Media; using Microsoft.Psi.Data.Annotations; + using Microsoft.Psi.Visualization.Helpers; using Microsoft.Psi.Visualization.VisualizationObjects; /// @@ -47,6 +54,22 @@ internal Brush GetBrush(System.Drawing.Color systemDrawingColor) return this.brushes[color]; } + /// + protected override void OnDataContextChanged(object sender, DependencyPropertyChangedEventArgs e) + { + if (e.OldValue is TimeIntervalAnnotationVisualizationObject oldVisualizationObject) + { + oldVisualizationObject.TimeIntervalAnnotationEdit -= this.VisualizationObject_TimeIntervalAnnotationEdit; + } + + if (e.NewValue is TimeIntervalAnnotationVisualizationObject newVisualizationObject) + { + newVisualizationObject.TimeIntervalAnnotationEdit += this.VisualizationObject_TimeIntervalAnnotationEdit; + } + + base.OnDataContextChanged(sender, e); + } + /// protected override void OnVisualizationObjectPropertyChanged(object sender, PropertyChangedEventArgs e) { @@ -74,6 +97,139 @@ protected override void OnTransformsChanged() this.Rerender(); } + private static Color ToMediaColor(System.Drawing.Color color) + { + return Color.FromArgb(color.A, color.R, color.G, color.B); + } + + private void VisualizationObject_TimeIntervalAnnotationEdit(object sender, TimeIntervalAnnotationEditEventArgs e) + { + // If we were already editing an unrestricted annotation, stop. + this.EditUnrestrictedAnnotationTextBox.Visibility = Visibility.Collapsed; + + // Check if there's an annotation to edit. + if (e.DisplayData != null) + { + // Check if a finite or an unrestricted annotation value is being edited + if (e.DisplayData.Definition.SchemaDefinitions[e.TrackId].Schema.IsFiniteAnnotationSchema) + { + this.EditFiniteAnnotationValue(e.DisplayData, e.TrackId); + } + else + { + this.EditUnrestrictedAnnotationValue(e.DisplayData, e.TrackId); + } + } + } + + private void EditFiniteAnnotationValue(TimeIntervalAnnotationDisplayData displayData, int trackId) + { + // Get the schema definition + AnnotationSchemaDefinition schemaDefinition = displayData.Definition.SchemaDefinitions[trackId]; + + // Get the collection of possible values + Type schemaType = schemaDefinition.Schema.GetType(); + MethodInfo valuesProperty = schemaType.GetProperty("Values").GetGetMethod(); + IEnumerable values = (IEnumerable)valuesProperty.Invoke(schemaDefinition.Schema, new object[] { }); + + // Create a new context menu + ContextMenu contextMenu = new ContextMenu(); + + // Create a menuitem for each value, with a command to update the value on the annotation. + foreach (object value in values) + { + var metadata = this.GetAnnotationValueMetadata(value, schemaDefinition.Schema); + contextMenu.Items.Add(MenuItemHelper.CreateAnnotationMenuItem( + value.ToString(), + metadata.BorderColor, + metadata.FillColor, + new PsiCommand(() => this.VisualizationObject.SetAnnotationValue(displayData.Annotation, schemaDefinition.Name, value)))); + } + + // Add a handler so that the timeline visualization panel continues to receive mouse move messages + // while the context menu is displayed, and remove the handler once the context menu closes. + MouseEventHandler mouseMoveHandler = new MouseEventHandler(this.FindTimelineVisualizationPanelView().ContextMenuMouseMove); + contextMenu.AddHandler(MouseMoveEvent, mouseMoveHandler, true); + contextMenu.Closed += (sender, e) => contextMenu.RemoveHandler(MouseMoveEvent, mouseMoveHandler); + + // Show the context menu + contextMenu.IsOpen = true; + } + + private void EditUnrestrictedAnnotationValue(TimeIntervalAnnotationDisplayData displayData, int trackId) + { + // Get the schema definition + AnnotationSchemaDefinition schemaDefinition = displayData.Definition.SchemaDefinitions[trackId]; + + // Get the current value + object value = displayData.Annotation.Data.Values[schemaDefinition.Name]; + + // Get the associated metadata + AnnotationSchemaValueMetadata schemaMetadata = this.GetAnnotationValueMetadata(value, schemaDefinition.Schema); + + // Style the text box to match the schema of the annotation value + this.EditUnrestrictedAnnotationTextBox.Foreground = new SolidColorBrush(ToMediaColor(schemaMetadata.TextColor)); + this.EditUnrestrictedAnnotationTextBox.Background = new SolidColorBrush(ToMediaColor(schemaMetadata.FillColor)); + this.EditUnrestrictedAnnotationTextBox.BorderBrush = new SolidColorBrush(ToMediaColor(schemaMetadata.BorderColor)); + this.EditUnrestrictedAnnotationTextBox.BorderThickness = new Thickness(schemaMetadata.BorderWidth); + + // The textbox's tag holds context information to allow us to update the value in the display object when + // the text in the textbox changes. Note that we must set the correct tag before we set the text, otherwise + // setting the text will cause it to be copied to the last annotation that we edited. + this.EditUnrestrictedAnnotationTextBox.Tag = new UnrestrictedAnnotationValueContext(displayData, schemaDefinition.Name); + this.EditUnrestrictedAnnotationTextBox.Text = value.ToString(); + + // Set the textbox position to exactly cover the annotation value + var navigatorViewDuration = this.Navigator.ViewRange.Duration.TotalSeconds; + var labelStart = Math.Min(navigatorViewDuration, Math.Max((displayData.StartTime - this.Navigator.ViewRange.StartTime).TotalSeconds, 0)); + var labelEnd = Math.Max(0, Math.Min((displayData.EndTime - this.Navigator.ViewRange.StartTime).TotalSeconds, navigatorViewDuration)); + + var verticalSpace = this.VisualizationObject.Padding / this.ScaleTransform.ScaleY; + var lo = (double)(trackId + verticalSpace) / this.VisualizationObject.TrackCount; + var hi = (double)(trackId + 1 - verticalSpace) / this.VisualizationObject.TrackCount; + + this.EditUnrestrictedAnnotationTextBox.Width = (labelEnd - labelStart) * this.Canvas.ActualWidth / this.Navigator.ViewRange.Duration.TotalSeconds; + this.EditUnrestrictedAnnotationTextBox.Height = (hi - lo) * this.Canvas.ActualHeight; + (this.EditUnrestrictedAnnotationTextBox.RenderTransform as TranslateTransform).X = labelStart * this.Canvas.ActualWidth / this.Navigator.ViewRange.Duration.TotalSeconds; + (this.EditUnrestrictedAnnotationTextBox.RenderTransform as TranslateTransform).Y = lo * this.Canvas.ActualHeight; + + // Initially select the entire text of the textbox, then show the textbox and set keyboard focus to it. + this.EditUnrestrictedAnnotationTextBox.SelectAll(); + this.EditUnrestrictedAnnotationTextBox.Visibility = Visibility.Visible; + this.EditUnrestrictedAnnotationTextBox.Focus(); + } + + private void EditUnrestrictedAnnotationTextBox_TextChanged(object sender, TextChangedEventArgs e) + { + UnrestrictedAnnotationValueContext context = this.EditUnrestrictedAnnotationTextBox.Tag as UnrestrictedAnnotationValueContext; + if (context != null) + { + context.DisplayData.SetValue(context.ValueName, this.EditUnrestrictedAnnotationTextBox.Text); + } + } + + private void TextBox_LostFocus(object sender, RoutedEventArgs e) + { + this.EditUnrestrictedAnnotationTextBox.Visibility = Visibility.Collapsed; + } + + private AnnotationSchemaValueMetadata GetAnnotationValueMetadata(object value, IAnnotationSchema annotationSchema) + { + MethodInfo getMetadataProperty = annotationSchema.GetType().GetMethod("GetMetadata"); + return (AnnotationSchemaValueMetadata)getMetadataProperty.Invoke(annotationSchema, new[] { value }); + } + + private TimelineVisualizationPanelView FindTimelineVisualizationPanelView() + { + DependencyObject timelinePanelView = this.VisualParent; + while (!(timelinePanelView is TimelineVisualizationPanelView)) + { + timelinePanelView = VisualTreeHelper.GetParent(timelinePanelView); + } + + return timelinePanelView as TimelineVisualizationPanelView; + } + private void Rerender() { for (int i = 0; i < this.VisualizationObject.DisplayData.Count; i++) @@ -101,5 +257,18 @@ private void Rerender() this.items.Remove(item); } } + + private class UnrestrictedAnnotationValueContext + { + public UnrestrictedAnnotationValueContext(TimeIntervalAnnotationDisplayData displayData, string valueName) + { + this.DisplayData = displayData; + this.ValueName = valueName; + } + + public TimeIntervalAnnotationDisplayData DisplayData { get; private set; } + + public string ValueName { get; private set; } + } } } \ No newline at end of file diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationEditEventArgs.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationEditEventArgs.cs new file mode 100644 index 000000000..779bb0576 --- /dev/null +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationEditEventArgs.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Visualization.VisualizationObjects +{ + using System; + + /// + /// Provides data for editing a time interval annotation value. + /// + public class TimeIntervalAnnotationEditEventArgs : EventArgs + { + /// + /// Initializes a new instance of the class. + /// + /// The annotation display object to edit. + /// The id of the track in the display data to edit. + public TimeIntervalAnnotationEditEventArgs(TimeIntervalAnnotationDisplayData displayData, int trackId) + { + this.DisplayData = displayData; + this.TrackId = trackId; + } + + /// + /// Gets the annotation to edit. + /// + public TimeIntervalAnnotationDisplayData DisplayData { get; private set; } + + /// + /// Gets the ID of the track in the annotation to edit. + /// + public int TrackId { get; private set; } + } +} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationVisualizationObject.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationVisualizationObject.cs index 504ce5df7..d170483ad 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationVisualizationObject.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationVisualizationObject.cs @@ -4,7 +4,6 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects { using System; - using System.Collections; using System.Collections.Generic; using System.Collections.Specialized; using System.ComponentModel; @@ -23,6 +22,8 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects using Microsoft.Psi.Visualization.Data; using Microsoft.Psi.Visualization.Helpers; using Microsoft.Psi.Visualization.Views.Visuals2D; + using Microsoft.Psi.Visualization.VisualizationPanels; + using Microsoft.Psi.Visualization.Windows; /// /// Class implements a . @@ -30,6 +31,11 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects [VisualizationObject("Time Interval Annotations")] public class TimeIntervalAnnotationVisualizationObject : TimelineVisualizationObject { + private const string ErrorStreamNotBound = "The visualization object is not currently bound to a stream."; + private const string ErrorEditingDisabled = "Annotation add/delete is currently disabled in the visualization object properties."; + private const string ErrorSelectionMarkersUnset = "Both the start and end selection markers must be set.\r\n\r\nYou can set the start and end selection markers with SHIFT + Left mouse button and SHIFT + Right mouse button."; + private const string ErrorOverlappingAnnotations = "Time interval annotations may not overlap."; + private double padding = 0; private double lineWidth = 2; private double fontSize = 10; @@ -38,6 +44,7 @@ public class TimeIntervalAnnotationVisualizationObject : TimelineVisualizationOb private RelayCommand nextAnnotationCommand; private RelayCommand previousAnnotationCommand; private RelayCommand mouseLeftButtonDownCommand; + private RelayCommand mouseRightButtonDownCommand; private RelayCommand mouseMoveCommand; private RelayCommand mouseLeftButtonUpCommand; private RelayCommand mouseDoubleClickCommand; @@ -46,6 +53,11 @@ public class TimeIntervalAnnotationVisualizationObject : TimelineVisualizationOb private TimeIntervalAnnotationDragInfo annotationDragInfo = null; + /// + /// Event that fires when an annotation value should be edited in the view. + /// + public event EventHandler TimeIntervalAnnotationEdit; + private enum AnnotationEdge { None, @@ -144,7 +156,7 @@ public override string IconSource } /// - /// Gets the mouse move command. + /// Gets the mouse left button down command. /// [Browsable(false)] public RelayCommand MouseLeftButtonDownCommand @@ -153,7 +165,6 @@ public RelayCommand MouseLeftButtonDownCommand { if (this.mouseLeftButtonDownCommand == null) { - // Ensure playback is stopped before exiting this.mouseLeftButtonDownCommand = new RelayCommand( (e) => { @@ -165,6 +176,27 @@ public RelayCommand MouseLeftButtonDownCommand } } + /// + /// Gets the mouse right button down command. + /// + [Browsable(false)] + public RelayCommand MouseRightButtonDownCommand + { + get + { + if (this.mouseRightButtonDownCommand == null) + { + this.mouseRightButtonDownCommand = new RelayCommand( + (e) => + { + this.DoMouseRightButtonDown(e); + }); + } + + return this.mouseRightButtonDownCommand; + } + } + /// /// Gets the mouse move command. /// @@ -175,7 +207,6 @@ public RelayCommand MouseMoveCommand { if (this.mouseMoveCommand == null) { - // Ensure playback is stopped before exiting this.mouseMoveCommand = new RelayCommand( (e) => { @@ -197,7 +228,6 @@ public RelayCommand MouseLeftButtonUpCommand { if (this.mouseLeftButtonUpCommand == null) { - // Ensure playback is stopped before exiting this.mouseLeftButtonUpCommand = new RelayCommand( (e) => { @@ -219,7 +249,6 @@ public RelayCommand MouseDoubleClickCommand { if (this.mouseDoubleClickCommand == null) { - // Ensure playback is stopped before exiting this.mouseDoubleClickCommand = new RelayCommand( (e) => { @@ -303,12 +332,15 @@ public override IEnumerable GetAdditionalContextMenuItems() { List menuItems = new List(); - // Add annotation edit menu items if we're above an annotation - this.AddAnnotationEditMenuItems(menuItems); - - // Add the add annotation and delete annotation context menu items. + // Add the add annotation context menu item menuItems.Add(MenuItemHelper.CreateMenuItem(null, "Add Annotation", this.GetAddAnnotationCommand())); - menuItems.Add(MenuItemHelper.CreateMenuItem(null, "Delete Annotation", this.GetDeleteAnnotationCommand())); + + // If the mouse is above an existing annotation, add the delete annotation context menu item. + ICommand deleteCommand = this.GetDeleteAnnotationCommand(); + if (deleteCommand != null) + { + menuItems.Add(MenuItemHelper.CreateMenuItem(null, "Delete Annotation", deleteCommand)); + } return menuItems; } @@ -354,83 +386,38 @@ protected override void OnDataCollectionChanged(NotifyCollectionChangedEventArgs base.OnDataCollectionChanged(e); } - private void AddAnnotationEditMenuItems(List menuItems) - { - // All of the following must be true to edit an annotation: - // - // 1) We must be bound to a source - // 2) Edit annotations values must be enabled. - // 3) The cursor must be over an annotation. - if (this.IsBound && this.EnableAnnotationValueEdit) - { - int index = this.GetAnnotationIndexByTime(this.Container.Navigator.Cursor); - if (index >= 0) - { - // Get the annotation to be edited - Message annotation = this.Data[index]; - - // Get the collection of schema definitions in the annotation - foreach (AnnotationSchemaDefinition schemaDefinition in this.Definition.SchemaDefinitions) - { - // Create a menuitem for the value - var valueMenuItem = MenuItemHelper.CreateMenuItem(IconSourcePath.Annotation, schemaDefinition.Name, null); - - // If this is a finite schema, then get the list of possible values - if (schemaDefinition.Schema.IsFiniteAnnotationSchema) - { - // Get the collection of possible values - Type schemaType = schemaDefinition.Schema.GetType(); - MethodInfo valuesProperty = schemaType.GetProperty("Values").GetGetMethod(); - IEnumerable values = (IEnumerable)valuesProperty.Invoke(schemaDefinition.Schema, new object[] { }); - - // Create a menuitem for each value, with a command to update the value on the annotation. - foreach (object value in values) - { - var metadata = this.GetAnnotationValueMetadata(value, schemaDefinition.Schema); - valueMenuItem.Items.Add(MenuItemHelper.CreateAnnotationMenuItem( - value.ToString(), - metadata.BorderColor, - metadata.FillColor, - new PsiCommand(() => this.SetAnnotationValue(annotation, schemaDefinition.Name, value)))); - } - } - else - { - valueMenuItem.Items.Add(MenuItemHelper.CreateMenuItem( - null, - annotation.Data.Values[schemaDefinition.Name].ToString(), - null)); - } - - menuItems.Add(valueMenuItem); - } - } - } - } - private ICommand GetAddAnnotationCommand() { // All of the following must be true to allow an annotation to be added: // // 1) We must be bound to a source // 2) Add/Delete annotations must be enabled. - // 3) The cursor must be within the selection markers. - // 4) Both of the selection markers must be visible in the current view. - // 5) There must be no annotations between the selection markers. + // 3) Both selection markers must be set. + // 4) There must be no annotations between the selection markers. DateTime cursor = this.Container.Navigator.Cursor; TimeInterval selectionInterval = this.Container.Navigator.SelectionRange.AsTimeInterval; - TimeInterval viewInterval = this.Container.Navigator.ViewRange.AsTimeInterval; - if (this.IsBound && - this.EnableAddOrDeleteAnnotation && - selectionInterval.PointIsWithin(cursor) && - selectionInterval.IsSubsetOf(viewInterval) && - !this.AnnotationIntersectsWith(selectionInterval)) + if (!this.IsBound) { - return new PsiCommand(() => this.AddAnnotation(selectionInterval)); + return this.CreateEditAnnotationErrorCommand(ErrorStreamNotBound); } - return null; + if (!this.EnableAddOrDeleteAnnotation) + { + return this.CreateEditAnnotationErrorCommand(ErrorEditingDisabled); + } + + if ((selectionInterval.Left <= DateTime.MinValue) || (selectionInterval.Right >= DateTime.MaxValue)) + { + return this.CreateEditAnnotationErrorCommand(ErrorSelectionMarkersUnset); + } + + if (this.AnnotationIntersectsWith(selectionInterval)) + { + return this.CreateEditAnnotationErrorCommand(ErrorOverlappingAnnotations); + } + + return new PsiCommand(() => this.AddAnnotation(selectionInterval)); } private ICommand GetDeleteAnnotationCommand() @@ -439,19 +426,33 @@ private ICommand GetDeleteAnnotationCommand() // // 1) We must be bound to a source // 2) Add/Delete annotations must be enabled. - // 3) The cursor must be over an annotation. - if (this.IsBound && this.EnableAddOrDeleteAnnotation) + // 3) The mouse cursor must be above an existing annotation. + TimeInterval selectionInterval = this.Container.Navigator.SelectionRange.AsTimeInterval; + + if (!this.IsBound) { - int index = this.GetAnnotationIndexByTime(this.Container.Navigator.Cursor); - if (index >= 0) - { - return new PsiCommand(() => this.DeleteAnnotation(this.Data[index])); - } + return this.CreateEditAnnotationErrorCommand(ErrorStreamNotBound); + } + + if (!this.EnableAddOrDeleteAnnotation) + { + return this.CreateEditAnnotationErrorCommand(ErrorEditingDisabled); + } + + int index = this.GetAnnotationIndexByTime(this.Container.Navigator.Cursor); + if (index >= 0) + { + return new PsiCommand(() => this.DeleteAnnotation(this.Data[index])); } return null; } + private PsiCommand CreateEditAnnotationErrorCommand(string errorMessage) + { + return new PsiCommand(() => new MessageBoxWindow(Application.Current.MainWindow, "Error Editing Annotation", errorMessage, "Close", null).ShowDialog()); + } + /// /// Adds a new annotation. /// @@ -466,6 +467,9 @@ private void AddAnnotation(TimeInterval timeInterval) // Update the stream with the new annotation DataManager.Instance.UpdateStream(this.StreamSource, new StreamUpdate[] { new StreamUpdate(StreamUpdateType.Add, message) }); + + // Display the properties of the new annotation + this.SelectDisplayObject(annotation); } /// @@ -506,8 +510,10 @@ private bool AnnotationIntersectsWith(TimeInterval timeInterval) { TimeIntervalAnnotation annotation = this.Data[index].Data; - // Check if the annotation intersects with the interval - if (timeInterval.IntersectsWith(annotation.Interval)) + // Check if the annotation is completely to the left of the interval + // NOTE: By default time intervals are inclusive of their endpoints, so abutting time intervals will + // test as intersecting. Use a non-inclusive time interval so that we can let annotations abut. + if (timeInterval.IntersectsWith(new TimeInterval(annotation.Interval.Left, false, annotation.Interval.Right, false))) { return true; } @@ -540,11 +546,8 @@ private void UpdateDisplayData() private void DoMouseDoubleClick(MouseButtonEventArgs e) { - // Get the timeline scroller - TimelineScroller timelineScroller = this.FindTimelineScroller(e.Source); - // Get the time at the mouse cursor - DateTime cursorTime = this.GetTimeAtMousePointer(e.GetPosition(e.Source as IInputElement), timelineScroller, false); + DateTime cursorTime = (this.Panel as TimelineVisualizationPanel).GetTimeAtMousePointer(e, false); // Get the item (if any) that straddles this time int index = this.GetAnnotationIndexByTime(cursorTime); @@ -568,23 +571,65 @@ private void DoMouseDoubleClick(MouseButtonEventArgs e) private void DoMouseLeftButtonDown(MouseButtonEventArgs e) { - // Get the timeline scroller - TimelineScroller timelineScroller = this.FindTimelineScroller(e.Source); + TimelineVisualizationPanel timelinePanel = this.Panel as TimelineVisualizationPanel; // Get the time at the mouse cursor - DateTime cursorTime = this.GetTimeAtMousePointer(e.GetPosition(e.Source as IInputElement), timelineScroller, false); + DateTime cursorTime = timelinePanel.GetTimeAtMousePointer(e, false); + + Message annotation = default; + AnnotationEdge annotationEdge = AnnotationEdge.None; // Get the item (if any) that straddles this time int index = this.GetAnnotationIndexByTime(cursorTime); if (index > -1) { // Get the annotation that was hit - Message annotation = this.Data[index]; - - Canvas canvas = this.FindCanvas(e.Source); + annotation = this.Data[index]; // Check if the mouse is over an edge of the annotation - AnnotationEdge annotationEdge = this.MouseOverAnnotationEdge(cursorTime, this.Data[index].Data, timelineScroller); + annotationEdge = this.MouseOverAnnotationEdge(cursorTime, this.Data[index].Data, timelinePanel.GetTimelineScroller(e.Source)); + } + + // If the shift key is down, the user is dropping the start selection marker. If there is no VO currently being snapped + // to and the mouse is over an annotation edge, then manually set the selection marker right on the edge. Otherwise + // let the event bubble up to the timeline visualization panel which will set the selection marker in the usual fashion. + if (Keyboard.IsKeyDown(Key.LeftShift) || Keyboard.IsKeyDown(Key.RightShift)) + { + if ((VisualizationContext.Instance.VisualizationContainer.SnapToVisualizationObject == null) && (annotationEdge != AnnotationEdge.None)) + { + DateTime selectionMarkerTime = annotationEdge == AnnotationEdge.Left ? annotation.Data.Interval.Left : annotation.Data.Interval.Right; + this.Navigator.SelectionRange.SetRange(selectionMarkerTime, this.Navigator.SelectionRange.EndTime >= selectionMarkerTime ? this.Navigator.SelectionRange.EndTime : DateTime.MaxValue); + e.Handled = true; + } + else + { + return; + } + } + + // If we're over an annotation + if (annotation != default) + { + if (annotationEdge == AnnotationEdge.None) + { + // We're over an annotation, but not an annotation edge, display the annotation's properties + this.SelectDisplayObject(annotation.Data); + + // Begin annotation edit if it's enabled + if (this.EnableAnnotationValueEdit) + { + // Work out the track number to be edited based on the mouse position + TimelineScroller timelineScroller = timelinePanel.GetTimelineScroller(e.Source); + Point point = e.GetPosition(timelineScroller); + int trackId = (int)(point.Y / timelineScroller.ActualHeight * (double)annotation.Data.Values.Count); + + // Find the display data object corresponding to the annotation and fire an edit event to the view + TimeIntervalAnnotationDisplayData displayObject = this.DisplayData.FirstOrDefault(d => d.Annotation.Data.Interval.Right == annotation.Data.Interval.Right); + this.TimeIntervalAnnotationEdit?.Invoke(this, new TimeIntervalAnnotationEditEventArgs(displayObject, trackId)); + } + } + + // Check if we're over an edge and annotation drag is enabled. if (annotationEdge != AnnotationEdge.None && this.EnableAnnotationDrag) { // Get the previous and next annotations (if any) and check if they abut the annotation whose edge we're going to drag @@ -642,19 +687,50 @@ private void DoMouseLeftButtonDown(MouseButtonEventArgs e) this.annotationDragInfo = new TimeIntervalAnnotationDragInfo(moveNeighborAnnotation && previousAnnotationAbuts ? previousAnnotation : null, annotation, minTime, maxTime); } } - else - { - this.SelectDisplayObject(annotation.Data); - } - - e.Handled = true; } else { + // We're not over any annotation, cancel any current edit operation in the view and display the VO's properties + this.TimeIntervalAnnotationEdit?.Invoke(this, new TimeIntervalAnnotationEditEventArgs(null, 0)); this.SelectDisplayObject(null); } } + private void DoMouseRightButtonDown(MouseButtonEventArgs e) + { + TimelineVisualizationPanel timelinePanel = this.Panel as TimelineVisualizationPanel; + + // Get the time at the mouse cursor + DateTime cursorTime = timelinePanel.GetTimeAtMousePointer(e, false); + + Message annotation = default; + AnnotationEdge annotationEdge = AnnotationEdge.None; + + // Get the item (if any) that straddles this time + int index = this.GetAnnotationIndexByTime(cursorTime); + if (index > -1) + { + // Get the annotation that was hit + annotation = this.Data[index]; + + // Check if the mouse is over an edge of the annotation + annotationEdge = this.MouseOverAnnotationEdge(cursorTime, this.Data[index].Data, timelinePanel.GetTimelineScroller(e.Source)); + } + + // If the shift key is down, the user is dropping the end selection marker. If there is no VO currently being snapped + // to and the mouse is over an annotation edge, then manually set the selection marker right on the edge. Otherwise + // let the event bubble up to the timeline visualization panel which will set the selection marker in the usual fashion. + if (Keyboard.IsKeyDown(Key.LeftShift) || Keyboard.IsKeyDown(Key.RightShift)) + { + if ((VisualizationContext.Instance.VisualizationContainer.SnapToVisualizationObject == null) && (annotationEdge != AnnotationEdge.None)) + { + DateTime selectionMarkerTime = annotationEdge == AnnotationEdge.Left ? annotation.Data.Interval.Left : annotation.Data.Interval.Right; + this.Navigator.SelectionRange.SetRange(this.Navigator.SelectionRange.StartTime <= selectionMarkerTime ? this.Navigator.SelectionRange.StartTime : DateTime.MinValue, selectionMarkerTime); + e.Handled = true; + } + } + } + private void DoMouseMove(MouseEventArgs e) { if (this.annotationDragInfo != null) @@ -664,20 +740,20 @@ private void DoMouseMove(MouseEventArgs e) } else { - // Get the timeline scroller and the canvas - TimelineScroller timelineScroller = this.FindTimelineScroller(e.Source); - Canvas canvas = this.FindCanvas(e.Source); + TimelineVisualizationPanel timelinePanel = this.Panel as TimelineVisualizationPanel; // Get the time at the mouse cursor - DateTime cursorTime = this.GetTimeAtMousePointer(e.GetPosition(e.Source as IInputElement), timelineScroller, false); + DateTime cursorTime = timelinePanel.GetTimeAtMousePointer(e, false); // Get the item (if any) that straddles this time if (this.EnableAnnotationDrag) { + Canvas canvas = this.FindCanvas(e.Source); + int index = this.GetAnnotationIndexByTime(cursorTime); if (index > -1) { - AnnotationEdge annotationEdge = this.MouseOverAnnotationEdge(cursorTime, this.Data[index].Data, timelineScroller); + AnnotationEdge annotationEdge = this.MouseOverAnnotationEdge(cursorTime, this.Data[index].Data, timelinePanel.GetTimelineScroller(e.Source)); if (annotationEdge != AnnotationEdge.None) { canvas.Cursor = Cursors.SizeWE; @@ -711,8 +787,6 @@ private void DoMouseLeftButtonUp(MouseEventArgs e) this.annotationDragInfo = null; } - - Canvas canvas = this.FindCanvas(e.Source); } private void SelectDisplayObject(TimeIntervalAnnotation annotation) @@ -813,7 +887,9 @@ private int GetTimeIntervalItemIndexByTime(DateTime time, int count, Func [DataMember] - [DisplayName("Image Transparency")] - [Description("The transparency level for the image.")] + [DisplayName("Image Transparency (%)")] + [Description("The transparency level (percentage) for the image.")] public int ImageTransparency { get { return this.imageTransparency; } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/InstantVisualizationObject{TData}.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/InstantVisualizationObject{TData}.cs index 64032f4a0..f04774811 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/InstantVisualizationObject{TData}.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/InstantVisualizationObject{TData}.cs @@ -157,6 +157,7 @@ protected override void OnStreamUnbound() { DataManager.Instance.UnregisterInstantDataTarget(this.registrationToken); this.registrationToken = Guid.Empty; + this.SetCurrentValue(null, false); } base.OnStreamUnbound(); diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/StreamVisualizationObject{TData}.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/StreamVisualizationObject{TData}.cs index 263ad1f09..f3eb96199 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/StreamVisualizationObject{TData}.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/StreamVisualizationObject{TData}.cs @@ -8,6 +8,7 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects using System.ComponentModel; using System.Linq; using System.Runtime.Serialization; + using System.Windows; using GalaSoft.MvvmLight.Command; using Microsoft.Psi; using Microsoft.Psi.PsiStudio.TypeSpec; @@ -17,6 +18,7 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects using Microsoft.Psi.Visualization.Helpers; using Microsoft.Psi.Visualization.Navigation; using Microsoft.Psi.Visualization.ViewModels; + using Microsoft.Psi.Visualization.Windows; /// /// Represents a stream visualization object. @@ -480,9 +482,21 @@ protected virtual void OnDataCollectionChanged(NotifyCollectionChangedEventArgs } } + /// + protected override void OnAddToPanel() + { + // Listen for stream read errors + DataManager.Instance.StreamReadError += this.OnStreamReadError; + + base.OnAddToPanel(); + } + /// protected override void OnRemoveFromPanel() { + // Stop listening for stream read errors + DataManager.Instance.StreamReadError -= this.OnStreamReadError; + // Unbind the visualization object from any source this.UpdateStreamSource(null); @@ -530,5 +544,25 @@ private void StreamSource_PropertyChanged(object sender, PropertyChangedEventArg this.RaisePropertyChanged(nameof(this.IconSource)); } } + + private void OnStreamReadError(object sender, StreamReadErrorEventArgs e) + { + // Check if the error is related to the stream that this visualization object currently references + if ((this.StreamSource != null) && + (this.StreamSource.StoreName == e.StoreName) && + (this.StreamSource.StorePath == e.StorePath) && + (this.StreamSource.StreamName == e.StreamName)) + { + Application.Current.Dispatcher.BeginInvoke((Action)(() => + { + // Mark this visualization object as unbound since we can't read any messages. + this.UpdateStreamSource(null); + + // Display an error message to the user. + string errorMessage = $"The format of the messages in the stream {e.StreamName} in store {e.StoreName} have changed and are unable to be deserialized, see error below:{Environment.NewLine}{Environment.NewLine}{e.Exception.Message}"; + new MessageBoxWindow(Application.Current.MainWindow, "Stream Type Mismatch", errorMessage, "Close", null).ShowDialog(); + })); + } + } } } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationContainer.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationContainer.cs index 9217152b4..a5f1a53c4 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationContainer.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationContainer.cs @@ -139,6 +139,13 @@ public void ReplaceChildVisualizationPanel(VisualizationPanel oldPanel, Visualiz // Replace the panel this.Panels[placeholderPanelIndex] = newPanel; this.UpdateChildPanelMargins(); + + // If the current visualization panel is the one we just replaced, + // then set it instead to the new panel we replaced it with. + if (this.Container?.CurrentPanel == oldPanel) + { + this.Container.CurrentPanel = newPanel; + } } /// diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationPlaceholderPanel.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationPlaceholderPanel.cs index b27c5be89..f32125d98 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationPlaceholderPanel.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationPlaceholderPanel.cs @@ -16,6 +16,14 @@ public class InstantVisualizationPlaceholderPanel : VisualizationPanel { private int relativeWidth = 100; + /// + /// Initializes a new instance of the class. + /// + public InstantVisualizationPlaceholderPanel() + { + this.Name = "Empty Instant Panel"; + } + /// /// Gets or sets the name of the relative width for the panel. /// diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/TimelineVisualizationPanel.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/TimelineVisualizationPanel.cs index 249ff81d4..b54cd3986 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/TimelineVisualizationPanel.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/TimelineVisualizationPanel.cs @@ -13,9 +13,7 @@ namespace Microsoft.Psi.Visualization.VisualizationPanels using System.Windows.Controls; using System.Windows.Input; using System.Windows.Media; - using System.Windows.Media.Imaging; using GalaSoft.MvvmLight.CommandWpf; - using Microsoft.Psi.PsiStudio; using Microsoft.Psi.Visualization; using Microsoft.Psi.Visualization.Controls; using Microsoft.Psi.Visualization.Helpers; @@ -39,7 +37,8 @@ public class TimelineVisualizationPanel : VisualizationPanel private RelayCommand showHideLegendCommand; private RelayCommand mouseLeftButtonDownCommand; private RelayCommand mouseRightButtonDownCommand; - private Point lastMouseLeftButtonDownPoint = new Point(0, 0); + + private TimelineScroller timelineScroller = null; /// /// Initializes a new instance of the class. @@ -51,12 +50,6 @@ public TimelineVisualizationPanel() this.VisualizationObjects.CollectionChanged += this.VisualizationObjects_CollectionChanged; } - /// - /// Gets the Mouse Position the last time the user clicked in this panel. - /// - [Browsable(false)] - public Point LastMouseLeftButtonDownPoint => this.lastMouseLeftButtonDownPoint; - /// /// Gets the show/hide legend command. /// @@ -149,8 +142,6 @@ public override RelayCommand MouseLeftButtonDownCommand this.mouseLeftButtonDownCommand = new RelayCommand( e => { - this.lastMouseLeftButtonDownPoint = e.GetPosition(e.Source as TimelineScroller); - // Set the current panel on click if (!this.IsCurrentPanel) { @@ -159,7 +150,7 @@ public override RelayCommand MouseLeftButtonDownCommand if (Keyboard.IsKeyDown(Key.LeftShift) || Keyboard.IsKeyDown(Key.RightShift)) { - DateTime time = this.GetTimeAtMousePointer(e); + DateTime time = this.GetTimeAtMousePointer(e, true); this.Navigator.SelectionRange.SetRange(time, this.Navigator.SelectionRange.EndTime >= time ? this.Navigator.SelectionRange.EndTime : DateTime.MaxValue); e.Handled = true; } @@ -186,7 +177,7 @@ public RelayCommand MouseRightButtonDownCommand { if (Keyboard.IsKeyDown(Key.LeftShift) || Keyboard.IsKeyDown(Key.RightShift)) { - DateTime time = this.GetTimeAtMousePointer(e); + DateTime time = this.GetTimeAtMousePointer(e, true); this.Navigator.SelectionRange.SetRange(this.Navigator.SelectionRange.StartTime <= time ? this.Navigator.SelectionRange.StartTime : DateTime.MinValue, time); e.Handled = true; } @@ -232,11 +223,11 @@ public bool ShowTimeTicks /// /// Called when the context menu is opening. /// - /// The sender. - public void OnContextMenuOpening(object sender) + /// The context menu being opened. + public void OnContextMenuOpening(ContextMenu contextMenu) { - // Create the empty context menu - ContextMenu contextMenu = new ContextMenu(); + // Clear the context menu + contextMenu.Items.Clear(); // Check with each of the visualization objects if they wish to add their own context menu items. foreach (VisualizationObject visualizationObject in this.VisualizationObjects) @@ -255,9 +246,58 @@ public void OnContextMenuOpening(object sender) // Add the context menu items for the panel this.InsertPanelContextMenuItems(contextMenu); + } + + /// + /// Gets the time at the mouse pointer, optionally adjusting for visualization object snap. + /// + /// A mouse event args object. + /// If true, and if a visualization object is currently being snapped to, then adjust the time to the nearest message in the visualization object being snapped to. + /// The time represented by the mouse pointer. + public DateTime GetTimeAtMousePointer(MouseEventArgs mouseEventArgs, bool useSnap) + { + TimelineScroller root = this.GetTimelineScroller(mouseEventArgs.Source); + if (root != null) + { + Point point = mouseEventArgs.GetPosition(root); + double percent = point.X / root.ActualWidth; + var viewRange = this.Navigator.ViewRange; + DateTime time = viewRange.StartTime + TimeSpan.FromTicks((long)((double)viewRange.Duration.Ticks * percent)); + + // If we're currently snapping to some Visualization Object, adjust the time to the timestamp of the nearest message + DateTime? snappedTime = null; + if (useSnap == true && VisualizationContext.Instance.VisualizationContainer.SnapToVisualizationObject is IStreamVisualizationObject snapToVisualizationObject) + { + snappedTime = snapToVisualizationObject.GetSnappedTime(time); + } + + return snappedTime ?? time; + } + + return DateTime.UtcNow; + } + + /// + /// Gets the timeline scroller parent of a framework element. + /// + /// The framework element to search from. + /// The timeline scroller object. + public TimelineScroller GetTimelineScroller(object sourceElement) + { + if (this.timelineScroller == null) + { + // Walk up the visual tree until we either find the + // Timeline Scroller or fall off the top of the tree + DependencyObject target = sourceElement as DependencyObject; + while (target != null && !(target is TimelineScroller)) + { + target = VisualTreeHelper.GetParent(target); + } - // set the context menu - (sender as FrameworkElement).ContextMenu = contextMenu; + this.timelineScroller = target as TimelineScroller; + } + + return this.timelineScroller; } /// @@ -313,43 +353,6 @@ private void InsertPanelContextMenuItems(ContextMenu contextMenu) contextMenu.Items.Add(MenuItemHelper.CreateMenuItem(IconSourcePath.ZoomToSession, "Zoom to Session Extents", this.ZoomToSessionExtentsCommand)); } - private DateTime GetTimeAtMousePointer(MouseEventArgs e) - { - TimelineScroller root = this.FindTimelineScroller(e.Source); - if (root != null) - { - Point point = e.GetPosition(root); - double percent = point.X / root.ActualWidth; - var viewRange = this.Navigator.ViewRange; - DateTime time = viewRange.StartTime + TimeSpan.FromTicks((long)((double)viewRange.Duration.Ticks * percent)); - - // If we're currently snapping to some Visualization Object, adjust the time to the timestamp of the nearest message - DateTime? snappedTime = null; - if (VisualizationContext.Instance.VisualizationContainer.SnapToVisualizationObject is IStreamVisualizationObject snapToVisualizationObject) - { - snappedTime = snapToVisualizationObject.GetSnappedTime(time); - } - - return snappedTime ?? time; - } - - Debug.WriteLine("TimelineVisualizationPanel.GetTimeAtMousePointer() - Could not find the TimelineScroller in the tree"); - return DateTime.UtcNow; - } - - private TimelineScroller FindTimelineScroller(object sourceElement) - { - // Walk up the visual tree until we either find the - // Timeline Scroller or fall off the top of the tree - FrameworkElement target = sourceElement as FrameworkElement; - while (target != null && !(target is TimelineScroller)) - { - target = target.Parent as FrameworkElement; - } - - return target as TimelineScroller; - } - private void VisualizationObjects_CollectionChanged(object sender, System.Collections.Specialized.NotifyCollectionChangedEventArgs e) { this.RaisePropertyChanged(nameof(this.CanZoomToPanel)); diff --git a/Sources/Visualization/Test.Psi.Visualization/Properties/AssemblyInfo.cs b/Sources/Visualization/Test.Psi.Visualization/Properties/AssemblyInfo.cs index 3b037cc48..4c875ffa3 100644 --- a/Sources/Visualization/Test.Psi.Visualization/Properties/AssemblyInfo.cs +++ b/Sources/Visualization/Test.Psi.Visualization/Properties/AssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyCopyright("Copyright (C) Microsoft Corporation. All rights reserved.")] [assembly: ComVisible(false)] [assembly: Guid("7cd463a8-61bb-4937-aa96-02da13e622d0")] -[assembly: AssemblyVersion("0.13.38.2")] -[assembly: AssemblyFileVersion("0.13.38.2")] -[assembly: AssemblyInformationalVersion("0.13.38.2-beta")] +[assembly: AssemblyVersion("0.14.35.3")] +[assembly: AssemblyFileVersion("0.14.35.3")] +[assembly: AssemblyInformationalVersion("0.14.35.3-beta")]