From ae2f8878f8a1490f628ded94c4597e9311e4d386 Mon Sep 17 00:00:00 2001 From: Dan Bohus Date: Wed, 20 Apr 2022 22:43:44 +0000 Subject: [PATCH] Merged PR 62453: Release 0.17 Release 0.17 --- .gitignore | 3 + Directory.Build.props | 2 +- Psi.sln | 47 +- .../Microsoft.Psi.Audio.Linux/AudioCapture.cs | 13 +- .../Microsoft.Psi.Audio.Linux/AudioPlayer.cs | 5 +- .../AudioCapture.cs | 19 +- .../AudioPlayer.cs | 11 +- .../AudioResampler.cs | 11 +- .../Microsoft.Psi.Audio.Windows/Operators.cs | 22 +- .../AcousticFeaturesExtractor.cs | 13 +- .../AcousticFeatures/FFT.cs | 7 +- .../AcousticFeatures/FFTPower.cs | 5 +- .../AcousticFeatures/FrameShift.cs | 15 +- .../AcousticFeatures/FrequencyDomainEnergy.cs | 5 +- .../AcousticFeatures/LogEnergy.cs | 5 +- .../AcousticFeatures/SpectralEntropy.cs | 9 +- .../AcousticFeatures/ToFloat.cs | 23 +- .../AcousticFeatures/ZeroCrossingRate.cs | 5 +- .../Audio/Microsoft.Psi.Audio/Operators.cs | 99 +- Sources/Audio/Microsoft.Psi.Audio/Reframe.cs | 10 +- .../WaveFileAudioSource.cs | 18 +- .../Microsoft.Psi.Audio/WaveFileStore.cs | 8 +- .../Microsoft.Psi.Audio/WaveFileWriter.cs | 7 +- .../WaveStreamSampleSource.cs | 5 +- .../CalibrationExtensions.cs | 18 +- .../CameraIntrinsics.cs | 37 +- .../ICameraIntrinsics.cs | 7 +- .../Microsoft.Psi.Calibration/ProjectTo3D.cs | 5 +- .../Annotations/AnnotationSchema.cs | 28 +- .../Annotations/Operators.cs | 117 +- Sources/Data/Microsoft.Psi.Data/Dataset.cs | 12 + .../Microsoft.Psi.Data/Json/JsonGenerator.cs | 16 +- .../Microsoft.Psi.Filters/MathNetFilters.cs | 3 +- .../DepthImageCompressor.cs | 12 +- .../DepthImageToPngStreamEncoder.cs | 4 +- .../ImageFromBitmapStreamDecoder.cs | 36 + .../ImageFromStreamDecoder.cs | 38 +- .../ImageToJpegStreamEncoder.cs | 3 + .../ImageToPngStreamEncoder.cs | 3 + .../ImagingOperators.cs | 51 +- .../DepthImageCompressor.cs | 11 +- .../DepthImageToPngStreamEncoder.cs | 3 + .../DepthImageToTiffStreamEncoder.cs | 3 + .../ImageFromBitmapStreamDecoder.cs | 43 + .../ImageFromStreamDecoder.cs | 105 +- .../ImageToGZipStreamEncoder.cs | 3 + .../ImageToJpegStreamEncoder.cs | 3 + .../ImageToPngStreamEncoder.cs | 3 + .../ImagingOperators.cs | 71 +- .../Microsoft.Psi.Imaging/DepthImage.cs | 906 +++++---- .../DepthImageDecoder.cs | 11 +- .../DepthImageEncoder.cs | 10 +- .../Microsoft.Psi.Imaging/DepthImagePool.cs | 17 +- .../DepthValueSemantics.cs} | 6 +- .../EncodedDepthImage.cs | 49 +- .../EncodedDepthImagePool.cs | 11 +- .../Microsoft.Psi.Imaging/EncodedImage.cs | 36 +- .../Microsoft.Psi.Imaging/IDepthImage.cs | 21 + .../IDepthImageToStreamEncoder.cs | 5 + .../Imaging/Microsoft.Psi.Imaging/IImage.cs | 26 + .../IImageToStreamEncoder.cs | 5 + .../Microsoft.Psi.Imaging/ImageBase.cs | 33 +- .../Microsoft.Psi.Imaging/ImageDecoder.cs | 5 +- .../Microsoft.Psi.Imaging/ImageEncoder.cs | 5 +- .../Microsoft.Psi.Imaging/ImageExtensions.cs | 34 +- .../ImageFromGZipStreamDecoder.cs | 55 + .../ImageFromNV12StreamDecoder.cs | 87 + .../Microsoft.Psi.Imaging/ImageTransformer.cs | 16 +- .../PixelFormatHelper.cs | 57 +- .../Microsoft.Psi.Imaging/StreamOperators.cs | 1667 +++++++++------- .../Properties/AssemblyInfo.cs | 6 +- .../FaceRecognizer.cs | 61 +- .../Operators.cs | 13 +- .../PersonalityChat.cs | 5 +- .../LUISIntentDetector.cs | 5 +- .../AzureSpeechRecognizer.cs | 45 +- .../BingSpeechRecognizer.cs | 41 +- .../ImageAnalyzer.cs | 8 +- .../Operators.cs | 200 +- .../ImageNet/ImageNetModelRunner.cs | 7 +- .../MaskRCNN/MaskRCNNDetection.cs | 48 + .../MaskRCNN/MaskRCNNDetectionResults.cs | 42 + .../MaskRCNN/MaskRCNNModelConfiguration.cs | 40 + .../MaskRCNN/MaskRCNNModelOutputParser.cs | 51 + .../MaskRCNN/MaskRCNNModelRunner.cs | 125 ++ .../MaskRCNN/MaskRCNNOnnxModel.cs | 93 + .../Common/ModelRunners/MaskRCNN/Operators.cs | 100 + .../TinyYoloV2/TinyYoloV2OnnxModelRunner.cs | 5 +- Sources/Integrations/Onnx/Common/OnnxModel.cs | 6 +- .../Onnx/Common/OnnxModelConfiguration.cs | 7 + .../Onnx/Common/OnnxModelRunner.cs | 5 +- .../Microsoft.Psi.Onnx.Cpu.csproj | 4 +- .../Microsoft.Psi.Onnx.Gpu.csproj | 4 +- .../MaskRCNNDetectionAdapter.cs | 31 + ...soft.Psi.Onnx.Visualization.Windows.csproj | 45 + .../stylecop.json | 16 + .../AzureKinectBodyTracker.cs | 6 +- .../AzureKinectBodyTrackerConfiguration.cs | 17 +- .../AzureKinectCore.cs | 14 +- .../AzureKinectSensor.cs | 12 +- .../Microsoft.Psi.AzureKinect.x64.csproj | 4 +- .../Microsoft.Psi.AzureKinect.x64.nuspec | 4 +- .../KinectSensor.cs | 6 +- .../Test.Psi.Kinect.Windows.x64/Mesh.cs | 2 +- .../Microsoft.Psi.Media.Linux/MediaCapture.cs | 120 +- .../Microsoft.Psi.Media.Native.x64.vcxproj | 2 +- .../MediaCapture.cs | 29 +- .../MediaSource.cs | 48 +- .../Mpeg4Writer.cs | 21 +- .../VisualCapture.cs | 10 +- .../WindowCapture.cs | 20 +- .../AssemblyInfo.cpp | 6 +- .../AssemblyInfo.rc | Bin 5104 -> 5104 bytes ...soft.Psi.Media_Interop.Windows.x64.vcxproj | 2 +- Sources/Media/Shared/FFMPEGMediaSource.cs | 5 +- .../HoloLensCapture/HoloLensCapture.sln | 141 ++ .../Assets/Logo/LockScreenLogo.scale-200.png | Bin 0 -> 1430 bytes .../Assets/Logo/SplashScreen.scale-200.png | Bin 0 -> 7700 bytes .../Logo/Square150x150Logo.scale-200.png | Bin 0 -> 2937 bytes .../Assets/Logo/Square44x44Logo.scale-200.png | Bin 0 -> 1647 bytes ...x44Logo.targetsize-24_altform-unplated.png | Bin 0 -> 1255 bytes .../Assets/Logo/StoreLogo.png | Bin 0 -> 1451 bytes .../Assets/Logo/Wide310x150Logo.scale-200.png | Bin 0 -> 3204 bytes .../HoloLensCaptureApp/HoloLensCaptureApp.cs | 871 +++++++++ .../HoloLensCaptureApp.csproj | 177 ++ .../HoloLensCaptureApp/Package.appxmanifest | 67 + .../Properties/AssemblyInfo.cs | 31 + .../Properties/Default.rd.xml | 31 + .../HoloLensCaptureApp/Readme.md | 28 + .../HoloLensCaptureApp/stylecop.json | 16 + .../HoloLensCaptureExporter/DataExporter.cs | 198 ++ .../HoloLensCaptureExporter.cs | 58 + .../HoloLensCaptureExporter.csproj | 44 + .../HoloLensCaptureExporter/Operators.cs | 515 +++++ .../HoloLensCaptureExporter/Readme.md | 288 +++ .../HoloLensCaptureExporter/Verbs.cs | 37 + .../HoloLensCaptureExporter/stylecop.json | 16 + .../HoloLensCaptureInterop.csproj | 46 + .../HoloLensCaptureInterop/Serializers.cs | 1672 +++++++++++++++++ .../HoloLensCaptureInterop/stylecop.json | 16 + .../HoloLensCaptureServer/App.config | 8 + .../HoloLensCaptureServer.cs | 542 ++++++ .../HoloLensCaptureServer.csproj | 48 + .../HoloLensCaptureServer/Readme.md | 35 + .../HoloLensCaptureServer/stylecop.json | 16 + .../MixedReality/HoloLensCapture/Readme.md | 9 + .../Accelerometer.cs | 6 +- .../DepthCamera.cs | 177 +- .../DepthCameraConfiguration.cs | 57 +- .../Gyroscope.cs | 5 +- .../ImageToGzipStreamEncoder.cs | 3 + .../ImageToJpegStreamEncoder.cs | 7 +- .../Magnetometer.cs | 5 +- ...t.Psi.MixedReality.UniversalWindows.csproj | 3 +- .../MixedReality.cs | 45 +- .../Operators.cs | 169 +- .../PhotoVideoCamera.cs | 137 +- .../PhotoVideoCameraConfiguration.cs | 49 +- .../Properties/AssemblyInfo.cs | 8 +- .../ResearchModeCamera.cs | 330 ++-- .../ResearchModeCameraConfiguration.cs | 64 + .../ResearchModeImu.cs | 35 +- .../SceneUnderstanding.cs | 36 +- .../SerializedCameraIntrinsics.cs | 57 + .../SpatialAnchorsSource.cs | 5 +- .../VisibleLightCamera.cs | 85 +- .../VisibleLightCameraConfiguration.cs | 59 +- .../HandVisualizationObject.cs | 4 +- .../HandVisualizationObjectAdapter.cs | 4 +- .../TrackedHandVisualizationObjectAdapter.cs | 77 + .../CalibrationPointsMap.cs | 30 +- .../Microsoft.Psi.MixedReality/EyesSensor.cs | 28 +- .../Microsoft.Psi.MixedReality/Hand.cs | 46 +- .../Microsoft.Psi.MixedReality/Handle.cs | 25 +- .../Microsoft.Psi.MixedReality/HandsSensor.cs | 67 +- .../Microsoft.Psi.MixedReality/HeadSensor.cs | 17 +- .../Microsoft.Psi.MixedReality/Microphone.cs | 5 +- .../Microsoft.Psi.MixedReality/Operators.cs | 283 ++- .../Properties/AssemblyInfo.cs | 6 + .../Microsoft.Psi.MixedReality/PsiInput.cs | 86 + .../Renderers/Box3DStereoKitRenderer.cs | 96 + ...ncodedImageRectangle3DStereoKitRenderer.cs | 55 + .../Renderers/HandsStereoKitRenderer.cs | 69 + .../Renderers/Mesh3DListStereoKitRenderer.cs | 76 - .../Renderers/Mesh3DStereoKitRenderer.cs | 57 + .../Renderers/MeshStereoKitRenderer.cs | 137 +- .../Renderers/ModelBasedStereoKitRenderer.cs | 176 -- .../Rectangle3DListStereoKitRenderer.cs | 89 - .../Renderers/Rectangle3DStereoKitRenderer.cs | 58 + .../Renderers/StereoKitRenderer.cs | 37 + .../Renderers/TextStereoKitRenderer.cs | 212 +++ .../TextStereoKitRendererConfiguration.cs | 100 + .../SpatialSound.cs | 20 +- .../StereoKitComponent.cs | 10 +- .../StereoKitTransforms.cs | 18 +- .../Microsoft.Psi.MixedReality/TimeHelper.cs | 90 + .../Properties/AssemblyInfo.cs | 6 +- .../RealSenseSensor.cs | 26 +- .../AssemblyInfo.cpp | 6 +- ....Psi.RealSense_Interop.Windows.x64.vcxproj | 2 +- .../Rendezvous/Operators.cs | 23 +- .../Rendezvous/Rendezvous.cs | 14 +- .../Rendezvous/RendezvousClient.cs | 2 +- .../Rendezvous/RendezvousClient.py | 11 +- .../Rendezvous/RendezvousRelay.cs | 5 +- .../Rendezvous/RendezvousServer.cs | 2 +- .../Transport/FileSource.cs | 21 +- .../Transport/FileWriter.cs | 8 +- .../Transport/NetMQSource.cs | 8 +- .../Transport/NetMQWriter.cs | 8 +- .../Transport/NetMQWriter{T}.cs | 8 +- .../Transport/TcpSource.cs | 31 +- .../Transport/TcpWriter.cs | 15 +- .../AdjacentValuesInterpolator.cs | 10 +- .../FirstAvailableInterpolator.cs | 7 + .../FirstReproducibleInterpolator.cs | 7 + .../LastAvailableInterpolator.cs | 7 + .../LastReproducibleInterpolator.cs | 7 + .../NearestAvailableInterpolator.cs | 15 + .../NearestReproducibleInterpolator.cs | 15 + .../Common/Intervals/RelativeTimeInterval.cs | 143 +- .../Microsoft.Psi/Common/PsiStreamMetadata.cs | 15 +- .../Microsoft.Psi/Components/Aggregator.cs | 11 +- .../Components/AsyncConsumerProducer.cs | 9 +- .../Components/ConsumerProducer.cs | 13 +- .../Microsoft.Psi/Components/DynamicWindow.cs | 288 +-- .../Microsoft.Psi/Components/EventSource.cs | 9 +- .../Runtime/Microsoft.Psi/Components/Fuse.cs | 12 +- .../Microsoft.Psi/Components/Generator.cs | 8 +- .../Microsoft.Psi/Components/Generator{T}.cs | 10 +- .../Runtime/Microsoft.Psi/Components/Join.cs | 6 +- .../Join{TPrimary,TSecondary,TOut}.cs | 6 +- .../Runtime/Microsoft.Psi/Components/Merge.cs | 8 +- .../Microsoft.Psi/Components/Merger.cs | 10 +- .../Runtime/Microsoft.Psi/Components/Pair.cs | 16 +- .../Components/ParallelSparseDo.cs | 6 +- .../Components/ParallelSparseSelect.cs | 4 +- .../Components/ParallelSparseSplitter.cs | 9 +- .../Components/ParallelVariableLength.cs | 12 +- .../Microsoft.Psi/Components/Processor.cs | 5 +- .../Components/RelativeIndexWindow.cs | 5 +- .../Components/RelativeTimeWindow.cs | 5 +- .../Components/SerializerComponent.cs | 5 +- .../Components/SimpleConsumer.cs | 9 +- .../Microsoft.Psi/Components/Splitter.cs | 10 +- .../Runtime/Microsoft.Psi/Components/Timer.cs | 8 +- .../Microsoft.Psi/Components/Timer{TOut}.cs | 7 +- .../Runtime/Microsoft.Psi/Components/Zip.cs | 8 +- .../Connectors/MessageConnector.cs | 5 +- .../Connectors/MessageEnvelopeConnector.cs | 5 +- .../Runtime/Microsoft.Psi/Data/Exporter.cs | 35 +- .../Runtime/Microsoft.Psi/Data/PsiStore.cs | 11 + .../Diagnostics/DiagnosticsCollector.cs | 2 + .../Diagnostics/DiagnosticsSampler.cs | 12 +- .../Microsoft.Psi/Executive/Pipeline.cs | 5 +- .../Microsoft.Psi/Operators/Aggregators.cs | 111 +- .../Microsoft.Psi/Operators/Connectors.cs | 2 +- .../Microsoft.Psi/Operators/Enumerable.cs | 15 +- .../Runtime/Microsoft.Psi/Operators/Fuses.cs | 198 +- .../Microsoft.Psi/Operators/Generators.cs | 63 +- .../Microsoft.Psi/Operators/Interpolators.cs | 51 +- .../Runtime/Microsoft.Psi/Operators/Joins.cs | 355 +++- .../Runtime/Microsoft.Psi/Operators/Merges.cs | 12 +- .../Microsoft.Psi/Operators/Observable.cs | 19 +- .../Runtime/Microsoft.Psi/Operators/Pairs.cs | 144 +- .../Microsoft.Psi/Operators/Pickers.cs | 35 +- .../Microsoft.Psi/Operators/Processors.cs | 74 +- .../Runtime/Microsoft.Psi/Operators/Time.cs | 14 +- .../Runtime/Microsoft.Psi/Operators/Timers.cs | 14 +- .../Operators/VectorProcessors.cs | 22 +- .../Microsoft.Psi/Operators/Windows.cs | 104 +- .../Runtime/Microsoft.Psi/Operators/Zips.cs | 12 +- .../Persistence/MessageReader.cs | 2 +- .../Persistence/PsiStoreCommon.cs | 14 +- .../Persistence/PsiStoreWriter.cs | 50 +- .../Remoting/RemoteClockExporter.cs | 6 +- .../Remoting/RemoteClockImporter.cs | 8 +- .../Microsoft.Psi/Remoting/RemoteImporter.cs | 12 +- .../Scheduling/FutureWorkItemQueue.cs | 5 +- .../Microsoft.Psi/Scheduling/Scheduler.cs | 20 +- .../Serialization/DictionarySerializer.cs | 227 +++ .../Serialization/KnownSerializers.cs | 1 + .../Serialization/RecyclingPool.cs | 20 +- .../Microsoft.Psi/Streams/Receiver{T}.cs | 18 +- .../CrossFrameworkSerializationTester.cs | 12 + Sources/Runtime/Test.Psi/PipelineTest.cs | 52 + .../Runtime/Test.Psi/SerializationTester.cs | 92 +- .../Adapters.cs | 153 +- .../Box3DListVisualizationObject.cs | 2 +- .../Box3DVisualizationObject.cs | 12 +- ...insicsPoseToDepthImageCameraViewAdapter.cs | 34 + ...IntrinsicsToDepthImageCameraViewAdapter.cs | 23 + ...mageToDepthImageWithDefaultPoseAdapter.cs} | 5 +- ...insicsPoseToDepthImageCameraViewAdapter.cs | 44 + ...IntrinsicsToDepthImageCameraViewAdapter.cs | 29 + ...mageToDepthImageWithDefaultPoseAdapter.cs} | 9 +- ...ImageCameraViewToImageCameraViewAdapter.cs | 37 + ...eIntrinsicsPoseToImageCameraViewAdapter.cs | 40 + ...ImageIntrinsicsToImageCameraViewAdapter.cs | 29 + ...dedImageWithPoseToImageWithPoseAdapter.cs} | 9 +- ...eIntrinsicsPoseToImageCameraViewAdapter.cs | 34 + .../ImageIntrisicsToImageCameraViewAdapter.cs | 23 + .../ImageToImageWithDefaultPoseAdapter.cs} | 7 +- ...aIntrinsicsWithPoseVisualizationObject.cs} | 53 +- ...ageCameraViewAsMeshVisualizationObject.cs} | 46 +- ...eraViewAsPointCloudVisualizationObject.cs} | 44 +- ...alFocalLengthAsMeshVisualizationObject.cs} | 32 +- ...lLengthAsPointCloudVisualizationObject.cs} | 32 +- .../ImageCameraViewVisualizationObject.cs} | 64 +- ...ndManualFocalLengthVisualizationObject.cs} | 32 +- .../ImageRectangles/Adapters.cs | 169 ++ ...ImageRectangle3DListVisualizationObject.cs | 5 +- ...epthImageRectangle3DVisualizationObject.cs | 6 +- ...ImageRectangle3DListVisualizationObject.cs | 17 + .../ImageRectangle3DVisualizationObject.cs | 2 +- ...ial.Euclidean.Visualization.Windows.csproj | 1 + .../AngularVelocity3D.cs | 14 +- .../Bounds3D.cs | 42 +- .../Microsoft.Psi.Spatial.Euclidean/Box3D.cs | 106 +- .../CameraViews/CameraView{T}.cs | 129 ++ .../CameraViews/DepthImageCameraView.cs | 63 + .../EncodedDepthImageCameraView.cs | 63 + .../CameraViews/EncodedImageCameraView.cs | 63 + .../CameraViews/ImageCameraView.cs | 63 + .../CameraViews/ImageCameraView{TImage}.cs | 38 + .../CameraViews/Operators.cs | 227 +++ .../CameraViews/PointCloud3DCameraView.cs | 25 + .../CoordinateSystemVelocity3D.cs | 6 +- .../DepthImageRectangle3D.cs | 70 +- .../EncodedDepthImageRectangle3D.cs | 22 +- .../EncodedImageRectangle3D.cs | 22 +- .../ImageRectangle3D.cs | 66 +- .../ImageRectangles/ImageRectangle3D{T}.cs | 134 ++ .../{Images => ImageRectangles}/Operators.cs | 66 +- .../Microsoft.Psi.Spatial.Euclidean/Mesh3D.cs | 6 + .../Operators.cs | 13 +- .../PointCloud3D.cs | 38 +- .../Rectangle3D.cs | 43 + .../VoxelGrid.cs | 13 +- .../SystemSpeechIntentDetector.cs | 11 +- .../SystemSpeechRecognizer.cs | 11 +- .../SystemSpeechSynthesizer.cs | 11 +- .../SystemVoiceActivityDetector.cs | 11 +- .../Microsoft.Psi.PsiStudio/MainWindow.xaml | 159 +- .../MainWindowViewModel.cs | 87 +- .../PsiStudioSettings.cs | 2 - .../CameraViewToSpatialCameraViewAdapter.cs | 21 - ...meraViewToSpatialDepthCameraViewAdapter.cs | 21 - ...dedCameraViewToSpatialCameraViewAdapter.cs | 27 - ...meraViewToSpatialDepthCameraViewAdapter.cs | 27 - .../EncodedDepthImageToDepthImageAdapter.cs | 6 +- .../EncodedDepthImageToImageAdapter.cs | 6 +- ...ialCameraViewToSpatialCameraViewAdapter.cs | 27 - ...meraViewToSpatialDepthCameraViewAdapter.cs | 27 - ...ialDepthImageToSpatialDepthImageAdapter.cs | 26 - .../Adapters/Ray3DListToNullableAdapter.cs | 21 + .../Adapters/StreamAdapterAttribute.cs | 21 + .../Common/ContextMenuName.cs | 5 + .../Common/IconSourcePath.cs | 34 +- .../Icons/go-to-time.png | Bin 0 -> 1555 bytes .../Icons/move-selection-left.png | Bin 0 -> 1573 bytes .../Icons/move-selection-right.png | Bin 0 -> 1573 bytes .../Icons/multiple-sessions-from-folder.png | Bin 0 -> 1469 bytes .../Icons/partition-add-multiple.png | Bin 0 -> 1537 bytes .../Icons/session-from-folder.png | Bin 0 -> 1469 bytes .../Icons/session-from-store.png | Bin 1469 -> 1469 bytes ...Microsoft.Psi.Visualization.Windows.csproj | 12 + .../Navigation/Navigator.cs | 79 +- .../ViewModels/DatasetViewModel.cs | 223 ++- .../ViewModels/DerivedMemberStreamTreeNode.cs | 9 +- .../ViewModels/PartitionViewModel.cs | 8 + .../ViewModels/SessionViewModel.cs | 137 +- .../ViewModels/StreamContainerTreeNode.cs | 37 +- .../ViewModels/StreamTreeNode.cs | 26 +- .../Views/CanvasVisualizationPanelView.xaml | 4 +- .../CanvasVisualizationPanelView.xaml.cs | 79 +- .../Views/InstantVisualizationPanelView.cs | 85 + ...tantVisualizationPlaceholderPanelView.xaml | 4 +- ...tVisualizationPlaceholderPanelView.xaml.cs | 9 +- .../StreamVisualizationObjectCanvasView.cs | 1 - .../Views/VisualizationPanelView.cs | 32 +- ...DiagnosticsVisualizationObjectView.xaml.cs | 2 + ...pelineDiagnosticsVisualizationPresenter.cs | 27 +- ...leListVisualizationObjectCanvasItemView.cs | 8 +- .../Views/XYVisualizationPanelView.xaml | 4 +- .../Views/XYVisualizationPanelView.xaml.cs | 70 +- .../Views/XYZVisualizationPanelView.xaml | 4 +- .../Views/XYZVisualizationPanelView.xaml.cs | 78 +- .../VisualizationContext.cs | 7 +- ...meIntervalAnnotationVisualizationObject.cs | 8 +- .../AudioVisualizationObject.cs | 2 +- ...LabeledRectangleListVisualizationObject.cs | 17 + .../ModelVisual3DVisualizationObject.cs | 2 +- .../PipelineDiagnosticsVisualizationObject.cs | 5 + .../Ray3DListVisualizationObject.cs | 15 + .../StreamVisualizationObject{TData}.cs | 2 +- .../UpdatableVisual3DList{TVisual3D}.cs | 2 +- .../VisualizationContainer.cs | 123 +- .../VisualizationObject.cs | 2 +- .../CanvasVisualizationPanel.cs | 21 +- .../InstantVisualizationContainer.cs | 9 + .../InstantVisualizationPanel.cs | 65 + .../InstantVisualizationPlaceholderPanel.cs | 16 +- .../XYVisualizationPanel.cs | 80 +- .../XYZVisualizationPanel.cs | 15 +- .../VisualizerMetadata.cs | 86 +- .../Windows/GetParameterWindow.xaml | 13 +- .../Windows/GetParameterWindow.xaml.cs | 35 +- .../Properties/AssemblyInfo.cs | 6 +- 409 files changed, 15690 insertions(+), 5080 deletions(-) create mode 100644 Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageFromBitmapStreamDecoder.cs create mode 100644 Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageFromBitmapStreamDecoder.cs rename Sources/{Calibration/Microsoft.Psi.Calibration/DepthPixelSemantics.cs => Imaging/Microsoft.Psi.Imaging/DepthValueSemantics.cs} (79%) create mode 100644 Sources/Imaging/Microsoft.Psi.Imaging/IDepthImage.cs create mode 100644 Sources/Imaging/Microsoft.Psi.Imaging/IImage.cs create mode 100644 Sources/Imaging/Microsoft.Psi.Imaging/ImageFromGZipStreamDecoder.cs create mode 100644 Sources/Imaging/Microsoft.Psi.Imaging/ImageFromNV12StreamDecoder.cs create mode 100644 Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNDetection.cs create mode 100644 Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNDetectionResults.cs create mode 100644 Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNModelConfiguration.cs create mode 100644 Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNModelOutputParser.cs create mode 100644 Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNModelRunner.cs create mode 100644 Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNOnnxModel.cs create mode 100644 Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/Operators.cs create mode 100644 Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Visualization.Windows/MaskRCNNDetectionAdapter.cs create mode 100644 Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Visualization.Windows/Microsoft.Psi.Onnx.Visualization.Windows.csproj create mode 100644 Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Visualization.Windows/stylecop.json create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCapture.sln create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/LockScreenLogo.scale-200.png create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/SplashScreen.scale-200.png create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/Square150x150Logo.scale-200.png create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/Square44x44Logo.scale-200.png create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/Square44x44Logo.targetsize-24_altform-unplated.png create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/StoreLogo.png create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/Wide310x150Logo.scale-200.png create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/HoloLensCaptureApp.cs create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/HoloLensCaptureApp.csproj create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Package.appxmanifest create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Properties/AssemblyInfo.cs create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Properties/Default.rd.xml create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Readme.md create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/stylecop.json create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/DataExporter.cs create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/HoloLensCaptureExporter.cs create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/HoloLensCaptureExporter.csproj create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/Operators.cs create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/Readme.md create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/Verbs.cs create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/stylecop.json create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureInterop/HoloLensCaptureInterop.csproj create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureInterop/Serializers.cs create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureInterop/stylecop.json create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/App.config create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/HoloLensCaptureServer.cs create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/HoloLensCaptureServer.csproj create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/Readme.md create mode 100644 Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/stylecop.json create mode 100644 Sources/MixedReality/HoloLensCapture/Readme.md create mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ResearchModeCameraConfiguration.cs create mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/SerializedCameraIntrinsics.cs create mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality.Visualization.Windows/TrackedHandVisualizationObjectAdapter.cs create mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality/Properties/AssemblyInfo.cs create mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality/PsiInput.cs create mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Box3DStereoKitRenderer.cs create mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/EncodedImageRectangle3DStereoKitRenderer.cs create mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/HandsStereoKitRenderer.cs delete mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Mesh3DListStereoKitRenderer.cs create mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Mesh3DStereoKitRenderer.cs delete mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/ModelBasedStereoKitRenderer.cs delete mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Rectangle3DListStereoKitRenderer.cs create mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Rectangle3DStereoKitRenderer.cs create mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/StereoKitRenderer.cs create mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/TextStereoKitRenderer.cs create mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/TextStereoKitRendererConfiguration.cs create mode 100644 Sources/MixedReality/Microsoft.Psi.MixedReality/TimeHelper.cs create mode 100644 Sources/Runtime/Microsoft.Psi/Serialization/DictionarySerializer.cs create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/DepthImageIntrinsicsPoseToDepthImageCameraViewAdapter.cs create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/DepthImageIntrinsicsToDepthImageCameraViewAdapter.cs rename Sources/{Visualization/Microsoft.Psi.Visualization.Windows/Adapters/DepthImageToSpatialDepthImageAdapter.cs => Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/DepthImageToDepthImageWithDefaultPoseAdapter.cs} (70%) create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedDepthImageIntrinsicsPoseToDepthImageCameraViewAdapter.cs create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedDepthImageIntrinsicsToDepthImageCameraViewAdapter.cs rename Sources/{Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthImageToSpatialDepthCameraViewManualFocalLengthAdapter.cs => Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedDepthImageToDepthImageWithDefaultPoseAdapter.cs} (67%) create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageCameraViewToImageCameraViewAdapter.cs create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageIntrinsicsPoseToImageCameraViewAdapter.cs create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageIntrinsicsToImageCameraViewAdapter.cs rename Sources/{Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialImageToSpatialImageAdapter.cs => Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageWithPoseToImageWithPoseAdapter.cs} (67%) create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/ImageIntrinsicsPoseToImageCameraViewAdapter.cs create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/ImageIntrisicsToImageCameraViewAdapter.cs rename Sources/{Visualization/Microsoft.Psi.Visualization.Windows/Adapters/ImageToSpatialImageAdapter.cs => Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/ImageToImageWithDefaultPoseAdapter.cs} (61%) rename Sources/{Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialCameraVisualizationObject.cs => Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/CameraIntrinsicsWithPoseVisualizationObject.cs} (72%) rename Sources/{Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsMeshVisualizationObject.cs => Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageCameraViewAsMeshVisualizationObject.cs} (82%) rename Sources/{Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsPointCloudVisualizationObject.cs => Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageCameraViewAsPointCloudVisualizationObject.cs} (76%) rename Sources/{Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsMeshManualFocalLengthVisualizationObject.cs => Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageWithPoseAndManualFocalLengthAsMeshVisualizationObject.cs} (67%) rename Sources/{Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsPointCloudManualFocalLengthVisualizationObject.cs => Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageWithPoseAndManualFocalLengthAsPointCloudVisualizationObject.cs} (66%) rename Sources/{Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialCameraViewVisualizationObject.cs => Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/ImageCameraViewVisualizationObject.cs} (74%) rename Sources/{Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialCameraViewManualFocalLengthVisualizationObject.cs => Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/ImageWithPoseAndManualFocalLengthVisualizationObject.cs} (68%) create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/Adapters.cs rename Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/{ => ImageRectangles}/DepthImageRectangle3DListVisualizationObject.cs (71%) rename Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/{ => ImageRectangles}/DepthImageRectangle3DVisualizationObject.cs (98%) create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/ImageRectangle3DListVisualizationObject.cs rename Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/{ => ImageRectangles}/ImageRectangle3DVisualizationObject.cs (99%) create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/CameraView{T}.cs create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/DepthImageCameraView.cs create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/EncodedDepthImageCameraView.cs create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/EncodedImageCameraView.cs create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/ImageCameraView.cs create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/ImageCameraView{TImage}.cs create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/Operators.cs create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/PointCloud3DCameraView.cs rename Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/{Images => ImageRectangles}/DepthImageRectangle3D.cs (63%) rename Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/{Images => ImageRectangles}/EncodedDepthImageRectangle3D.cs (87%) rename Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/{Images => ImageRectangles}/EncodedImageRectangle3D.cs (87%) rename Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/{Images => ImageRectangles}/ImageRectangle3D.cs (69%) create mode 100644 Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/ImageRectangle3D{T}.cs rename Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/{Images => ImageRectangles}/Operators.cs (67%) delete mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/CameraViewToSpatialCameraViewAdapter.cs delete mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/DepthCameraViewToSpatialDepthCameraViewAdapter.cs delete mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedCameraViewToSpatialCameraViewAdapter.cs delete mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthCameraViewToSpatialDepthCameraViewAdapter.cs delete mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialCameraViewToSpatialCameraViewAdapter.cs delete mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialDepthCameraViewToSpatialDepthCameraViewAdapter.cs delete mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialDepthImageToSpatialDepthImageAdapter.cs create mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/Ray3DListToNullableAdapter.cs create mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/go-to-time.png create mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/move-selection-left.png create mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/move-selection-right.png create mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/multiple-sessions-from-folder.png create mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/partition-add-multiple.png create mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/session-from-folder.png create mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/InstantVisualizationPanelView.cs create mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Ray3DListVisualizationObject.cs create mode 100644 Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationPanel.cs diff --git a/.gitignore b/.gitignore index 7f1c644b0..d033613f3 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,9 @@ bld/ # Uncomment if you have tasks that create the project's static files in wwwroot #wwwroot/ +# Visual Studio 2017 auto generated files +Generated\ Files/ + # MSTest test Results [Tt]est[Rr]esult*/ [Bb]uild[Ll]og.* diff --git a/Directory.Build.props b/Directory.Build.props index 7b8affd68..04ffdd660 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -6,7 +6,7 @@ Microsoft Corporation microsoft,psi Microsoft - 0.16.92.1 + 0.17.52.1 $(AssemblyVersion) $(AssemblyVersion)-beta false diff --git a/Psi.sln b/Psi.sln index ea34bff28..34ea431f9 100644 --- a/Psi.sln +++ b/Psi.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.28917.182 +# Visual Studio Version 17 +VisualStudioVersion = 17.2.32210.308 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Sources", "Sources", "{A0856299-D28A-4513-B964-3FA5290FF160}" EndProject @@ -214,6 +214,21 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Psi.MixedReality. EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi.MixedReality.Visualization.Windows", "Sources\MixedReality\Microsoft.Psi.MixedReality.Visualization.Windows\Microsoft.Psi.MixedReality.Visualization.Windows.csproj", "{BE95524A-F9C2-4D0D-8F7E-1C7019B5A114}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi.Onnx.Visualization.Windows", "Sources\Integrations\Onnx\Microsoft.Psi.Onnx.Visualization.Windows\Microsoft.Psi.Onnx.Visualization.Windows.csproj", "{74504D41-B716-4B0B-B265-ED2A91A2A5C2}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HoloLensCapture", "HoloLensCapture", "{49072585-8CC1-43A5-BE0D-ABCE888BC5D1}" + ProjectSection(SolutionItems) = preProject + Sources\MixedReality\HoloLensCapture\Readme.md = Sources\MixedReality\HoloLensCapture\Readme.md + EndProjectSection +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HoloLensCaptureApp", "Sources\MixedReality\HoloLensCapture\HoloLensCaptureApp\HoloLensCaptureApp.csproj", "{D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HoloLensCaptureServer", "Sources\MixedReality\HoloLensCapture\HoloLensCaptureServer\HoloLensCaptureServer.csproj", "{272CEB19-2B5A-49BC-B8EA-CBC79AA87C37}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HoloLensCaptureExporter", "Sources\MixedReality\HoloLensCapture\HoloLensCaptureExporter\HoloLensCaptureExporter.csproj", "{1C844B9E-A51C-483C-A045-7AB8F2012581}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HoloLensCaptureInterop", "Sources\MixedReality\HoloLensCapture\HoloLensCaptureInterop\HoloLensCaptureInterop.csproj", "{76E38559-0AF2-4AE3-BCE8-8653277A5B07}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -476,6 +491,28 @@ Global {BE95524A-F9C2-4D0D-8F7E-1C7019B5A114}.Debug|Any CPU.Build.0 = Debug|Any CPU {BE95524A-F9C2-4D0D-8F7E-1C7019B5A114}.Release|Any CPU.ActiveCfg = Release|Any CPU {BE95524A-F9C2-4D0D-8F7E-1C7019B5A114}.Release|Any CPU.Build.0 = Release|Any CPU + {74504D41-B716-4B0B-B265-ED2A91A2A5C2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {74504D41-B716-4B0B-B265-ED2A91A2A5C2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {74504D41-B716-4B0B-B265-ED2A91A2A5C2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {74504D41-B716-4B0B-B265-ED2A91A2A5C2}.Release|Any CPU.Build.0 = Release|Any CPU + {D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D}.Debug|Any CPU.ActiveCfg = Debug|ARM + {D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D}.Debug|Any CPU.Build.0 = Debug|ARM + {D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D}.Debug|Any CPU.Deploy.0 = Debug|ARM + {D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D}.Release|Any CPU.ActiveCfg = Release|ARM + {D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D}.Release|Any CPU.Build.0 = Release|ARM + {D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D}.Release|Any CPU.Deploy.0 = Release|ARM + {272CEB19-2B5A-49BC-B8EA-CBC79AA87C37}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {272CEB19-2B5A-49BC-B8EA-CBC79AA87C37}.Debug|Any CPU.Build.0 = Debug|Any CPU + {272CEB19-2B5A-49BC-B8EA-CBC79AA87C37}.Release|Any CPU.ActiveCfg = Release|Any CPU + {272CEB19-2B5A-49BC-B8EA-CBC79AA87C37}.Release|Any CPU.Build.0 = Release|Any CPU + {1C844B9E-A51C-483C-A045-7AB8F2012581}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1C844B9E-A51C-483C-A045-7AB8F2012581}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1C844B9E-A51C-483C-A045-7AB8F2012581}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1C844B9E-A51C-483C-A045-7AB8F2012581}.Release|Any CPU.Build.0 = Release|Any CPU + {76E38559-0AF2-4AE3-BCE8-8653277A5B07}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {76E38559-0AF2-4AE3-BCE8-8653277A5B07}.Debug|Any CPU.Build.0 = Debug|Any CPU + {76E38559-0AF2-4AE3-BCE8-8653277A5B07}.Release|Any CPU.ActiveCfg = Release|Any CPU + {76E38559-0AF2-4AE3-BCE8-8653277A5B07}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -570,6 +607,12 @@ Global {3434D5B2-B06F-4356-9E9B-90171CEF482B} = {32023088-0392-4B48-B2CF-3754B55C6DE9} {ECD9E150-8104-4DA3-B807-A6A4392A67C6} = {32023088-0392-4B48-B2CF-3754B55C6DE9} {BE95524A-F9C2-4D0D-8F7E-1C7019B5A114} = {32023088-0392-4B48-B2CF-3754B55C6DE9} + {74504D41-B716-4B0B-B265-ED2A91A2A5C2} = {EE4035A8-CEFE-4E3A-9CD9-4AE7E88DA2C4} + {49072585-8CC1-43A5-BE0D-ABCE888BC5D1} = {32023088-0392-4B48-B2CF-3754B55C6DE9} + {D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D} = {49072585-8CC1-43A5-BE0D-ABCE888BC5D1} + {272CEB19-2B5A-49BC-B8EA-CBC79AA87C37} = {49072585-8CC1-43A5-BE0D-ABCE888BC5D1} + {1C844B9E-A51C-483C-A045-7AB8F2012581} = {49072585-8CC1-43A5-BE0D-ABCE888BC5D1} + {76E38559-0AF2-4AE3-BCE8-8653277A5B07} = {49072585-8CC1-43A5-BE0D-ABCE888BC5D1} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {EAF15EE9-DCC5-411B-A9E5-7C2F3D132331} diff --git a/Sources/Audio/Microsoft.Psi.Audio.Linux/AudioCapture.cs b/Sources/Audio/Microsoft.Psi.Audio.Linux/AudioCapture.cs index f9873742e..9ab2a503e 100644 --- a/Sources/Audio/Microsoft.Psi.Audio.Linux/AudioCapture.cs +++ b/Sources/Audio/Microsoft.Psi.Audio.Linux/AudioCapture.cs @@ -20,6 +20,7 @@ namespace Microsoft.Psi.Audio public sealed class AudioCapture : IProducer, ISourceComponent, IDisposable { private readonly Pipeline pipeline; + private readonly string name; /// /// The configuration for this component. @@ -41,11 +42,6 @@ public sealed class AudioCapture : IProducer, ISourceComponent, IDi /// private AudioBuffer buffer; - /// - /// Keep track of the timestamp of the last audio buffer (computed from the value reported to us by the capture driver). - /// - private DateTime lastPostedAudioTime = DateTime.MinValue; - private Thread background; private volatile bool isStopping; @@ -54,9 +50,11 @@ public sealed class AudioCapture : IProducer, ISourceComponent, IDi /// /// The pipeline to add the component to. /// The component configuration. - public AudioCapture(Pipeline pipeline, AudioCaptureConfiguration configuration) + /// An optional name for this component. + public AudioCapture(Pipeline pipeline, AudioCaptureConfiguration configuration, string name = nameof(AudioCapture)) { this.pipeline = pipeline; + this.name = name; this.configuration = configuration; this.audioBuffers = pipeline.CreateEmitter(this, "AudioBuffers"); } @@ -176,6 +174,9 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) notifyCompleted(); } + /// + public override string ToString() => this.name; + private void Stop() { // stop any running background thread and wait for it to terminate diff --git a/Sources/Audio/Microsoft.Psi.Audio.Linux/AudioPlayer.cs b/Sources/Audio/Microsoft.Psi.Audio.Linux/AudioPlayer.cs index a450f4b97..80b26bce0 100644 --- a/Sources/Audio/Microsoft.Psi.Audio.Linux/AudioPlayer.cs +++ b/Sources/Audio/Microsoft.Psi.Audio.Linux/AudioPlayer.cs @@ -40,8 +40,9 @@ public sealed class AudioPlayer : SimpleConsumer, IDisposable /// /// The pipeline to add the component to. /// The component configuration. - public AudioPlayer(Pipeline pipeline, AudioPlayerConfiguration configuration) - : base(pipeline) + /// An optional name for this component. + public AudioPlayer(Pipeline pipeline, AudioPlayerConfiguration configuration, string name = nameof(AudioPlayer)) + : base(pipeline, name) { pipeline.PipelineRun += (s, e) => this.OnPipelineRun(); this.In.Unsubscribed += _ => this.OnUnsubscribed(); diff --git a/Sources/Audio/Microsoft.Psi.Audio.Windows/AudioCapture.cs b/Sources/Audio/Microsoft.Psi.Audio.Windows/AudioCapture.cs index 30e854f24..ad8e65631 100644 --- a/Sources/Audio/Microsoft.Psi.Audio.Windows/AudioCapture.cs +++ b/Sources/Audio/Microsoft.Psi.Audio.Windows/AudioCapture.cs @@ -23,6 +23,7 @@ namespace Microsoft.Psi.Audio public sealed class AudioCapture : IProducer, ISourceComponent, IDisposable { private readonly Pipeline pipeline; + private readonly string name; /// /// The configuration for this component. @@ -59,9 +60,11 @@ public sealed class AudioCapture : IProducer, ISourceComponent, IDi /// /// The pipeline to add the component to. /// The component configuration. - public AudioCapture(Pipeline pipeline, AudioCaptureConfiguration configuration) + /// An optional name for the component. + public AudioCapture(Pipeline pipeline, AudioCaptureConfiguration configuration, string name = nameof(AudioCapture)) { this.pipeline = pipeline; + this.name = name; this.configuration = configuration; this.audioBuffers = pipeline.CreateEmitter(this, "AudioBuffers"); this.AudioLevelInput = pipeline.CreateReceiver(this, this.SetAudioLevel, nameof(this.AudioLevelInput), true); @@ -81,10 +84,12 @@ public AudioCapture(Pipeline pipeline, AudioCaptureConfiguration configuration) /// /// The pipeline to add the component to. /// The component configuration file. - public AudioCapture(Pipeline pipeline, string configurationFilename = null) + /// An optional name for the component. + public AudioCapture(Pipeline pipeline, string configurationFilename = null, string name = nameof(AudioCapture)) : this( pipeline, - (configurationFilename == null) ? new AudioCaptureConfiguration() : new ConfigurationHelper(configurationFilename).Configuration) + (configurationFilename == null) ? new AudioCaptureConfiguration() : new ConfigurationHelper(configurationFilename).Configuration, + name) { } @@ -94,8 +99,9 @@ public AudioCapture(Pipeline pipeline, string configurationFilename = null) /// The pipeline to add the component to. /// The output format to use. /// The name of the audio device. - public AudioCapture(Pipeline pipeline, WaveFormat outputFormat, string deviceName = null) - : this(pipeline, new AudioCaptureConfiguration() { Format = outputFormat, DeviceName = deviceName }) + /// An optional name for the component. + public AudioCapture(Pipeline pipeline, WaveFormat outputFormat, string deviceName = null, string name = nameof(AudioCapture)) + : this(pipeline, new AudioCaptureConfiguration() { Format = outputFormat, DeviceName = deviceName }, name) { } @@ -198,6 +204,9 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) this.sourceFormat = null; } + /// + public override string ToString() => this.name; + /// /// The event handler that processes new audio data packets. /// diff --git a/Sources/Audio/Microsoft.Psi.Audio.Windows/AudioPlayer.cs b/Sources/Audio/Microsoft.Psi.Audio.Windows/AudioPlayer.cs index d287b2863..45e3bc82c 100644 --- a/Sources/Audio/Microsoft.Psi.Audio.Windows/AudioPlayer.cs +++ b/Sources/Audio/Microsoft.Psi.Audio.Windows/AudioPlayer.cs @@ -35,8 +35,9 @@ public sealed class AudioPlayer : SimpleConsumer, ISourceComponent, /// /// The pipeline to add the component to. /// The component configuration. - public AudioPlayer(Pipeline pipeline, AudioPlayerConfiguration configuration) - : base(pipeline) + /// An optional name for the component. + public AudioPlayer(Pipeline pipeline, AudioPlayerConfiguration configuration, string name = nameof(AudioPlayer)) + : base(pipeline, name) { this.pipeline = pipeline; this.configuration = configuration; @@ -58,10 +59,12 @@ public AudioPlayer(Pipeline pipeline, AudioPlayerConfiguration configuration) /// /// The pipeline to add the component to. /// The component configuration file. - public AudioPlayer(Pipeline pipeline, string configurationFilename = null) + /// An optional name for the component. + public AudioPlayer(Pipeline pipeline, string configurationFilename = null, string name = nameof(AudioPlayer)) : this( pipeline, - (configurationFilename == null) ? new AudioPlayerConfiguration() : new ConfigurationHelper(configurationFilename).Configuration) + (configurationFilename == null) ? new AudioPlayerConfiguration() : new ConfigurationHelper(configurationFilename).Configuration, + name) { } diff --git a/Sources/Audio/Microsoft.Psi.Audio.Windows/AudioResampler.cs b/Sources/Audio/Microsoft.Psi.Audio.Windows/AudioResampler.cs index 7479fc6a8..23727b3df 100644 --- a/Sources/Audio/Microsoft.Psi.Audio.Windows/AudioResampler.cs +++ b/Sources/Audio/Microsoft.Psi.Audio.Windows/AudioResampler.cs @@ -51,8 +51,9 @@ public sealed class AudioResampler : ConsumerProducer, /// /// The pipeline to add the component to. /// The component configuration. - public AudioResampler(Pipeline pipeline, AudioResamplerConfiguration configuration) - : base(pipeline) + /// An optional name for the component. + public AudioResampler(Pipeline pipeline, AudioResamplerConfiguration configuration, string name = nameof(AudioResampler)) + : base(pipeline, name) { this.configuration = configuration; this.currentInputFormat = configuration.InputFormat; @@ -73,10 +74,12 @@ public AudioResampler(Pipeline pipeline, AudioResamplerConfiguration configurati /// /// The pipeline to add the component to. /// The component configuration file. - public AudioResampler(Pipeline pipeline, string configurationFilename = null) + /// An optional name for the component. + public AudioResampler(Pipeline pipeline, string configurationFilename = null, string name = nameof(AudioResampler)) : this( pipeline, - (configurationFilename == null) ? new AudioResamplerConfiguration() : new ConfigurationHelper(configurationFilename).Configuration) + (configurationFilename == null) ? new AudioResamplerConfiguration() : new ConfigurationHelper(configurationFilename).Configuration, + name) { } diff --git a/Sources/Audio/Microsoft.Psi.Audio.Windows/Operators.cs b/Sources/Audio/Microsoft.Psi.Audio.Windows/Operators.cs index 85b778e16..cff675567 100644 --- a/Sources/Audio/Microsoft.Psi.Audio.Windows/Operators.cs +++ b/Sources/Audio/Microsoft.Psi.Audio.Windows/Operators.cs @@ -17,11 +17,14 @@ public static class Operators /// A stream of audio to be resampled. /// The resampler configuration. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of resampled audio. - public static IProducer Resample(this IProducer source, AudioResamplerConfiguration configuration, DeliveryPolicy deliveryPolicy = null) - { - return source.PipeTo(new AudioResampler(source.Out.Pipeline, configuration), deliveryPolicy); - } + public static IProducer Resample( + this IProducer source, + AudioResamplerConfiguration configuration, + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Resample)) + => source.PipeTo(new AudioResampler(source.Out.Pipeline, configuration, name), deliveryPolicy); /// /// Resamples an audio stream. @@ -29,10 +32,13 @@ public static IProducer Resample(this IProducer source /// A stream audio to be resampled. /// The desired audio output format for the resampled stream. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of resampled audio. - public static IProducer Resample(this IProducer source, WaveFormat outputFormat, DeliveryPolicy deliveryPolicy = null) - { - return Resample(source, new AudioResamplerConfiguration() { OutputFormat = outputFormat }, deliveryPolicy); - } + public static IProducer Resample( + this IProducer source, + WaveFormat outputFormat, + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Resample)) + => Resample(source, new AudioResamplerConfiguration() { OutputFormat = outputFormat }, deliveryPolicy, name); } } diff --git a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/AcousticFeaturesExtractor.cs b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/AcousticFeaturesExtractor.cs index 40ad3dc16..fcefa2c21 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/AcousticFeaturesExtractor.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/AcousticFeaturesExtractor.cs @@ -17,6 +17,7 @@ namespace Microsoft.Psi.Audio /// public sealed class AcousticFeaturesExtractor : IConsumer { + private readonly string name; private readonly Connector inAudio; /// @@ -24,8 +25,9 @@ public sealed class AcousticFeaturesExtractor : IConsumer /// /// The pipeline to add the component to. /// The component configuration file. - public AcousticFeaturesExtractor(Pipeline pipeline, string configurationFilename = null) - : this(pipeline, new ConfigurationHelper(configurationFilename).Configuration) + /// An optional name for the component. + public AcousticFeaturesExtractor(Pipeline pipeline, string configurationFilename = null, string name = nameof(AcousticFeaturesExtractor)) + : this(pipeline, new ConfigurationHelper(configurationFilename).Configuration, name) { } @@ -34,9 +36,11 @@ public AcousticFeaturesExtractor(Pipeline pipeline, string configurationFilename /// /// The pipeline to add the component to. /// The component configuration. - public AcousticFeaturesExtractor(Pipeline pipeline, AcousticFeaturesExtractorConfiguration configuration) + /// An optional name for the component. + public AcousticFeaturesExtractor(Pipeline pipeline, AcousticFeaturesExtractorConfiguration configuration, string name = nameof(AcousticFeaturesExtractor)) { // Create the Audio passthrough emitter and hook it up to the receiver + this.name = name; this.inAudio = pipeline.CreateConnector(nameof(this.inAudio)); this.In = this.inAudio.In; @@ -163,5 +167,8 @@ public AcousticFeaturesExtractor(Pipeline pipeline, AcousticFeaturesExtractorCon /// Gets the stream containing the spectral entropy. /// public IProducer SpectralEntropy { get; } + + /// + public override string ToString() => this.name; } } diff --git a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FFT.cs b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FFT.cs index fc5bd9bc9..ed31c607e 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FFT.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FFT.cs @@ -10,7 +10,7 @@ namespace Microsoft.Psi.Audio /// public sealed class FFT : ConsumerProducer { - private FastFourierTransform fft; + private readonly FastFourierTransform fft; private float[] fftOutput; /// @@ -19,8 +19,9 @@ public sealed class FFT : ConsumerProducer /// The pipeline to add the component to. /// The FFT size. /// The window size. - public FFT(Pipeline pipeline, int fftSize, int inputSize) - : base(pipeline) + /// An optional name for this component. + public FFT(Pipeline pipeline, int fftSize, int inputSize, string name = nameof(FFT)) + : base(pipeline, name) { this.fft = new FastFourierTransform(fftSize, inputSize); } diff --git a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FFTPower.cs b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FFTPower.cs index 129cfb84a..f54bdff55 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FFTPower.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FFTPower.cs @@ -16,8 +16,9 @@ public sealed class FFTPower : ConsumerProducer /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - public FFTPower(Pipeline pipeline) - : base(pipeline) + /// An optional name for this component. + public FFTPower(Pipeline pipeline, string name = nameof(FFTPower)) + : base(pipeline, name) { } diff --git a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FrameShift.cs b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FrameShift.cs index b5f7eaa0f..ba625552e 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FrameShift.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FrameShift.cs @@ -11,11 +11,11 @@ namespace Microsoft.Psi.Audio /// public sealed class FrameShift : ConsumerProducer { - private int frameSizeInBytes; - private int frameShiftInBytes; - private int frameOverlapInBytes; - private byte[] frameBuffer; - private double bytesPerSec; + private readonly int frameSizeInBytes; + private readonly int frameShiftInBytes; + private readonly int frameOverlapInBytes; + private readonly byte[] frameBuffer; + private readonly double bytesPerSec; private int frameBytesRemaining; private DateTime lastOriginatingTime = DateTime.MinValue; @@ -26,8 +26,9 @@ public sealed class FrameShift : ConsumerProducer /// The frame size in bytes. /// The number of bytes to shift by. /// The sampling frequency in bytes per second. - public FrameShift(Pipeline pipeline, int frameSizeInBytes, int frameShiftInBytes, double bytesPerSec) - : base(pipeline) + /// An optional name for this component. + public FrameShift(Pipeline pipeline, int frameSizeInBytes, int frameShiftInBytes, double bytesPerSec, string name = nameof(FrameShift)) + : base(pipeline, name) { this.frameSizeInBytes = frameSizeInBytes; this.frameShiftInBytes = frameShiftInBytes; diff --git a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FrequencyDomainEnergy.cs b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FrequencyDomainEnergy.cs index f00c0bf0a..620772622 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FrequencyDomainEnergy.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/FrequencyDomainEnergy.cs @@ -19,8 +19,9 @@ public sealed class FrequencyDomainEnergy : ConsumerProducer /// The pipeline to add the component to. /// The starting frequency of the band. /// The ending frequency of the band. - public FrequencyDomainEnergy(Pipeline pipeline, int start, int end) - : base(pipeline) + /// An optional name for this component. + public FrequencyDomainEnergy(Pipeline pipeline, int start, int end, string name = nameof(FrequencyDomainEnergy)) + : base(pipeline, name) { this.start = start; this.end = end; diff --git a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/LogEnergy.cs b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/LogEnergy.cs index 2e12913cf..27a5a6f83 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/LogEnergy.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/LogEnergy.cs @@ -25,8 +25,9 @@ public sealed class LogEnergy : ConsumerProducer /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - public LogEnergy(Pipeline pipeline) - : base(pipeline) + /// An optional name for this component. + public LogEnergy(Pipeline pipeline, string name = nameof(LogEnergy)) + : base(pipeline, name) { } diff --git a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/SpectralEntropy.cs b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/SpectralEntropy.cs index daa9d8111..7f4d7eef1 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/SpectralEntropy.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/SpectralEntropy.cs @@ -11,8 +11,8 @@ namespace Microsoft.Psi.Audio /// public sealed class SpectralEntropy : ConsumerProducer { - private int start; - private int end; + private readonly int start; + private readonly int end; /// /// Initializes a new instance of the class. @@ -20,8 +20,9 @@ public sealed class SpectralEntropy : ConsumerProducer /// The pipeline to add the component to. /// The starting frequency of the band. /// The ending frequency of the band. - public SpectralEntropy(Pipeline pipeline, int start, int end) - : base(pipeline) + /// An optional name for this component. + public SpectralEntropy(Pipeline pipeline, int start, int end, string name = nameof(SpectralEntropy)) + : base(pipeline, name) { this.start = start; this.end = end; diff --git a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/ToFloat.cs b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/ToFloat.cs index 3ac937e1a..f04600653 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/ToFloat.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/ToFloat.cs @@ -11,27 +11,28 @@ namespace Microsoft.Psi.Audio /// public sealed class ToFloat : ConsumerProducer { - private ushort bytesPerSample; + private readonly ushort bytesPerSample; + private readonly Func convertSample; private float[] buffer; - private Func convertSample; /// /// Initializes a new instance of the class. /// /// The pipeline to add the component to. /// The format of the input audio. - public ToFloat(Pipeline pipeline, WaveFormat format) - : base(pipeline) + /// An optional name for this component. + public ToFloat(Pipeline pipeline, WaveFormat format, string name = nameof(ToFloat)) + : base(pipeline, name) { this.bytesPerSample = format.BlockAlign; - switch (format.BitsPerSample) + this.convertSample = format.BitsPerSample switch { - case 8: this.convertSample = (a, i) => a[i]; break; - case 16: this.convertSample = (a, i) => BitConverter.ToInt16(a, i); break; - case 24: this.convertSample = (a, i) => BitConverter.ToInt32(new byte[] { a[i], a[i + 1], a[i + 2], (byte)((a[i + 2] & 0x80) == 0 ? 0 : 0xFF) }, 0); break; - case 32: this.convertSample = (a, i) => BitConverter.ToInt32(a, i); break; - default: throw new FormatException("Valid sample sizes are 8, 16, 24 or 32 bits"); - } + 8 => (a, i) => a[i], + 16 => (a, i) => BitConverter.ToInt16(a, i), + 24 => (a, i) => BitConverter.ToInt32(new byte[] { a[i], a[i + 1], a[i + 2], (byte)((a[i + 2] & 0x80) == 0 ? 0 : 0xFF) }, 0), + 32 => (a, i) => BitConverter.ToInt32(a, i), + _ => throw new FormatException("Valid sample sizes are 8, 16, 24 or 32 bits"), + }; } /// diff --git a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/ZeroCrossingRate.cs b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/ZeroCrossingRate.cs index 03d7e9913..532ed68ae 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/ZeroCrossingRate.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/AcousticFeatures/ZeroCrossingRate.cs @@ -14,8 +14,9 @@ public sealed class ZeroCrossingRate : ConsumerProducer /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - public ZeroCrossingRate(Pipeline pipeline) - : base(pipeline) + /// An optional name for this component. + public ZeroCrossingRate(Pipeline pipeline, string name = nameof(ZeroCrossingRate)) + : base(pipeline, name) { } diff --git a/Sources/Audio/Microsoft.Psi.Audio/Operators.cs b/Sources/Audio/Microsoft.Psi.Audio/Operators.cs index c1a1afa37..a1819bd94 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/Operators.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/Operators.cs @@ -21,11 +21,10 @@ public static class Operators /// A stream containing the input audio. /// The output frame size in bytes. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream containing the reframed audio. - public static IProducer Reframe(this IProducer source, int frameSizeInBytes, DeliveryPolicy deliveryPolicy = null) - { - return source.PipeTo(new Reframe(source.Out.Pipeline, frameSizeInBytes), deliveryPolicy); - } + public static IProducer Reframe(this IProducer source, int frameSizeInBytes, DeliveryPolicy deliveryPolicy = null, string name = nameof(Reframe)) + => source.PipeTo(new Reframe(source.Out.Pipeline, frameSizeInBytes, name), deliveryPolicy); /// /// Reframes the bytes in an stream, producing a new @@ -34,22 +33,20 @@ public static IProducer Reframe(this IProducer source, /// A stream containing the input audio. /// The output frame duration. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream containing the reframed audio. - public static IProducer Reframe(this IProducer source, TimeSpan frameDuration, DeliveryPolicy deliveryPolicy = null) - { - return source.PipeTo(new Reframe(source.Out.Pipeline, frameDuration), deliveryPolicy); - } + public static IProducer Reframe(this IProducer source, TimeSpan frameDuration, DeliveryPolicy deliveryPolicy = null, string name = nameof(Reframe)) + => source.PipeTo(new Reframe(source.Out.Pipeline, frameDuration, name), deliveryPolicy); /// /// Transforms an stream to a stream of byte arrays containing the raw audio. /// /// A stream of audio buffers. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of byte arrays containing the raw audio. - public static IProducer ToByteArray(this IProducer source, DeliveryPolicy deliveryPolicy = null) - { - return source.Select(x => x.Data, deliveryPolicy); - } + public static IProducer ToByteArray(this IProducer source, DeliveryPolicy deliveryPolicy = null, string name = nameof(ToByteArray)) + => source.Select(x => x.Data, deliveryPolicy, name); /// /// Transforms a stream of byte arrays containing raw audio to an stream. @@ -57,11 +54,10 @@ public static IProducer ToByteArray(this IProducer source, /// A stream of raw audio byte arrays. /// The audio format of the raw audio contained within the byte arrays. /// /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of audio buffers. - public static IProducer ToAudioBuffer(this IProducer source, WaveFormat audioFormat, DeliveryPolicy deliveryPolicy = null) - { - return source.Select(x => new AudioBuffer(x, audioFormat), deliveryPolicy); - } + public static IProducer ToAudioBuffer(this IProducer source, WaveFormat audioFormat, DeliveryPolicy deliveryPolicy = null, string name = nameof(ToAudioBuffer)) + => source.Select(x => new AudioBuffer(x, audioFormat), deliveryPolicy, name); /// /// The frame shift operator. @@ -71,11 +67,10 @@ public static IProducer ToAudioBuffer(this IProducer source /// The number of bytes by which to shift the data. /// The sampling frequency in bytes per second. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream containing the frame-shifted data. - public static IProducer FrameShift(this IProducer source, int frameSizeInBytes, int frameShiftInBytes, double bytesPerSec, DeliveryPolicy deliveryPolicy = null) - { - return source.PipeTo(new FrameShift(source.Out.Pipeline, frameSizeInBytes, frameShiftInBytes, bytesPerSec), deliveryPolicy); - } + public static IProducer FrameShift(this IProducer source, int frameSizeInBytes, int frameShiftInBytes, double bytesPerSec, DeliveryPolicy deliveryPolicy = null, string name = nameof(FrameShift)) + => source.PipeTo(new FrameShift(source.Out.Pipeline, frameSizeInBytes, frameShiftInBytes, bytesPerSec, name), deliveryPolicy); /// /// Converts a stream of audio data to a stream of floating point values. @@ -83,11 +78,10 @@ public static IProducer FrameShift(this IProducer source, int fr /// A stream containing the input audio data. /// The audio format of the input audio. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of floating point audio sample values. - public static IProducer ToFloat(this IProducer source, WaveFormat format, DeliveryPolicy deliveryPolicy = null) - { - return source.PipeTo(new ToFloat(source.Out.Pipeline, format), deliveryPolicy); - } + public static IProducer ToFloat(this IProducer source, WaveFormat format, DeliveryPolicy deliveryPolicy = null, string name = nameof(ToFloat)) + => source.PipeTo(new ToFloat(source.Out.Pipeline, format, name), deliveryPolicy); /// /// Applies dithering to input sample values. @@ -96,11 +90,12 @@ public static IProducer ToFloat(this IProducer source, WaveForm /// The scale factor of the dither. /// An initial random seed value. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of floating point sample values with dithering. - public static IProducer Dither(this IProducer source, float scaleFactor, int randomSeed = 0, DeliveryPolicy deliveryPolicy = null) + public static IProducer Dither(this IProducer source, float scaleFactor, int randomSeed = 0, DeliveryPolicy deliveryPolicy = null, string name = nameof(Dither)) { float[] dithered = null; - Random random = new Random(randomSeed); + var random = new Random(randomSeed); return source.Select( values => { @@ -116,7 +111,8 @@ public static IProducer Dither(this IProducer source, float sc return dithered; }, - deliveryPolicy); + deliveryPolicy, + name); } /// @@ -125,11 +121,10 @@ public static IProducer Dither(this IProducer source, float sc /// A stream of floating point input sample values. /// The Hanning window length. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of floating point sample values with Hanning window applied. - public static IProducer HanningWindow(this IProducer source, int kernelLength, DeliveryPolicy deliveryPolicy = null) - { - return source.Select(new HanningWindow(kernelLength).Apply, deliveryPolicy); - } + public static IProducer HanningWindow(this IProducer source, int kernelLength, DeliveryPolicy deliveryPolicy = null, string name = nameof(HanningWindow)) + => source.Select(new HanningWindow(kernelLength).Apply, deliveryPolicy, name); /// /// Performs a Fast Fourier Transform on input sample buffers. @@ -138,44 +133,40 @@ public static IProducer HanningWindow(this IProducer source, i /// The FFT size. /// The window size. /// An optional delivery policy. + /// An optional name for this stream operator. /// A stream of FFTs of the input sample buffers. - public static IProducer FFT(this IProducer source, int fftSize, int inputSize, DeliveryPolicy deliveryPolicy = null) - { - return source.PipeTo(new FFT(source.Out.Pipeline, fftSize, inputSize), deliveryPolicy); - } + public static IProducer FFT(this IProducer source, int fftSize, int inputSize, DeliveryPolicy deliveryPolicy = null, string name = nameof(FFT)) + => source.PipeTo(new FFT(source.Out.Pipeline, fftSize, inputSize, name), deliveryPolicy); /// /// Converts a stream of FFTs to FFT power spectra. /// /// A stream of FFTs. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of FFT power spectra. - public static IProducer FFTPower(this IProducer source, DeliveryPolicy deliveryPolicy = null) - { - return source.PipeTo(new FFTPower(source.Out.Pipeline), deliveryPolicy); - } + public static IProducer FFTPower(this IProducer source, DeliveryPolicy deliveryPolicy = null, string name = nameof(FFTPower)) + => source.PipeTo(new FFTPower(source.Out.Pipeline, name), deliveryPolicy); /// /// Computes the log energy of a stream of input samples in the time domain. /// /// A stream of floating point input sample values. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of log energy values for the input samples. - public static IProducer LogEnergy(this IProducer source, DeliveryPolicy deliveryPolicy = null) - { - return source.PipeTo(new LogEnergy(source.Out.Pipeline), deliveryPolicy); - } + public static IProducer LogEnergy(this IProducer source, DeliveryPolicy deliveryPolicy = null, string name = nameof(LogEnergy)) + => source.PipeTo(new LogEnergy(source.Out.Pipeline, name), deliveryPolicy); /// /// Computes the zero-crossing rate of input samples. /// /// A stream of floating point input sample values. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of zero-crossing rates for the input samples. - public static IProducer ZeroCrossingRate(this IProducer source, DeliveryPolicy deliveryPolicy = null) - { - return source.PipeTo(new ZeroCrossingRate(source.Out.Pipeline), deliveryPolicy); - } + public static IProducer ZeroCrossingRate(this IProducer source, DeliveryPolicy deliveryPolicy = null, string name = nameof(ZeroCrossingRate)) + => source.PipeTo(new ZeroCrossingRate(source.Out.Pipeline, name), deliveryPolicy); /// /// Computes the frequency domain energy from the FFT power spectra. @@ -184,11 +175,10 @@ public static IProducer ZeroCrossingRate(this IProducer source, /// The index of the starting frequency of the band. /// The index of the ending frequency of the band. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of frequency domain energy values. - public static IProducer FrequencyDomainEnergy(this IProducer source, int start, int end, DeliveryPolicy deliveryPolicy = null) - { - return source.PipeTo(new FrequencyDomainEnergy(source.Out.Pipeline, start, end), deliveryPolicy); - } + public static IProducer FrequencyDomainEnergy(this IProducer source, int start, int end, DeliveryPolicy deliveryPolicy = null, string name = nameof(FrequencyDomainEnergy)) + => source.PipeTo(new FrequencyDomainEnergy(source.Out.Pipeline, start, end, name), deliveryPolicy); /// /// Computes the spectral entropy within a frequency band. @@ -197,10 +187,9 @@ public static IProducer FrequencyDomainEnergy(this IProducer sou /// The starting frequency of the band. /// The ending frequency of the band. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of spectral entropy values. - public static IProducer SpectralEntropy(this IProducer source, int start, int end, DeliveryPolicy deliveryPolicy = null) - { - return source.PipeTo(new SpectralEntropy(source.Out.Pipeline, start, end), deliveryPolicy); - } + public static IProducer SpectralEntropy(this IProducer source, int start, int end, DeliveryPolicy deliveryPolicy = null, string name = nameof(SpectralEntropy)) + => source.PipeTo(new SpectralEntropy(source.Out.Pipeline, start, end, name), deliveryPolicy); } } diff --git a/Sources/Audio/Microsoft.Psi.Audio/Reframe.cs b/Sources/Audio/Microsoft.Psi.Audio/Reframe.cs index 379f1fc1c..b0d177203 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/Reframe.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/Reframe.cs @@ -22,8 +22,9 @@ public sealed class Reframe : ConsumerProducer /// /// The pipeline to add the component to. /// The output frame size in bytes. - public Reframe(Pipeline pipeline, int frameSizeInBytes) - : base(pipeline) + /// An optional name for this component. + public Reframe(Pipeline pipeline, int frameSizeInBytes, string name = nameof(Reframe)) + : base(pipeline, name) { if (frameSizeInBytes <= 0) { @@ -38,8 +39,9 @@ public Reframe(Pipeline pipeline, int frameSizeInBytes) /// /// The pipeline to add the component to. /// The output frame duration. - public Reframe(Pipeline pipeline, TimeSpan frameDuration) - : base(pipeline) + /// An optional name for this component. + public Reframe(Pipeline pipeline, TimeSpan frameDuration, string name = nameof(Reframe)) + : base(pipeline, name) { if (frameDuration <= TimeSpan.Zero) { diff --git a/Sources/Audio/Microsoft.Psi.Audio/WaveFileAudioSource.cs b/Sources/Audio/Microsoft.Psi.Audio/WaveFileAudioSource.cs index d8beb93d1..dcec3947d 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/WaveFileAudioSource.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/WaveFileAudioSource.cs @@ -12,23 +12,31 @@ namespace Microsoft.Psi.Audio /// public sealed class WaveFileAudioSource : IProducer { + private readonly string name; + /// /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - /// The path name of the WAVE file. + /// The path name of the WAVE file. /// Indicates a time to use for the start of the audio source. If null, the current time will be used. /// The size of each data buffer to post, determined by the amount of audio data it can hold. - public WaveFileAudioSource(Pipeline pipeline, string filename, DateTime? audioStartTime = null, int audioBufferSizeMs = 20) + /// An optional name for this component. + public WaveFileAudioSource(Pipeline pipeline, string path, DateTime? audioStartTime = null, int audioBufferSizeMs = 20, string name = nameof(WaveFileAudioSource)) { - var name = Path.GetFileName(filename); - var path = Path.GetDirectoryName(filename); - var importer = WaveFileStore.Open(pipeline, name, path, audioStartTime ?? DateTime.UtcNow, audioBufferSizeMs); + this.name = name; + + var filename = Path.GetFileName(path); + var directoryName = Path.GetDirectoryName(path); + var importer = WaveFileStore.Open(pipeline, filename, directoryName, audioStartTime ?? DateTime.UtcNow, audioBufferSizeMs); var audio = importer.OpenStream(WaveFileStreamReader.AudioStreamName); this.Out = audio.Out; } /// public Emitter Out { get; private set; } + + /// + public override string ToString() => this.name; } } diff --git a/Sources/Audio/Microsoft.Psi.Audio/WaveFileStore.cs b/Sources/Audio/Microsoft.Psi.Audio/WaveFileStore.cs index 32015ed3e..fb6064336 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/WaveFileStore.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/WaveFileStore.cs @@ -4,11 +4,7 @@ namespace Microsoft.Psi.Audio { using System; - using System.Collections.Generic; - using System.IO; - using System.Threading; using Microsoft.Psi; - using Microsoft.Psi.Data; /// /// Provides static methods to access WAVE file stores. @@ -26,8 +22,6 @@ public static class WaveFileStore /// The size of each data buffer in milliseconds. /// A instance. public static WaveFileImporter Open(Pipeline pipeline, string name, string path, DateTime startTime, int audioBufferSizeMs = WaveFileStreamReader.DefaultAudioBufferSizeMs) - { - return new WaveFileImporter(pipeline, name, path, startTime, audioBufferSizeMs); - } + => new (pipeline, name, path, startTime, audioBufferSizeMs); } } diff --git a/Sources/Audio/Microsoft.Psi.Audio/WaveFileWriter.cs b/Sources/Audio/Microsoft.Psi.Audio/WaveFileWriter.cs index 08f9800ec..c66b4a4fa 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/WaveFileWriter.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/WaveFileWriter.cs @@ -13,16 +13,17 @@ namespace Microsoft.Psi.Audio /// public sealed class WaveFileWriter : SimpleConsumer, IDisposable { + private readonly string outputFilename; private WaveDataWriterClass writer; - private string outputFilename; /// /// Initializes a new instance of the class. /// /// The pipeline to add the component to. /// The path name of the Wave file. - public WaveFileWriter(Pipeline pipeline, string filename) - : base(pipeline) + /// An optional name for this component. + public WaveFileWriter(Pipeline pipeline, string filename, string name = nameof(WaveFileWriter)) + : base(pipeline, name) { this.outputFilename = filename; } diff --git a/Sources/Audio/Microsoft.Psi.Audio/WaveStreamSampleSource.cs b/Sources/Audio/Microsoft.Psi.Audio/WaveStreamSampleSource.cs index b9fe35266..da79ccefa 100644 --- a/Sources/Audio/Microsoft.Psi.Audio/WaveStreamSampleSource.cs +++ b/Sources/Audio/Microsoft.Psi.Audio/WaveStreamSampleSource.cs @@ -18,6 +18,7 @@ namespace Microsoft.Psi.Audio public class WaveStreamSampleSource : IConsumerProducer { private readonly Pipeline pipeline; + private readonly string name; private readonly AudioBuffer[] audioData; /// @@ -25,9 +26,11 @@ public class WaveStreamSampleSource : IConsumerProducer /// /// The pipeline to add the component to. /// Audio stream in WAVE format (48KHz, 1-channel, IEEE Float). - public WaveStreamSampleSource(Pipeline pipeline, Stream stream) + /// An optional name for this component. + public WaveStreamSampleSource(Pipeline pipeline, Stream stream, string name = nameof(WaveStreamSampleSource)) { this.pipeline = pipeline; + this.name = name; this.In = pipeline.CreateReceiver(this, this.Play, nameof(this.In)); this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); diff --git a/Sources/Calibration/Microsoft.Psi.Calibration/CalibrationExtensions.cs b/Sources/Calibration/Microsoft.Psi.Calibration/CalibrationExtensions.cs index e0053bc88..654240236 100644 --- a/Sources/Calibration/Microsoft.Psi.Calibration/CalibrationExtensions.cs +++ b/Sources/Calibration/Microsoft.Psi.Calibration/CalibrationExtensions.cs @@ -59,6 +59,7 @@ public static class CalibrationExtensions pi = 0; cameraMatrix[0, 0] = computedParameters[pi++]; // fx cameraMatrix[1, 1] = computedParameters[pi++]; // fy + cameraMatrix[2, 2] = 1; cameraMatrix[0, 2] = computedParameters[pi++]; // cx cameraMatrix[1, 2] = computedParameters[pi++]; // cy distortionCoefficients[0] = computedParameters[pi++]; // k1 @@ -127,6 +128,7 @@ public static class CalibrationExtensions pi = 0; cameraMatrix[0, 0] = computedParameters[pi++]; // fx cameraMatrix[1, 1] = computedParameters[pi++]; // fy + cameraMatrix[2, 2] = 1; cameraMatrix[0, 2] = computedParameters[pi++]; // cx cameraMatrix[1, 2] = computedParameters[pi++]; // cy distortionCoefficients[0] = computedParameters[pi++]; // k1 @@ -188,7 +190,7 @@ public static ICameraIntrinsics CreateCameraIntrinsics(this ImageBase image, dou public static Point3D? ProjectToCameraSpace(IDepthDeviceCalibrationInfo depthDeviceCalibrationInfo, Point2D point2D, Shared depthImage) { var colorExtrinsicsInverse = depthDeviceCalibrationInfo.ColorPose; - var pointInCameraSpace = depthDeviceCalibrationInfo.ColorIntrinsics.GetCameraSpacePosition(point2D, 1.0, true); + var pointInCameraSpace = depthDeviceCalibrationInfo.ColorIntrinsics.GetCameraSpacePosition(point2D, 1.0, depthImage.Resource.DepthValueSemantics, true); double x = pointInCameraSpace.X * colorExtrinsicsInverse[0, 0] + pointInCameraSpace.Y * colorExtrinsicsInverse[0, 1] + pointInCameraSpace.Z * colorExtrinsicsInverse[0, 2] + colorExtrinsicsInverse[0, 3]; double y = pointInCameraSpace.X * colorExtrinsicsInverse[1, 0] + pointInCameraSpace.Y * colorExtrinsicsInverse[1, 1] + pointInCameraSpace.Z * colorExtrinsicsInverse[1, 2] + colorExtrinsicsInverse[1, 3]; double z = pointInCameraSpace.X * colorExtrinsicsInverse[2, 0] + pointInCameraSpace.Y * colorExtrinsicsInverse[2, 1] + pointInCameraSpace.Z * colorExtrinsicsInverse[2, 2] + colorExtrinsicsInverse[2, 3]; @@ -203,14 +205,13 @@ public static ICameraIntrinsics CreateCameraIntrinsics(this ImageBase image, dou /// /// Tuple of depth image, list of points to project, and calibration information. /// An optional delivery policy. + /// An optional name for the stream operator. /// Returns a producer that generates a list of corresponding 3D points in Kinect camera space. public static IProducer> ProjectTo3D( - this IProducer<(Shared, List, IDepthDeviceCalibrationInfo)> source, DeliveryPolicy<(Shared, List, IDepthDeviceCalibrationInfo)> deliveryPolicy = null) - { - var projectTo3D = new ProjectTo3D(source.Out.Pipeline); - source.PipeTo(projectTo3D, deliveryPolicy); - return projectTo3D; - } + this IProducer<(Shared, List, IDepthDeviceCalibrationInfo)> source, + DeliveryPolicy<(Shared, List, IDepthDeviceCalibrationInfo)> deliveryPolicy = null, + string name = nameof(ProjectTo3D)) + => source.PipeTo(new ProjectTo3D(source.Out.Pipeline, name), deliveryPolicy); /// /// Performs a ray/mesh intersection with the depth map. @@ -302,7 +303,7 @@ public static Matrix AxisAngleToMatrix(Vector vectorRotation) /// /// Input rotation matrix. /// Same rotation in axis-angle representation (L2-Norm of the vector represents angular distance). - /// An optional angle epsilon parameter used to determine when the specified matrix contains a zero-rotation. + /// An optional angle epsilon parameter used to determine when the specified matrix contains a zero-rotation (by default 0.01 degrees). public static Vector MatrixToAxisAngle(Matrix m, double epsilon = 0.01 * Math.PI / 180) { if (m.RowCount != 3 || m.ColumnCount != 3) @@ -478,6 +479,7 @@ Vector OptimizationFunction(Vector p) var k = Matrix.Build.DenseIdentity(3, 3); k[0, 0] = p[pi++]; // fx k[1, 1] = p[pi++]; // fy + k[2, 2] = 1; k[0, 2] = p[pi++]; // cx k[1, 2] = p[pi++]; // cy diff --git a/Sources/Calibration/Microsoft.Psi.Calibration/CameraIntrinsics.cs b/Sources/Calibration/Microsoft.Psi.Calibration/CameraIntrinsics.cs index 4ce80c235..56d035089 100644 --- a/Sources/Calibration/Microsoft.Psi.Calibration/CameraIntrinsics.cs +++ b/Sources/Calibration/Microsoft.Psi.Calibration/CameraIntrinsics.cs @@ -7,6 +7,7 @@ namespace Microsoft.Psi.Calibration using System.Runtime.Serialization; using MathNet.Numerics.LinearAlgebra; using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Imaging; /// /// CameraIntrinsics defines the intrinsic properties for a given camera. @@ -18,9 +19,6 @@ public class CameraIntrinsics : ICameraIntrinsics, IEquatable [OptionalField] private bool closedFormDistorts; - [OptionalField] - private DepthPixelSemantics depthPixelSemantics; - /// /// Initializes a new instance of the class. /// @@ -30,15 +28,13 @@ public class CameraIntrinsics : ICameraIntrinsics, IEquatable /// The radial distortion parameters (up to 6). /// The tangential distortion parameters (up to 2). /// Indicates which direction the closed form equation for Brown-Conrady Distortion model goes. I.e. does it perform distortion or undistortion. Default is to distort (thus making projection simpler and unprojection more complicated). - /// Defines how depth pixel values should be interpreted. public CameraIntrinsics( int imageWidth, int imageHeight, Matrix transform, Vector radialDistortion = null, Vector tangentialDistortion = null, - bool closedFormDistorts = true, - DepthPixelSemantics depthPixelSemantics = DepthPixelSemantics.DistanceToPlane) + bool closedFormDistorts = true) { this.ImageWidth = imageWidth; this.ImageHeight = imageHeight; @@ -64,7 +60,6 @@ public class CameraIntrinsics : ICameraIntrinsics, IEquatable this.FocalLengthXY = new Point2D(this.Transform[0, 0], this.Transform[1, 1]); this.PrincipalPoint = new Point2D(this.Transform[0, 2], this.Transform[1, 2]); this.ClosedFormDistorts = closedFormDistorts; - this.depthPixelSemantics = depthPixelSemantics; } /// @@ -114,22 +109,6 @@ private set } } - /// - /// Gets pixel semantics. - /// - public DepthPixelSemantics DepthPixelSemantics - { - get - { - return this.depthPixelSemantics; - } - - private set - { - this.depthPixelSemantics = value; - } - } - /// public int ImageWidth { get; private set; } @@ -183,12 +162,12 @@ private set public bool TryGetPixelPosition(Point3D point3D, bool distort, out Point2D pixelPosition, bool nullIfOutsideFieldOfView = true) { var point2D = this.GetPixelPosition(point3D, distort, nullIfOutsideFieldOfView); - pixelPosition = point2D.HasValue ? point2D.Value : default; + pixelPosition = point2D ?? default; return point2D.HasValue; } /// - public Point3D GetCameraSpacePosition(Point2D point2D, double depth, bool undistort) + public Point3D GetCameraSpacePosition(Point2D point2D, double depth, DepthValueSemantics depthValueSemantics, bool undistort) { // Convert from pixel coordinates to NDC var tmp = new Point3D(point2D.X, point2D.Y, 1.0); @@ -201,7 +180,7 @@ public Point3D GetCameraSpacePosition(Point2D point2D, double depth, bool undist this.TryUndistortPoint(pixelPoint2D, out pixelPoint2D); } - if (this.depthPixelSemantics == DepthPixelSemantics.DistanceToPoint) + if (depthValueSemantics == DepthValueSemantics.DistanceToPoint) { double norm = Math.Sqrt(pixelPoint2D.X * pixelPoint2D.X + pixelPoint2D.Y * pixelPoint2D.Y + 1); depth /= norm; @@ -234,7 +213,7 @@ public bool TryDistortPoint(Point2D undistortedPt, out Point2D distortedPt) } /// - public Point3D[,] GetPixelToCameraSpaceMapping(bool undistort) + public Point3D[,] GetPixelToCameraSpaceMapping(DepthValueSemantics depthValueSemantics, bool undistort) { var result = new Point3D[this.ImageWidth, this.ImageHeight]; for (int i = 0; i < this.ImageWidth; i++) @@ -252,7 +231,7 @@ public bool TryDistortPoint(Point2D undistortedPt, out Point2D distortedPt) this.TryUndistortPoint(pixelPoint2D, out pixelPoint2D); } - if (this.depthPixelSemantics == DepthPixelSemantics.DistanceToPoint) + if (depthValueSemantics == DepthValueSemantics.DistanceToPoint) { double norm = Math.Sqrt(pixelPoint2D.X * pixelPoint2D.X + pixelPoint2D.Y * pixelPoint2D.Y + 1); result[i, j] = new Point3D(1 / norm, -pixelPoint2D.X / norm, -pixelPoint2D.Y / norm); @@ -271,7 +250,6 @@ public bool TryDistortPoint(Point2D undistortedPt, out Point2D distortedPt) public override int GetHashCode() { var hashCode = default(HashCode); - hashCode.Add(this.depthPixelSemantics); hashCode.Add(this.ClosedFormDistorts); hashCode.Add(this.FocalLengthXY); hashCode.Add(this.ImageHeight); @@ -289,7 +267,6 @@ public override int GetHashCode() /// public bool Equals(ICameraIntrinsics other) => other is CameraIntrinsics cameraIntrinsics && - Equals(this.depthPixelSemantics, cameraIntrinsics.depthPixelSemantics) && Equals(this.ClosedFormDistorts, cameraIntrinsics.ClosedFormDistorts) && Equals(this.FocalLengthXY, cameraIntrinsics.FocalLengthXY) && Equals(this.ImageHeight, cameraIntrinsics.ImageHeight) && diff --git a/Sources/Calibration/Microsoft.Psi.Calibration/ICameraIntrinsics.cs b/Sources/Calibration/Microsoft.Psi.Calibration/ICameraIntrinsics.cs index ac660f29c..660d47fca 100644 --- a/Sources/Calibration/Microsoft.Psi.Calibration/ICameraIntrinsics.cs +++ b/Sources/Calibration/Microsoft.Psi.Calibration/ICameraIntrinsics.cs @@ -6,6 +6,7 @@ namespace Microsoft.Psi.Calibration using System; using MathNet.Numerics.LinearAlgebra; using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Imaging; /// /// ICameraIntrinsics defines our interface for specifying the intrinsics @@ -93,20 +94,22 @@ public interface ICameraIntrinsics : IEquatable /// /// The pixel position. /// The depth along the specified pixel position. + /// How depth values should be interpreted. /// Indicates whether to apply undistortion. /// Point in 3D space, assuming MathNet basis (Forward=X, Left=Y, Up=Z). - Point3D GetCameraSpacePosition(Point2D point2D, double depth, bool undistort); + Point3D GetCameraSpacePosition(Point2D point2D, double depth, DepthValueSemantics depthValueSemantics, bool undistort); /// /// Gets a mapping matrix that can be used to transform pixels into 3D space. /// + /// How depth values should be interpreted. /// Indicates whether to apply undistortion. /// /// A matrix of 3D points that can be used to transform depth values at a specified pixel /// into 3D space. To use this matrix simply piecewise multiply the depth value by the X /// Y and Z dimensions of the in the matrix at the location indexed /// by the pixel. - public Point3D[,] GetPixelToCameraSpaceMapping(bool undistort); + Point3D[,] GetPixelToCameraSpaceMapping(DepthValueSemantics depthValueSemantics, bool undistort); /// /// Applies the distortion model to a point in the camera post-projection coordinates. diff --git a/Sources/Calibration/Microsoft.Psi.Calibration/ProjectTo3D.cs b/Sources/Calibration/Microsoft.Psi.Calibration/ProjectTo3D.cs index a16ecd40e..b2b24539a 100644 --- a/Sources/Calibration/Microsoft.Psi.Calibration/ProjectTo3D.cs +++ b/Sources/Calibration/Microsoft.Psi.Calibration/ProjectTo3D.cs @@ -21,8 +21,9 @@ public sealed class ProjectTo3D : ConsumerProducer<(Shared, List class. /// /// The pipeline to add the component to. - public ProjectTo3D(Pipeline pipeline) - : base(pipeline) + /// An optional name for the component. + public ProjectTo3D(Pipeline pipeline, string name = nameof(ProjectTo3D)) + : base(pipeline, name) { } diff --git a/Sources/Data/Microsoft.Psi.Data/Annotations/AnnotationSchema.cs b/Sources/Data/Microsoft.Psi.Data/Annotations/AnnotationSchema.cs index 5264f0b6a..b6bfc3d01 100644 --- a/Sources/Data/Microsoft.Psi.Data/Annotations/AnnotationSchema.cs +++ b/Sources/Data/Microsoft.Psi.Data/Annotations/AnnotationSchema.cs @@ -50,23 +50,37 @@ public AnnotationSchema(string name) /// Loads an annotation schema from disk. /// /// The full path and filename of the annotation schema to load. - /// The requested annotation schema if it exists, otherwise null. - public static AnnotationSchema Load(string fileName) + /// The requested annotation schema. + public static AnnotationSchema LoadFrom(string fileName) { + using var streamReader = new StreamReader(fileName); + return LoadFrom(streamReader); + } + + /// + /// Tries to load an annotation schema from disk. + /// + /// The full path and filename of the annotation schema to load. + /// The loaded annotation schema if successful. + /// True if the annotation schema is loaded successfully, otherwise null. + public static bool TryLoadFrom(string fileName, out AnnotationSchema annotationSchema) + { + annotationSchema = null; if (!File.Exists(fileName)) { - return null; + return false; } try { - using var streamReader = new StreamReader(fileName); - return Load(streamReader); + annotationSchema = LoadFrom(fileName); } catch (Exception) { - return null; + return false; } + + return true; } /// @@ -177,7 +191,7 @@ public void Save(string fileName) } } - private static AnnotationSchema Load(StreamReader streamReader) + private static AnnotationSchema LoadFrom(StreamReader streamReader) { var reader = new JsonTextReader(streamReader); var serializer = JsonSerializer.Create(JsonSerializerSettings); diff --git a/Sources/Data/Microsoft.Psi.Data/Annotations/Operators.cs b/Sources/Data/Microsoft.Psi.Data/Annotations/Operators.cs index a78ddd03a..087fc3078 100644 --- a/Sources/Data/Microsoft.Psi.Data/Annotations/Operators.cs +++ b/Sources/Data/Microsoft.Psi.Data/Annotations/Operators.cs @@ -20,15 +20,42 @@ public static class Operators /// The source stream. /// A function that, given a key, produces a track name and set of attribute values for the annotation. /// An optional delivery policy. + /// An optional name for the stream operator. /// A time interval annotation stream. public static IProducer ToTimeIntervalAnnotations( this IProducer> source, Func AttributeValues)> annotationConstructor, - DeliveryPolicy> deliveryPolicy = null) + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(ToTimeIntervalAnnotations)) + => source.ToTimeIntervalAnnotations( + dict => dict.Where(kvp => kvp.Value).ToDictionary(kvp => kvp.Key, kvp => kvp.Value), + (k, _) => + { + var (track, attributeValues) = annotationConstructor(k); + return (true, track, attributeValues); + }, + deliveryPolicy, + name); + + /// + /// Converts a stream of dictionaries with boolean values into a corresponding stream of time interval annotations. + /// + /// The type of key in the source stream. + /// The source stream. + /// A function that, given a key, produces a value indicating whether to create an annotation, a track name and set of attribute values for the annotation. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A time interval annotation stream. + public static IProducer ToTimeIntervalAnnotations( + this IProducer> source, + Func AttributeValues)> annotationConstructor, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(ToTimeIntervalAnnotations)) => source.ToTimeIntervalAnnotations( dict => dict.Where(kvp => kvp.Value).ToDictionary(kvp => kvp.Key, kvp => kvp.Value), (k, _) => annotationConstructor(k), - deliveryPolicy); + deliveryPolicy, + name); /// /// Converts a stream of dictionaries into a corresponding stream of time interval annotations. @@ -38,15 +65,43 @@ public static class Operators /// The source stream. /// A function that, given a key and value, produces a track name and set of attribute values for the annotation. /// An optional delivery policy. + /// An optional name for the stream operator. /// A time interval annotation stream. public static IProducer ToTimeIntervalAnnotations( this IProducer> source, Func AttributeValues)> annotationConstructor, - DeliveryPolicy> deliveryPolicy = null) + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(ToTimeIntervalAnnotations)) + => source.ToTimeIntervalAnnotations( + _ => _, + (k, v) => + { + var (track, attributeValues) = annotationConstructor(k, v); + return (true, track, attributeValues); + }, + deliveryPolicy, + name); + + /// + /// Converts a stream of dictionaries into a corresponding stream of time interval annotations. + /// + /// The type of key in the source stream. + /// The type of values in the source stream. + /// The source stream. + /// A function that, given a key and value, produces a value indicating whether to create an annotation, a track name and set of attribute values for the annotation. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A time interval annotation stream. + public static IProducer ToTimeIntervalAnnotations( + this IProducer> source, + Func AttributeValues)> annotationConstructor, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(ToTimeIntervalAnnotations)) => source.ToTimeIntervalAnnotations( _ => _, annotationConstructor, - deliveryPolicy); + deliveryPolicy, + name); /// /// Converts a stream into a corresponding stream of time interval annotations. @@ -56,14 +111,16 @@ public static class Operators /// The type of values in the source stream. /// The source stream. /// A function that, given an input message produces a dictionary of key-values that generates the annotation set. - /// A function that, given a key and value, produces a track name and set of attribute values for the annotation. + /// A function that, given a key and value, produces a value indicating whether to create an annotation, a track name, and set of attribute values for the annotation. /// An optional delivery policy. + /// An optional name for the stream operator. /// A time interval annotation stream. private static IProducer ToTimeIntervalAnnotations( this IProducer source, Func> selector, - Func AttributeValues)> annotationConstructor, - DeliveryPolicy deliveryPolicy = null) + Func AttributeValues)> annotationConstructor, + DeliveryPolicy deliveryPolicy = null, + string name = nameof(ToTimeIntervalAnnotations)) { var intervals = new Dictionary(); @@ -96,15 +153,18 @@ public static class Operators { // In this case we need to post the object removeKeys.Add(key); - (var annotationTrack, var attributeValues) = annotationConstructor(key, intervals[key].Value); - var annotation = new TimeIntervalAnnotation(intervals[key].TimeInterval, annotationTrack, attributeValues); - if (timeIntervalAnnotationSet == null) + (var create, var annotationTrack, var attributeValues) = annotationConstructor(key, intervals[key].Value); + if (create) { - timeIntervalAnnotationSet = new TimeIntervalAnnotationSet(annotation); - } - else - { - timeIntervalAnnotationSet.AddAnnotation(annotation); + var annotation = new TimeIntervalAnnotation(intervals[key].TimeInterval, annotationTrack, attributeValues); + if (timeIntervalAnnotationSet == null) + { + timeIntervalAnnotationSet = new TimeIntervalAnnotationSet(annotation); + } + else + { + timeIntervalAnnotationSet.AddAnnotation(annotation); + } } } } @@ -133,22 +193,29 @@ public static class Operators var newInterval = new TimeInterval(intervals[key].TimeInterval.Left, closingTime); // Append to the annotation set - (var annotationTrack, var attributeValues) = annotationConstructor(key, intervals[key].Value); - var annotation = new TimeIntervalAnnotation(newInterval, annotationTrack, attributeValues); - if (timeIntervalAnnotationSet == null) + (var create, var annotationTrack, var attributeValues) = annotationConstructor(key, intervals[key].Value); + if (create) { - timeIntervalAnnotationSet = new TimeIntervalAnnotationSet(annotation); - } - else - { - timeIntervalAnnotationSet.AddAnnotation(annotation); + var annotation = new TimeIntervalAnnotation(newInterval, annotationTrack, attributeValues); + if (timeIntervalAnnotationSet == null) + { + timeIntervalAnnotationSet = new TimeIntervalAnnotationSet(annotation); + } + else + { + timeIntervalAnnotationSet.AddAnnotation(annotation); + } } } // Post the value - emitter.Post(timeIntervalAnnotationSet, closingTime); + if (timeIntervalAnnotationSet != null) + { + emitter.Post(timeIntervalAnnotationSet, closingTime); + } } - }); + }, + name: name); return source.PipeTo(processor, deliveryPolicy); } diff --git a/Sources/Data/Microsoft.Psi.Data/Dataset.cs b/Sources/Data/Microsoft.Psi.Data/Dataset.cs index f821d515c..c23d8b961 100644 --- a/Sources/Data/Microsoft.Psi.Data/Dataset.cs +++ b/Sources/Data/Microsoft.Psi.Data/Dataset.cs @@ -231,6 +231,18 @@ public void Save() this.HasUnsavedChanges = false; } + /// + /// Creates and adds an empty session with the specified name to the dataset. + /// + /// The session name. + /// The session. + public Session AddSession(string sessionName) + { + var session = new Session(this, sessionName); + this.AddSession(session); + return session; + } + /// /// Creates and adds a session to this dataset using the specified parameters. /// diff --git a/Sources/Data/Microsoft.Psi.Data/Json/JsonGenerator.cs b/Sources/Data/Microsoft.Psi.Data/Json/JsonGenerator.cs index 8fcffc12d..9d5193b6e 100644 --- a/Sources/Data/Microsoft.Psi.Data/Json/JsonGenerator.cs +++ b/Sources/Data/Microsoft.Psi.Data/Json/JsonGenerator.cs @@ -10,7 +10,7 @@ namespace Microsoft.Psi.Data.Json using Newtonsoft.Json.Linq; /// - /// Defines a component that plays back data from a JSON store. + /// Component that plays back data from a JSON store. /// public class JsonGenerator : Generator, IDisposable { @@ -23,10 +23,11 @@ public class JsonGenerator : Generator, IDisposable /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - /// The name of the application that generated the persisted files, or the root name of the files. - /// The directory in which the main persisted file resides. - public JsonGenerator(Pipeline pipeline, string name, string path) - : this(pipeline, new JsonStoreReader(name, path)) + /// The name of the application that generated the persisted files, or the root name of the files. + /// The directory in which the main persisted file resides. + /// An optional name for the component. + public JsonGenerator(Pipeline pipeline, string storeName, string storePath, string name = nameof(JsonGenerator)) + : this(pipeline, new JsonStoreReader(storeName, storePath), name) { } @@ -35,8 +36,9 @@ public JsonGenerator(Pipeline pipeline, string name, string path) /// /// The pipeline to add the component to. /// The underlying store reader. - protected JsonGenerator(Pipeline pipeline, JsonStoreReader reader) - : base(pipeline) + /// An optional name for the component. + protected JsonGenerator(Pipeline pipeline, JsonStoreReader reader, string name = nameof(JsonGenerator)) + : base(pipeline, name: name) { this.pipeline = pipeline; this.reader = reader; diff --git a/Sources/Filters/Microsoft.Psi.Filters/MathNetFilters.cs b/Sources/Filters/Microsoft.Psi.Filters/MathNetFilters.cs index b4f0d9dcf..f6100c39e 100644 --- a/Sources/Filters/Microsoft.Psi.Filters/MathNetFilters.cs +++ b/Sources/Filters/Microsoft.Psi.Filters/MathNetFilters.cs @@ -262,7 +262,8 @@ public static IProducer DenoiseFilter(this IProducer input, int { // Sample the input stream at the desired rate and process through the given filter. var clock = Generators.Repeat(input.Out.Pipeline, 0, TimeSpan.FromSeconds(1.0 / sampleRate), alignmentDateTime); - return input.Interpolate(clock, sampleInterpolator, sourceDeliveryPolicy: deliveryPolicy, clockDeliveryPolicy: DeliveryPolicy.Unlimited) + return input + .Interpolate(clock, sampleInterpolator, sourceDeliveryPolicy: deliveryPolicy, clockDeliveryPolicy: DeliveryPolicy.Unlimited) .Select(s => filter.ProcessSample(s), DeliveryPolicy.SynchronousOrThrottle); } } diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Linux/DepthImageCompressor.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Linux/DepthImageCompressor.cs index 9ef29fbed..00538f05b 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging.Linux/DepthImageCompressor.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Linux/DepthImageCompressor.cs @@ -46,7 +46,11 @@ public void Serialize(BufferWriter writer, DepthImage depthImage, SerializationC { if (this.encoder != null) { - using var sharedEncodedDepthImage = EncodedDepthImagePool.GetOrCreate(depthImage.Width, depthImage.Height); + using var sharedEncodedDepthImage = EncodedDepthImagePool.GetOrCreate( + depthImage.Width, + depthImage.Height, + depthImage.DepthValueSemantics, + depthImage.DepthValueToMetersScaleFactor); sharedEncodedDepthImage.Resource.EncodeFrom(depthImage, this.encoder); Serializer.Serialize(writer, sharedEncodedDepthImage, context); } @@ -61,7 +65,11 @@ public void Deserialize(BufferReader reader, ref DepthImage depthImage, Serializ { Shared sharedEncodedDepthImage = null; Serializer.Deserialize(reader, ref sharedEncodedDepthImage, context); - using var sharedDepthImage = DepthImagePool.GetOrCreate(sharedEncodedDepthImage.Resource.Width, sharedEncodedDepthImage.Resource.Height); + using var sharedDepthImage = DepthImagePool.GetOrCreate( + sharedEncodedDepthImage.Resource.Width, + sharedEncodedDepthImage.Resource.Height, + sharedEncodedDepthImage.Resource.DepthValueSemantics, + sharedEncodedDepthImage.Resource.DepthValueToMetersScaleFactor); sharedDepthImage.Resource.DecodeFrom(sharedEncodedDepthImage.Resource, this.decoder); depthImage = sharedDepthImage.Resource.DeepClone(); sharedEncodedDepthImage.Dispose(); diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Linux/DepthImageToPngStreamEncoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Linux/DepthImageToPngStreamEncoder.cs index 991df2ef7..dce6643f0 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging.Linux/DepthImageToPngStreamEncoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Linux/DepthImageToPngStreamEncoder.cs @@ -3,7 +3,6 @@ namespace Microsoft.Psi.Imaging { - using System; using System.IO; using SkiaSharp; @@ -12,6 +11,9 @@ namespace Microsoft.Psi.Imaging /// public class DepthImageToPngStreamEncoder : IDepthImageToStreamEncoder { + /// + public string Description => "Png"; + /// public void EncodeToStream(DepthImage depthImage, Stream stream) { diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageFromBitmapStreamDecoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageFromBitmapStreamDecoder.cs new file mode 100644 index 000000000..c80733e06 --- /dev/null +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageFromBitmapStreamDecoder.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Imaging +{ + using System.IO; + using System.Runtime.InteropServices; + using SkiaSharp; + + /// + /// Implements a general bitmap image decoder. + /// + /// Internally uses SkiaSharp.SKBitmap.Decode(). + public class ImageFromBitmapStreamDecoder : IImageFromStreamDecoder + { + /// + public void DecodeFromStream(Stream stream, Image image) + { + // decode JPEG, PNG, ... + var decoded = SKBitmap.Decode(stream); + Marshal.Copy(decoded.Bytes, 0, image.ImageData, decoded.ByteCount); + } + + /// + public PixelFormat GetPixelFormat(Stream stream) + { + var decoded = SKBitmap.Decode(stream); + return decoded.ColorType switch + { + SKColorType.Bgra8888 => PixelFormat.BGRA_32bpp, + SKColorType.Gray8 => PixelFormat.Gray_8bpp, + _ => PixelFormat.Undefined, + }; + } + } +} \ No newline at end of file diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageFromStreamDecoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageFromStreamDecoder.cs index f4be85153..e663e78ad 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageFromStreamDecoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageFromStreamDecoder.cs @@ -4,31 +4,49 @@ namespace Microsoft.Psi.Imaging { using System.IO; - using System.Runtime.InteropServices; - using SkiaSharp; /// /// Implements an image decoder. /// public class ImageFromStreamDecoder : IImageFromStreamDecoder { + private static readonly ImageFromGZipStreamDecoder GzipDecoder = new (); + private static readonly ImageFromNV12StreamDecoder Nv12Decoder = new (); + private static readonly ImageFromBitmapStreamDecoder BitmapDecoder = new (); + /// public void DecodeFromStream(Stream stream, Image image) { - var decoded = SKBitmap.Decode(stream); - Marshal.Copy(decoded.Bytes, 0, image.ImageData, decoded.ByteCount); + if (GzipDecoder.HasGZipHeader(stream)) + { + GzipDecoder.DecodeFromStream(stream, image); + return; + } + + if (Nv12Decoder.HasNV12Header(stream)) + { + Nv12Decoder.DecodeFromStream(stream, image); + return; + } + + // default to decode JPEG, PNG, ... + BitmapDecoder.DecodeFromStream(stream, image); } /// public PixelFormat GetPixelFormat(Stream stream) { - var decoded = SKBitmap.Decode(stream); - return decoded.ColorType switch + if (GzipDecoder.HasGZipHeader(stream)) + { + return GzipDecoder.GetPixelFormat(stream); + } + + if (Nv12Decoder.HasNV12Header(stream)) { - SKColorType.Bgra8888 => PixelFormat.BGRA_32bpp, - SKColorType.Gray8 => PixelFormat.Gray_8bpp, - _ => PixelFormat.Undefined, - }; + return Nv12Decoder.GetPixelFormat(stream); + } + + return BitmapDecoder.GetPixelFormat(stream); } } } \ No newline at end of file diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageToJpegStreamEncoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageToJpegStreamEncoder.cs index 9fdf5c11f..9abdaa2be 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageToJpegStreamEncoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageToJpegStreamEncoder.cs @@ -11,6 +11,9 @@ namespace Microsoft.Psi.Imaging /// public class ImageToJpegStreamEncoder : IImageToStreamEncoder { + /// + public string Description => $"Jpeg({this.QualityLevel})"; + /// /// Gets or sets JPEG image quality (0-100). /// diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageToPngStreamEncoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageToPngStreamEncoder.cs index 3294c4605..c8147b862 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageToPngStreamEncoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImageToPngStreamEncoder.cs @@ -11,6 +11,9 @@ namespace Microsoft.Psi.Imaging /// public class ImageToPngStreamEncoder : IImageToStreamEncoder { + /// + public string Description => "Png"; + /// public void EncodeToStream(Image image, Stream stream) { diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImagingOperators.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImagingOperators.cs index 8eed4256e..9cdf2110c 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImagingOperators.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Linux/ImagingOperators.cs @@ -17,55 +17,66 @@ public static partial class ImagingOperators /// A producer of images to encode. /// JPEG quality to use. /// An optional delivery policy. + /// An optional name for the stream operator. /// A producer that generates the JPEG images. - public static IProducer> EncodeJpeg(this IProducer> source, int quality = 90, DeliveryPolicy> deliveryPolicy = null) - { - return source.Encode(new ImageToJpegStreamEncoder { QualityLevel = quality }, deliveryPolicy); - } + public static IProducer> EncodeJpeg( + this IProducer> source, + int quality = 90, + DeliveryPolicy> deliveryPolicy = null, + string name = null) + => source.Encode(new ImageToJpegStreamEncoder { QualityLevel = quality }, deliveryPolicy, name ?? $"{nameof(EncodeJpeg)}({quality})"); /// /// Encodes an image to a PNG format. /// /// A producer of images to encoder. /// An optional delivery policy. + /// An optional name for the stream operator. /// A producer that generates the PNG images. - public static IProducer> EncodePng(this IProducer> source, DeliveryPolicy> deliveryPolicy = null) - { - return source.Encode(new ImageToPngStreamEncoder(), deliveryPolicy); - } + public static IProducer> EncodePng( + this IProducer> source, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(EncodePng)) + => source.Encode(new ImageToPngStreamEncoder(), deliveryPolicy, name); /// /// Decodes an encoded image. /// /// A producer of encoded images to decode. /// An optional delivery policy. + /// An optional name for the stream operator. /// A producer that generates the decoded images. - public static IProducer> Decode(this IProducer> source, DeliveryPolicy> deliveryPolicy = null) - { - return source.Decode(new ImageFromStreamDecoder(), deliveryPolicy); - } + public static IProducer> Decode( + this IProducer> source, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(Decode)) + => source.Decode(new ImageFromStreamDecoder(), deliveryPolicy, name); /// /// Encodes a depth image to a PNG format. /// /// A producer of depth images to encode. /// An optional delivery policy. + /// An optional name for the stream operator. /// A producer that generates the PNG-encoded depth images. - public static IProducer> EncodePng(this IProducer> source, DeliveryPolicy> deliveryPolicy = null) - { - return source.Encode(new DepthImageToPngStreamEncoder(), deliveryPolicy); - } + public static IProducer> EncodePng( + this IProducer> source, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(EncodePng)) + => source.Encode(new DepthImageToPngStreamEncoder(), deliveryPolicy, name); /// /// Decodes an encoded depth image. /// /// A producer of encoded depth images to decode. /// An optional delivery policy. + /// An optional name for the stream operator. /// A producer that generates the decoded depth images. - public static IProducer> Decode(this IProducer> source, DeliveryPolicy> deliveryPolicy = null) - { - return source.Decode(new DepthImageFromStreamDecoder(), deliveryPolicy); - } + public static IProducer> Decode( + this IProducer> source, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(Decode)) + => source.Decode(new DepthImageFromStreamDecoder(), deliveryPolicy, name); /// /// Converts an image to a SkiaSharp SKImage. diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/DepthImageCompressor.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/DepthImageCompressor.cs index 16acca6f1..84d6a4d20 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/DepthImageCompressor.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/DepthImageCompressor.cs @@ -47,7 +47,10 @@ public void Serialize(BufferWriter writer, DepthImage depthImage, SerializationC if (this.encoder != null) { using var sharedEncodedDepthImage = EncodedDepthImagePool.GetOrCreate( - depthImage.Width, depthImage.Height); + depthImage.Width, + depthImage.Height, + depthImage.DepthValueSemantics, + depthImage.DepthValueToMetersScaleFactor); sharedEncodedDepthImage.Resource.EncodeFrom(depthImage, this.encoder); Serializer.Serialize(writer, sharedEncodedDepthImage, context); } @@ -62,7 +65,11 @@ public void Deserialize(BufferReader reader, ref DepthImage depthImage, Serializ { Shared sharedEncodedDepthImage = null; Serializer.Deserialize(reader, ref sharedEncodedDepthImage, context); - using var sharedDepthImage = DepthImagePool.GetOrCreate(sharedEncodedDepthImage.Resource.Width, sharedEncodedDepthImage.Resource.Height); + using var sharedDepthImage = DepthImagePool.GetOrCreate( + sharedEncodedDepthImage.Resource.Width, + sharedEncodedDepthImage.Resource.Height, + sharedEncodedDepthImage.Resource.DepthValueSemantics, + sharedEncodedDepthImage.Resource.DepthValueToMetersScaleFactor); sharedDepthImage.Resource.DecodeFrom(sharedEncodedDepthImage.Resource, this.decoder); depthImage = sharedDepthImage.Resource.DeepClone(); sharedEncodedDepthImage.Dispose(); diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/DepthImageToPngStreamEncoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/DepthImageToPngStreamEncoder.cs index cc45b495e..d34facb40 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/DepthImageToPngStreamEncoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/DepthImageToPngStreamEncoder.cs @@ -11,6 +11,9 @@ namespace Microsoft.Psi.Imaging /// public class DepthImageToPngStreamEncoder : IDepthImageToStreamEncoder { + /// + public string Description => "Png"; + /// public void EncodeToStream(DepthImage depthImage, Stream stream) { diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/DepthImageToTiffStreamEncoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/DepthImageToTiffStreamEncoder.cs index e1eab1a52..1c254ba6f 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/DepthImageToTiffStreamEncoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/DepthImageToTiffStreamEncoder.cs @@ -11,6 +11,9 @@ namespace Microsoft.Psi.Imaging /// public class DepthImageToTiffStreamEncoder : IDepthImageToStreamEncoder { + /// + public string Description => "Tiff"; + /// public void EncodeToStream(DepthImage depthImage, Stream stream) { diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageFromBitmapStreamDecoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageFromBitmapStreamDecoder.cs new file mode 100644 index 000000000..c6effe0e0 --- /dev/null +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageFromBitmapStreamDecoder.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Imaging +{ + using System.IO; + using System.Windows; + using System.Windows.Media.Imaging; + + /// + /// Implements a general bitmap image decoder. + /// + /// Internally uses System.Windows.Media.Imaging.BitmapDecoder. + public class ImageFromBitmapStreamDecoder : IImageFromStreamDecoder + { + /// + public void DecodeFromStream(Stream stream, Image image) + { + // decode JPEG, PNG, ... + var decoder = BitmapDecoder.Create(stream, BitmapCreateOptions.PreservePixelFormat, BitmapCacheOption.Default); + BitmapSource bitmapSource = decoder.Frames[0]; + var fmt = bitmapSource.Format.ToPixelFormat(); + if (fmt != image.PixelFormat) + { + using var img = ImagePool.GetOrCreate(image.Width, image.Height, fmt); + bitmapSource.CopyPixels(Int32Rect.Empty, img.Resource.ImageData, img.Resource.Stride * img.Resource.Height, img.Resource.Stride); + img.Resource.CopyTo(image); + } + else + { + bitmapSource.CopyPixels(Int32Rect.Empty, image.ImageData, image.Stride * image.Height, image.Stride); + } + } + + /// + public PixelFormat GetPixelFormat(Stream stream) + { + var decoder = BitmapDecoder.Create(stream, BitmapCreateOptions.PreservePixelFormat, BitmapCacheOption.Default); + BitmapSource bitmapSource = decoder.Frames[0]; + return bitmapSource.Format.ToPixelFormat(); + } + } +} \ No newline at end of file diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageFromStreamDecoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageFromStreamDecoder.cs index 68d480efd..fa51ed6eb 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageFromStreamDecoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageFromStreamDecoder.cs @@ -1,59 +1,52 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace Microsoft.Psi.Imaging -{ - using System.IO; - using System.IO.Compression; - using System.Windows; - using System.Windows.Media.Imaging; - - /// - /// Implements an image decoder. - /// - public class ImageFromStreamDecoder : IImageFromStreamDecoder - { - /// - public void DecodeFromStream(Stream stream, Image image) - { - // GZip indentified by 1f8b header (see section 2.3.1 of RFC 1952 https://www.ietf.org/rfc/rfc1952.txt) - if (stream.Length >= 2 && stream.ReadByte() == 0x1f && stream.ReadByte() == 0x8b) +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Imaging +{ + using System.IO; + + /// + /// Implements an image decoder. + /// + public class ImageFromStreamDecoder : IImageFromStreamDecoder + { + private static readonly ImageFromGZipStreamDecoder GzipDecoder = new (); + private static readonly ImageFromNV12StreamDecoder Nv12Decoder = new (); + private static readonly ImageFromBitmapStreamDecoder BitmapDecoder = new (); + + /// + public void DecodeFromStream(Stream stream, Image image) + { + if (GzipDecoder.HasGZipHeader(stream)) { - // decode GZip - stream.Position = 0; // advanced by if (... stream.ReadByte() ...) above - var size = image.Stride * image.Height; - using var decompressor = new GZipStream(stream, CompressionMode.Decompress); - unsafe - { - decompressor.CopyTo(new UnmanagedMemoryStream((byte*)image.ImageData.ToPointer(), size, size, FileAccess.ReadWrite)); - } - } - else + GzipDecoder.DecodeFromStream(stream, image); + return; + } + + if (Nv12Decoder.HasNV12Header(stream)) { - // decode JPEG, PNG, ... - stream.Position = 0; // advanced by if (... stream.ReadByte() ...) above - var decoder = BitmapDecoder.Create(stream, BitmapCreateOptions.PreservePixelFormat, BitmapCacheOption.Default); - BitmapSource bitmapSource = decoder.Frames[0]; - var fmt = bitmapSource.Format.ToPixelFormat(); - if (fmt != image.PixelFormat) - { - using var img = Microsoft.Psi.Imaging.ImagePool.GetOrCreate(image.Width, image.Height, fmt); - bitmapSource.CopyPixels(Int32Rect.Empty, img.Resource.ImageData, img.Resource.Stride * img.Resource.Height, img.Resource.Stride); - img.Resource.CopyTo(image); - } - else - { - bitmapSource.CopyPixels(Int32Rect.Empty, image.ImageData, image.Stride * image.Height, image.Stride); - } - } - } - - /// - public PixelFormat GetPixelFormat(Stream stream) - { - var decoder = BitmapDecoder.Create(stream, BitmapCreateOptions.PreservePixelFormat, BitmapCacheOption.Default); - BitmapSource bitmapSource = decoder.Frames[0]; - return bitmapSource.Format.ToPixelFormat(); - } - } + Nv12Decoder.DecodeFromStream(stream, image); + return; + } + + // default to decode JPEG, PNG, ... + BitmapDecoder.DecodeFromStream(stream, image); + } + + /// + public PixelFormat GetPixelFormat(Stream stream) + { + if (GzipDecoder.HasGZipHeader(stream)) + { + return GzipDecoder.GetPixelFormat(stream); + } + + if (Nv12Decoder.HasNV12Header(stream)) + { + return Nv12Decoder.GetPixelFormat(stream); + } + + return BitmapDecoder.GetPixelFormat(stream); + } + } } \ No newline at end of file diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageToGZipStreamEncoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageToGZipStreamEncoder.cs index c18546c8c..ca19a9f9f 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageToGZipStreamEncoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageToGZipStreamEncoder.cs @@ -11,6 +11,9 @@ namespace Microsoft.Psi.Imaging /// public class ImageToGZipStreamEncoder : IImageToStreamEncoder { + /// + public string Description => "GZip"; + /// public void EncodeToStream(Image image, Stream stream) { diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageToJpegStreamEncoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageToJpegStreamEncoder.cs index b918736ae..76945688e 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageToJpegStreamEncoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageToJpegStreamEncoder.cs @@ -11,6 +11,9 @@ namespace Microsoft.Psi.Imaging /// public class ImageToJpegStreamEncoder : IImageToStreamEncoder { + /// + public string Description => $"Jpeg({this.QualityLevel})"; + /// /// Gets or sets JPEG image quality (0-100). /// diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageToPngStreamEncoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageToPngStreamEncoder.cs index 353138f76..82d73998a 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageToPngStreamEncoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImageToPngStreamEncoder.cs @@ -11,6 +11,9 @@ namespace Microsoft.Psi.Imaging /// public class ImageToPngStreamEncoder : IImageToStreamEncoder { + /// + public string Description => $"Png"; + /// public void EncodeToStream(Image image, Stream stream) { diff --git a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImagingOperators.cs b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImagingOperators.cs index 895c1c3cf..be9f29297 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImagingOperators.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging.Windows/ImagingOperators.cs @@ -14,76 +14,91 @@ public static partial class ImagingOperators /// A producer of images to encode. /// JPEG quality to use. /// An optional delivery policy. + /// An optional name for the stream operator. /// A producer that generates the JPEG images. - public static IProducer> EncodeJpeg(this IProducer> source, int quality = 90, DeliveryPolicy> deliveryPolicy = null) - { - return source.Encode(new ImageToJpegStreamEncoder { QualityLevel = quality }, deliveryPolicy); - } + public static IProducer> EncodeJpeg( + this IProducer> source, + int quality = 90, + DeliveryPolicy> deliveryPolicy = null, + string name = null) + => source.Encode(new ImageToJpegStreamEncoder { QualityLevel = quality }, deliveryPolicy, name ?? $"{nameof(EncodeJpeg)}({quality})"); /// /// Encodes an image to a PNG format. /// /// A producer of images to encode. /// An optional delivery policy. + /// An optional name for the stream operator. /// A producer that generates the PNG images. - public static IProducer> EncodePng(this IProducer> source, DeliveryPolicy> deliveryPolicy = null) - { - return source.Encode(new ImageToPngStreamEncoder(), deliveryPolicy); - } + public static IProducer> EncodePng( + this IProducer> source, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(EncodePng)) + => source.Encode(new ImageToPngStreamEncoder(), deliveryPolicy, name); /// /// Encodes an image to a GZIP format. /// /// A producer of images to encode. /// An optional delivery policy. + /// An optional name for the stream operator. /// A producer that generates the GZipped images. - public static IProducer> EncodeGZip(this IProducer> source, DeliveryPolicy> deliveryPolicy = null) - { - return source.Encode(new ImageToGZipStreamEncoder(), deliveryPolicy); - } + public static IProducer> EncodeGZip( + this IProducer> source, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(EncodeGZip)) + => source.Encode(new ImageToGZipStreamEncoder(), deliveryPolicy, name); /// /// Decodes an encoded image. /// /// A producer of encoded images to decode. /// An optional delivery policy. + /// An optional name for the stream operator. /// A producer that generates the decoded images. - public static IProducer> Decode(this IProducer> source, DeliveryPolicy> deliveryPolicy = null) - { - return source.Decode(new ImageFromStreamDecoder(), deliveryPolicy); - } + public static IProducer> Decode( + this IProducer> source, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(Decode)) + => source.Decode(new ImageFromStreamDecoder(), deliveryPolicy, name); /// /// Encodes a depth image to a PNG format. /// /// A producer of depth images to encode. /// An optional delivery policy. + /// An optional name for the stream operator. /// A producer that generates the PNG-encoded depth images. - public static IProducer> EncodePng(this IProducer> source, DeliveryPolicy> deliveryPolicy = null) - { - return source.Encode(new DepthImageToPngStreamEncoder(), deliveryPolicy); - } + public static IProducer> EncodePng( + this IProducer> source, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(EncodePng)) + => source.Encode(new DepthImageToPngStreamEncoder(), deliveryPolicy, name); /// /// Encodes a depth image to a TIFF format. /// /// A producer of depth images to encode. /// An optional delivery policy. + /// An optional name for the stream operator. /// A producer that generates the TIFF-encoded depth images. - public static IProducer> EncodeTiff(this IProducer> source, DeliveryPolicy> deliveryPolicy = null) - { - return source.Encode(new DepthImageToTiffStreamEncoder(), deliveryPolicy); - } + public static IProducer> EncodeTiff( + this IProducer> source, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(EncodeTiff)) + => source.Encode(new DepthImageToTiffStreamEncoder(), deliveryPolicy, name); /// /// Decodes an encoded depth image. /// /// A producer of encoded depth images to decode. /// An optional delivery policy. + /// An optional name for the stream operator. /// A producer that generates the decoded depth images. - public static IProducer> Decode(this IProducer> source, DeliveryPolicy> deliveryPolicy = null) - { - return source.Decode(new DepthImageFromStreamDecoder(), deliveryPolicy); - } + public static IProducer> Decode( + this IProducer> source, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(Decode)) + => source.Decode(new DepthImageFromStreamDecoder(), deliveryPolicy, name); } } \ No newline at end of file diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/DepthImage.cs b/Sources/Imaging/Microsoft.Psi.Imaging/DepthImage.cs index 54b6288a5..8f5d5f42d 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/DepthImage.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/DepthImage.cs @@ -1,332 +1,374 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace Microsoft.Psi.Imaging -{ - using System; - using System.Drawing; +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Imaging +{ + using System; + using System.Collections.Generic; + using System.Drawing; using System.Drawing.Imaging; - using Microsoft.Psi.Common; - using Microsoft.Psi.Serialization; - - /// - /// Represents a depth image, stored in unmanaged memory. - /// - /// Using this class it is possible as to allocate a new depth image in unmanaged memory, - /// as to just wrap provided pointer to unmanaged memory, where an image is stored. - [Serializer(typeof(DepthImage.CustomSerializer))] - public class DepthImage : ImageBase - { - /// - /// Initializes a new instance of the class. - /// - /// The unmanaged array containing the image. - /// Depth image width in pixels. - /// Depth image height in pixels. - /// Depth image stride (line size in bytes). - /// Using this constructor, make sure all specified image attributes are correct - /// and correspond to unmanaged memory buffer. If some attributes are specified incorrectly, - /// this may lead to exceptions working with the unmanaged memory. - public DepthImage(UnmanagedBuffer unmanagedBuffer, int width, int height, int stride) - : base(unmanagedBuffer, width, height, stride, PixelFormat.Gray_16bpp) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// Pointer to image data in unmanaged memory. - /// Depth image width in pixels. - /// Depth image height in pixels. - /// Depth image stride (line size in bytes). - /// Using this constructor, make sure all specified image attributes are correct - /// and correspond to unmanaged memory buffer. If some attributes are specified incorrectly, - /// this may lead to exceptions working with the unmanaged memory. - public DepthImage(IntPtr imageData, int width, int height, int stride) - : base(imageData, width, height, stride, PixelFormat.Gray_16bpp) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// Depth image width in pixels. - /// Depth image height in pixels. - public DepthImage(int width, int height) - : base(width, height, PixelFormat.Gray_16bpp) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// Depth image width in pixels. - /// Depth image height in pixels. - /// Depth image stride (line size in bytes). - /// Using this constructor, make sure all specified image attributes are correct - /// and correspond to unmanaged memory buffer. If some attributes are specified incorrectly, - /// this may lead to exceptions working with the unmanaged memory. - public DepthImage(int width, int height, int stride) - : base(width, height, stride, PixelFormat.Gray_16bpp) - { - } - - /// - /// Initializes a new instance of the class. - /// - /// Locked bitmap data. - /// Indicates whether a copy is made (default is false). - /// - /// When the parameter is false (default), the depth image simply wraps - /// the bitmap data. As such, the bitmap data must stay locked for the duration of using the object. - /// - /// If the parameter is set to true, a copy of the bitmap - /// data is made, and the bitmap data can be released right after the has been constructed. - /// - /// - public DepthImage(BitmapData bitmapData, bool makeCopy = false) - : base(bitmapData, makeCopy) - { - CheckPixelFormat(bitmapData.PixelFormat); - } - - /// - /// Create a new from a specified bitmap. - /// - /// A bitmap to create the depth image from. - /// A new depth image, which contains a copy of the specified bitmap. - public static DepthImage CreateFrom(Bitmap bitmap) - { - CheckPixelFormat(bitmap.PixelFormat); - - DepthImage depthImage = null; - BitmapData sourceData = bitmap.LockBits( - new Rectangle(0, 0, bitmap.Width, bitmap.Height), - ImageLockMode.ReadOnly, - bitmap.PixelFormat); - - try - { - depthImage = new DepthImage(sourceData, true); - } - finally - { - bitmap.UnlockBits(sourceData); - } - - return depthImage; - } - - /// - /// Copies the depth image contents from a specified source locked bitmap data. - /// - /// Source locked bitmap data. - /// The method copies data from the specified bitmap into the depth image. - /// The depth image must be allocated and must have the same size as the specified - /// bitmap data. - public void CopyFrom(BitmapData bitmapData) - { - CheckPixelFormat(bitmapData.PixelFormat); - int numBytes = bitmapData.Height * bitmapData.Stride; + using System.Runtime.Serialization; + using Microsoft.Psi.Common; + using Microsoft.Psi.Serialization; + + /// + /// Represents a depth image, stored in unmanaged memory. + /// + /// Using this class it is possible as to allocate a new depth image in unmanaged memory, + /// as to just wrap provided pointer to unmanaged memory, where an image is stored. + [Serializer(typeof(CustomSerializer))] + public class DepthImage : ImageBase, IDepthImage + { + [OptionalField] + private DepthValueSemantics? depthValueSemantics; + + [OptionalField] + private double? depthValueToMetersScaleFactor; + + /// + /// Initializes a new instance of the class. + /// + /// The unmanaged array containing the image. + /// Depth image width in pixels. + /// Depth image height in pixels. + /// Depth image stride (line size in bytes). + /// Optional depth value semantics. + /// Optional scale factor to convert from depth values to meters. + /// Using this constructor, make sure all specified image attributes are correct + /// and correspond to unmanaged memory buffer. If some attributes are specified incorrectly, + /// this may lead to exceptions working with the unmanaged memory. + public DepthImage(UnmanagedBuffer unmanagedBuffer, int width, int height, int stride, DepthValueSemantics depthValueSemantics = DepthValueSemantics.DistanceToPlane, double depthValueToMetersScaleFactor = 0.001) + : base(unmanagedBuffer, width, height, stride, PixelFormat.Gray_16bpp) + { + this.depthValueSemantics = depthValueSemantics; + this.depthValueToMetersScaleFactor = depthValueToMetersScaleFactor; + } + + /// + /// Initializes a new instance of the class. + /// + /// Pointer to image data in unmanaged memory. + /// Depth image width in pixels. + /// Depth image height in pixels. + /// Depth image stride (line size in bytes). + /// Optional depth value semantics. + /// Optional scale factor to convert from depth values to meters. + /// Using this constructor, make sure all specified image attributes are correct + /// and correspond to unmanaged memory buffer. If some attributes are specified incorrectly, + /// this may lead to exceptions working with the unmanaged memory. + public DepthImage(IntPtr imageData, int width, int height, int stride, DepthValueSemantics depthValueSemantics = DepthValueSemantics.DistanceToPlane, double depthValueToMetersScaleFactor = 0.001) + : base(imageData, width, height, stride, PixelFormat.Gray_16bpp) + { + this.depthValueSemantics = depthValueSemantics; + this.depthValueToMetersScaleFactor = depthValueToMetersScaleFactor; + } + + /// + /// Initializes a new instance of the class. + /// + /// Depth image width in pixels. + /// Depth image height in pixels. + /// Optional depth image semantics. + /// Optional scale factor to convert from depth values to meters. + public DepthImage(int width, int height, DepthValueSemantics depthValueSemantics = DepthValueSemantics.DistanceToPlane, double depthValueToMetersScaleFactor = 0.001) + : base(width, height, PixelFormat.Gray_16bpp) + { + this.depthValueSemantics = depthValueSemantics; + this.depthValueToMetersScaleFactor = depthValueToMetersScaleFactor; + } + + /// + /// Initializes a new instance of the class. + /// + /// Depth image width in pixels. + /// Depth image height in pixels. + /// Depth image stride (line size in bytes). + /// Optional depth value semantics. + /// Optional scale factor to convert from depth values to meters. + /// Using this constructor, make sure all specified image attributes are correct + /// and correspond to unmanaged memory buffer. If some attributes are specified incorrectly, + /// this may lead to exceptions working with the unmanaged memory. + public DepthImage(int width, int height, int stride, DepthValueSemantics depthValueSemantics = DepthValueSemantics.DistanceToPlane, double depthValueToMetersScaleFactor = 0.001) + : base(width, height, stride, PixelFormat.Gray_16bpp) + { + this.depthValueSemantics = depthValueSemantics; + this.depthValueToMetersScaleFactor = depthValueToMetersScaleFactor; + } + + /// + /// Initializes a new instance of the class. + /// + /// Locked bitmap data. + /// Optional depth value semantics. + /// Optional scale factor to convert from depth values to meters. + /// Indicates whether a copy is made (default is false). + /// + /// When the parameter is false (default), the depth image simply wraps + /// the bitmap data. As such, the bitmap data must stay locked for the duration of using the object. + /// + /// If the parameter is set to true, a copy of the bitmap + /// data is made, and the bitmap data can be released right after the has been constructed. + /// + /// + public DepthImage(BitmapData bitmapData, DepthValueSemantics depthValueSemantics = DepthValueSemantics.DistanceToPlane, double depthValueToMetersScaleFactor = 0.001, bool makeCopy = false) + : base(bitmapData, makeCopy) + { + CheckPixelFormat(bitmapData.PixelFormat); + this.depthValueSemantics = depthValueSemantics; + this.depthValueToMetersScaleFactor = depthValueToMetersScaleFactor; + } + + /// + public DepthValueSemantics DepthValueSemantics => this.depthValueSemantics ?? DepthValueSemantics.DistanceToPlane; + + /// + public double DepthValueToMetersScaleFactor => this.depthValueToMetersScaleFactor ?? 0.001; + + /// + /// Create a new from a specified bitmap. + /// + /// A bitmap to create the depth image from. + /// Optional depth value semantics. + /// Optional scale factor to convert from depth values to meters. + /// A new depth image, which contains a copy of the specified bitmap. + public static DepthImage CreateFrom(Bitmap bitmap, DepthValueSemantics depthValueSemantics = DepthValueSemantics.DistanceToPlane, double depthValueToMetersScaleFactor = 0.001) + { + CheckPixelFormat(bitmap.PixelFormat); + + DepthImage depthImage = null; + BitmapData sourceData = bitmap.LockBits( + new Rectangle(0, 0, bitmap.Width, bitmap.Height), + ImageLockMode.ReadOnly, + bitmap.PixelFormat); + + try + { + depthImage = new DepthImage(sourceData, depthValueSemantics, depthValueToMetersScaleFactor, true); + } + finally + { + bitmap.UnlockBits(sourceData); + } + + return depthImage; + } + + /// + /// Copies the depth image contents from a specified source locked bitmap data. + /// + /// Source locked bitmap data. + /// The method copies data from the specified bitmap into the depth image. + /// The depth image must be allocated and must have the same size as the specified + /// bitmap data. + public void CopyFrom(BitmapData bitmapData) + { + CheckPixelFormat(bitmapData.PixelFormat); + int numBytes = bitmapData.Height * bitmapData.Stride; if (numBytes > this.UnmanagedBuffer.Size) { throw new InvalidOperationException("Buffer too small."); - } - - this.UnmanagedBuffer.CopyFrom(bitmapData.Scan0, numBytes); - } - - /// - /// Copies the depth image contents from a specified bitmap. - /// - /// A bitmap to copy from. - /// The method copies data from the specified bitmap into the image. - /// The image must be allocated and must have the same size. - public void CopyFrom(Bitmap bitmap) - { - BitmapData bitmapData = bitmap.LockBits( - new Rectangle(0, 0, this.Width, this.Height), - ImageLockMode.ReadWrite, - PixelFormatHelper.ToSystemPixelFormat(this.PixelFormat)); - try + } + + this.UnmanagedBuffer.CopyFrom(bitmapData.Scan0, numBytes); + } + + /// + /// Copies the depth image contents from a specified bitmap. + /// + /// A bitmap to copy from. + /// The method copies data from the specified bitmap into the image. + /// The image must be allocated and must have the same size. + public void CopyFrom(Bitmap bitmap) + { + BitmapData bitmapData = bitmap.LockBits( + new Rectangle(0, 0, this.Width, this.Height), + ImageLockMode.ReadWrite, + PixelFormatHelper.ToSystemPixelFormat(this.PixelFormat)); + try { int numBytes = bitmapData.Height * bitmapData.Stride; if (numBytes > this.UnmanagedBuffer.Size) { throw new InvalidOperationException("Buffer too small."); - } - - this.UnmanagedBuffer.CopyFrom(bitmapData.Scan0, numBytes); - } - finally - { - bitmap.UnlockBits(bitmapData); - } - } - - /// - /// Copies the depth image from a specified source depth image of the same size. - /// - /// Source depth image to copy the depth image from. - /// The method copies the current depth image from the specified source depth image. - /// The size of the images must be the same. - public void CopyFrom(DepthImage source) - { - source.CopyTo(this); - } - - /// - /// Copies the depth image from a specified source image of the same size and format. - /// - /// Source image to copy the depth image from. - /// The method copies the current depth image from the specified source image. - /// The size of the images must be the same, and the source image must have format. - public void CopyFrom(Image source) - { - source.CopyTo(this); - } - - /// - /// Decodes a specified encoded depth image with a specified decoder into the current depth image. - /// - /// The encoded depth image to decode. - /// The depth image decoder to use. - /// The depth image width, height and pixel format must match. The method should not be called concurrently. - public void DecodeFrom(EncodedDepthImage encodedDepthImage, IDepthImageFromStreamDecoder depthImageDecoder) - { - if (encodedDepthImage.Width != this.Width || encodedDepthImage.Height != this.Height || encodedDepthImage.PixelFormat != this.PixelFormat) - { - throw new InvalidOperationException("Cannot decode from an encoded depth image that has a different width, height, or pixel format."); - } - - depthImageDecoder.DecodeFromStream(encodedDepthImage.ToStream(), this); - } - - /// - /// Encodes the depth image using a specified encoder. - /// - /// The depth image encoder to use. - /// A new, corresponding encoded depth image. - public EncodedDepthImage Encode(IDepthImageToStreamEncoder depthImageEncoder) - { - var encodedDepthImage = new EncodedDepthImage(this.Width, this.Height); - encodedDepthImage.EncodeFrom(this, depthImageEncoder); - return encodedDepthImage; - } - - /// - /// Copies the depth image into a target depth image of the same size. - /// - /// Target depth image to copy this depth image to. - /// The method copies the current depth image into the specified depth image. - /// The size of the images must be the same. - public void CopyTo(DepthImage target) - { - this.CopyTo(target.ImageData, target.Width, target.Height, target.Stride, target.PixelFormat); - } - - /// - /// Copies the depth image into a target image of the same size. - /// - /// Target image to copy this depth image to. - /// The method copies the current depth image into the specified image. - /// The size of the images must be the same. The method implements a translation of pixel formats. - public void CopyTo(Image target) - { - this.CopyTo(target.ImageData, target.Width, target.Height, target.Stride, target.PixelFormat); - } - - /// - /// Sets a pixel in the depth image. - /// - /// Pixel's X coordinate. - /// Pixel's Y coordinate. - /// Gray value to set pixel to. - public void SetPixel(int x, int y, ushort gray) - { - if (x < 0 || x >= this.Width) - { - throw new ArgumentException("X coordinate is outside bounds", nameof(x)); - } - - if (y < 0 || y >= this.Height) - { - throw new ArgumentException("Y coordinate is outside bounds", nameof(y)); - } - - unsafe - { - byte* src = (byte*)this.ImageData.ToPointer(); - int pixelOffset = x * this.BitsPerPixel / 8 + y * this.Stride; - *(ushort*)(src + pixelOffset) = gray; - } - } - - /// - /// Gets the value of a pixel in the depth image. - /// - /// Pixel's X coordinate. - /// Pixel's Y coordinate. - /// The value of the pixel at the specified coordinates. - public ushort GetPixel(int x, int y) - { - if (x < 0 || x >= this.Width) - { - throw new ArgumentException("X coordinate is outside bounds", nameof(x)); - } - - if (y < 0 || y >= this.Height) - { - throw new ArgumentException("Y coordinate is outside bounds", nameof(y)); - } - - unsafe - { - return *(ushort*)((byte*)this.ImageData.ToPointer() + y * this.Stride + x * this.BitsPerPixel / 8); - } - } - - /// - /// Try to gets the value of a pixel in the depth image. - /// - /// Pixel's X coordinate. - /// Pixel's Y coordinate. - /// The output value of the pixel. - /// True if a pixel value is returned, otherwise false. - public bool TryGetPixel(int x, int y, out ushort value) - { - value = 0; - - if (x < 0 || x >= this.Width) - { - return false; - } - - if (y < 0 || y >= this.Height) - { - return false; - } - - unsafe - { - value = *(ushort*)((byte*)this.ImageData.ToPointer() + y * this.Stride + x * this.BitsPerPixel / 8); - } - - return true; - } - - /// - /// Gets the range of values in the depth image. - /// - /// A tuple describing the range of values in the depth image. - public (ushort, ushort) GetPixelRange() - { - ushort minRange = 65535; - ushort maxRange = 0; - - unsafe - { - byte* src = (byte*)this.ImageData.ToPointer(); + } + + this.UnmanagedBuffer.CopyFrom(bitmapData.Scan0, numBytes); + } + finally + { + bitmap.UnlockBits(bitmapData); + } + } + + /// + /// Copies the depth image from a specified source depth image of the same size. + /// + /// Source depth image to copy the depth image from. + /// The method copies the current depth image from the specified source depth image. + /// The images must have the same size, the same depth value sematics and the same depth value scale factor. + public void CopyFrom(DepthImage source) => source.CopyTo(this); + + /// + /// Copies the depth image from a specified source image of the same size and format. + /// + /// Source image to copy the depth image from. + /// The method copies the current depth image from the specified source image. + /// The size of the images must be the same, and the source image must have format. + public void CopyFrom(Image source) => source.CopyTo(this); + + /// + /// Decodes a specified encoded depth image with a specified decoder into the current depth image. + /// + /// The encoded depth image to decode. + /// The depth image decoder to use. + /// The depth image width, height and pixel format must match. The method should not be called concurrently. + public void DecodeFrom(EncodedDepthImage encodedDepthImage, IDepthImageFromStreamDecoder depthImageDecoder) + { + if (encodedDepthImage.Width != this.Width || + encodedDepthImage.Height != this.Height || + encodedDepthImage.PixelFormat != this.PixelFormat || + encodedDepthImage.DepthValueSemantics != this.DepthValueSemantics || + encodedDepthImage.DepthValueToMetersScaleFactor != this.DepthValueToMetersScaleFactor) + { + throw new InvalidOperationException("Cannot decode from an encoded depth image that has a different width, height, pixel format, depth value semantics or depth value scale factor."); + } + + depthImageDecoder.DecodeFromStream(encodedDepthImage.ToStream(), this); + } + + /// + /// Encodes the depth image using a specified encoder. + /// + /// The depth image encoder to use. + /// A new, corresponding encoded depth image. + public EncodedDepthImage Encode(IDepthImageToStreamEncoder depthImageEncoder) + { + var encodedDepthImage = new EncodedDepthImage(this.Width, this.Height, this.DepthValueSemantics, this.DepthValueToMetersScaleFactor); + encodedDepthImage.EncodeFrom(this, depthImageEncoder); + return encodedDepthImage; + } + + /// + /// Copies the depth image into a target depth image of the same size. + /// + /// Target depth image to copy this depth image to. + /// The method copies the current depth image into the specified depth image. + /// The size of the images must be the same. + public void CopyTo(DepthImage target) + { + if (this.depthValueSemantics != target.depthValueSemantics) + { + throw new InvalidOperationException("Destination image has a different depth value semantics."); + } + + if (this.depthValueToMetersScaleFactor != target.depthValueToMetersScaleFactor) + { + throw new InvalidOperationException("Destination image has a different depth value scale factor."); + } + + this.CopyTo(target.ImageData, target.Width, target.Height, target.Stride, target.PixelFormat); + } + + /// + /// Copies the depth image into a target image of the same size. + /// + /// Target image to copy this depth image to. + /// The method copies the current depth image into the specified image. + /// The size of the images must be the same. The method implements a translation of pixel formats. + public void CopyTo(Image target) + => this.CopyTo(target.ImageData, target.Width, target.Height, target.Stride, target.PixelFormat); + + /// + /// Sets a pixel in the depth image. + /// + /// Pixel's X coordinate. + /// Pixel's Y coordinate. + /// Gray value to set pixel to. + public void SetPixel(int x, int y, ushort gray) + { + if (x < 0 || x >= this.Width) + { + throw new ArgumentException("X coordinate is outside bounds", nameof(x)); + } + + if (y < 0 || y >= this.Height) + { + throw new ArgumentException("Y coordinate is outside bounds", nameof(y)); + } + + unsafe + { + byte* src = (byte*)this.ImageData.ToPointer(); + int pixelOffset = x * this.BitsPerPixel / 8 + y * this.Stride; + *(ushort*)(src + pixelOffset) = gray; + } + } + + /// + /// Gets the value of a pixel in the depth image. + /// + /// Pixel's X coordinate. + /// Pixel's Y coordinate. + /// The value of the pixel at the specified coordinates. + public ushort GetPixel(int x, int y) + { + if (x < 0 || x >= this.Width) + { + throw new ArgumentException("X coordinate is outside bounds", nameof(x)); + } + + if (y < 0 || y >= this.Height) + { + throw new ArgumentException("Y coordinate is outside bounds", nameof(y)); + } + + unsafe + { + return *(ushort*)((byte*)this.ImageData.ToPointer() + y * this.Stride + x * this.BitsPerPixel / 8); + } + } + + /// + /// Try to gets the value of a pixel in the depth image. + /// + /// Pixel's X coordinate. + /// Pixel's Y coordinate. + /// The output value of the pixel. + /// True if a pixel value is returned, otherwise false. + public bool TryGetPixel(int x, int y, out ushort value) + { + value = 0; + + if (x < 0 || x >= this.Width) + { + return false; + } + + if (y < 0 || y >= this.Height) + { + return false; + } + + unsafe + { + value = *(ushort*)((byte*)this.ImageData.ToPointer() + y * this.Stride + x * this.BitsPerPixel / 8); + } + + return true; + } + + /// + /// Gets the range of values in the depth image. + /// + /// A tuple describing the range of values in the depth image. + public (ushort, ushort) GetPixelRange() + { + ushort minRange = 65535; + ushort maxRange = 0; + + unsafe + { + byte* src = (byte*)this.ImageData.ToPointer(); for (int y = 0; y < this.Height; y++) { var strideOffset = y * this.Stride; @@ -345,76 +387,154 @@ public bool TryGetPixel(int x, int y, out ushort value) maxRange = value; } } - } - } - - return (minRange, maxRange); - } - - /// - public override ImageBase CreateEmptyOfSameSize() - { - return new DepthImage(this.Width, this.Height); - } - - private static void CheckPixelFormat(System.Drawing.Imaging.PixelFormat pixelFormat) - { - if (pixelFormat != System.Drawing.Imaging.PixelFormat.Format16bppGrayScale) - { - throw new InvalidOperationException( - $"Depth images can only be constructed from bitmaps with {nameof(System.Drawing.Imaging.PixelFormat.Format16bppGrayScale)} format."); - } - } - - /// - /// Custom serializer used for reading/writing depth images. - /// - public class CustomSerializer : ImageBase.CustomSerializer - { - private static IDepthImageCompressor depthImageCompressor = null; - - /// - /// Configure the type of compression to use when serializing depth images. Default is no compression. - /// - /// Compressor to be used. - public static void ConfigureCompression(IDepthImageCompressor depthImageCompressor) - { - CustomSerializer.depthImageCompressor = depthImageCompressor; - } - - /// - public override void Serialize(BufferWriter writer, DepthImage instance, SerializationContext context) - { - DepthCompressionMethod depthCompressionMethod = (depthImageCompressor == null) ? DepthCompressionMethod.None : depthImageCompressor.DepthCompressionMethod; - Serializer.Serialize(writer, depthCompressionMethod, context); - if (depthCompressionMethod == DepthCompressionMethod.None) - { - base.Serialize(writer, instance, context); - } - else - { - depthImageCompressor.Serialize(writer, instance, context); - } - } - - /// - public override void Deserialize(BufferReader reader, ref DepthImage target, SerializationContext context) - { - var depthCompressionMethod = DepthCompressionMethod.None; - if (this.Schema.Version >= 4) - { - Serializer.Deserialize(reader, ref depthCompressionMethod, context); - } - - if (depthCompressionMethod == DepthCompressionMethod.None) - { - base.Deserialize(reader, ref target, context); - } - else - { - depthImageCompressor.Deserialize(reader, ref target, context); - } - } - } - } -} + } + } + + return (minRange, maxRange); + } + + /// + public override ImageBase CreateEmptyOfSameSize() + => new DepthImage(this.Width, this.Height, this.DepthValueSemantics, this.DepthValueToMetersScaleFactor); + + private static void CheckPixelFormat(System.Drawing.Imaging.PixelFormat pixelFormat) + { + if (pixelFormat != System.Drawing.Imaging.PixelFormat.Format16bppGrayScale) + { + throw new InvalidOperationException( + $"Depth images can only be constructed from bitmaps with {nameof(System.Drawing.Imaging.PixelFormat.Format16bppGrayScale)} format."); + } + } + + /// + /// Custom serializer used for reading/writing depth images. + /// + public class CustomSerializer : ImageBase.CustomSerializer + { + private static IDepthImageCompressor depthImageCompressor = null; + + /// + /// Configure the type of compression to use when serializing depth images. Default is no compression. + /// + /// Compressor to be used. + public static void ConfigureCompression(IDepthImageCompressor depthImageCompressor) + { + CustomSerializer.depthImageCompressor = depthImageCompressor; + } + + /// + public override TypeSchema Initialize(KnownSerializers serializers, TypeSchema targetSchema) + { + if (targetSchema == null) + { + var baseSchema = base.Initialize(serializers, targetSchema); + var schemaMembers = new List(); + schemaMembers.AddRange(baseSchema.Members); + schemaMembers.Add(new TypeMemberSchema(nameof(DepthImage.depthValueSemantics), typeof(DepthValueSemantics?).AssemblyQualifiedName, false)); + schemaMembers.Add(new TypeMemberSchema(nameof(DepthImage.depthValueToMetersScaleFactor), typeof(double).AssemblyQualifiedName, false)); + + var type = typeof(DepthImage); + var name = TypeSchema.GetContractName(type, serializers.RuntimeVersion); + this.Schema = new TypeSchema(name, TypeSchema.GetId(name), type.AssemblyQualifiedName, TypeFlags.IsClass, schemaMembers, Version); + } + else + { + this.Schema = targetSchema; + } + + return this.Schema; + } + + /// + /// Serialize depth image. + /// + /// Writer to which to serialize. + /// Depth image instance to serialize. + /// Serialization context. + public override void Serialize(BufferWriter writer, DepthImage instance, SerializationContext context) + { + DepthCompressionMethod depthCompressionMethod = (depthImageCompressor == null) ? DepthCompressionMethod.None : depthImageCompressor.DepthCompressionMethod; + Serializer.Serialize(writer, depthCompressionMethod, context); + + if (depthCompressionMethod == DepthCompressionMethod.None) + { + base.Serialize(writer, instance, context); + } + else + { + depthImageCompressor.Serialize(writer, instance, context); + } + + if (this.Schema.Version >= 5) + { + Serializer.Serialize(writer, instance.depthValueSemantics, context); + Serializer.Serialize(writer, instance.depthValueToMetersScaleFactor, context); + } + } + + /// + /// Prepare target for cloning. + /// + /// Called before Clone, to ensure the target is valid. + /// Depth image instance from which to clone. + /// Depth image into which to clone. + /// Serialization context. + public override void PrepareCloningTarget(DepthImage instance, ref DepthImage target, SerializationContext context) + { + if (target == null || + target.Width != instance.Width || + target.Height != instance.Height || + target.PixelFormat != instance.PixelFormat || + target.DepthValueSemantics != instance.DepthValueSemantics || + target.DepthValueToMetersScaleFactor != instance.DepthValueToMetersScaleFactor) + { + target?.Dispose(); + target = (DepthImage)instance.CreateEmptyOfSameSize(); + } + } + + /// + /// Clone depth image. + /// + /// Depth image instance to clone. + /// Target depth image into which to clone. + /// Serialization context. + public override void Clone(DepthImage instance, ref DepthImage target, SerializationContext context) + { + base.Clone(instance, ref target, context); + Serializer.Clone(instance.depthValueSemantics, ref target.depthValueSemantics, context); + Serializer.Clone(instance.depthValueToMetersScaleFactor, ref target.depthValueToMetersScaleFactor, context); + } + + /// + /// Deserialize depth image. + /// + /// Buffer reader being used. + /// Target depth image into which to deserialize. + /// Serialization context. + public override void Deserialize(BufferReader reader, ref DepthImage target, SerializationContext context) + { + var depthCompressionMethod = DepthCompressionMethod.None; + if (this.Schema.Version >= 4) + { + Serializer.Deserialize(reader, ref depthCompressionMethod, context); + } + + if (depthCompressionMethod == DepthCompressionMethod.None) + { + base.Deserialize(reader, ref target, context); + } + else + { + depthImageCompressor.Deserialize(reader, ref target, context); + } + + if (this.Schema.Version >= 5) + { + Serializer.Deserialize(reader, ref target.depthValueSemantics, context); + Serializer.Deserialize(reader, ref target.depthValueToMetersScaleFactor, context); + } + } + } + } +} diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/DepthImageDecoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging/DepthImageDecoder.cs index 4ee88e202..50d7ac1fa 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/DepthImageDecoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/DepthImageDecoder.cs @@ -18,8 +18,9 @@ public class DepthImageDecoder : ConsumerProducer, Sha /// /// The pipeline to add the component to. /// The depth image decoder to use. - public DepthImageDecoder(Pipeline pipeline, IDepthImageFromStreamDecoder decoder) - : base(pipeline) + /// An optional name for the component. + public DepthImageDecoder(Pipeline pipeline, IDepthImageFromStreamDecoder decoder, string name = nameof(DepthImageDecoder)) + : base(pipeline, name) { this.decoder = decoder; } @@ -27,7 +28,11 @@ public DepthImageDecoder(Pipeline pipeline, IDepthImageFromStreamDecoder decoder /// protected override void Receive(Shared sharedEncodedDepthImage, Envelope envelope) { - using var sharedDepthImage = DepthImagePool.GetOrCreate(sharedEncodedDepthImage.Resource.Width, sharedEncodedDepthImage.Resource.Height); + using var sharedDepthImage = DepthImagePool.GetOrCreate( + sharedEncodedDepthImage.Resource.Width, + sharedEncodedDepthImage.Resource.Height, + sharedEncodedDepthImage.Resource.DepthValueSemantics, + sharedEncodedDepthImage.Resource.DepthValueToMetersScaleFactor); sharedDepthImage.Resource.DecodeFrom(sharedEncodedDepthImage.Resource, this.decoder); this.Out.Post(sharedDepthImage, envelope.OriginatingTime); } diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/DepthImageEncoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging/DepthImageEncoder.cs index e4693f614..d3ad93e25 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/DepthImageEncoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/DepthImageEncoder.cs @@ -18,8 +18,9 @@ public class DepthImageEncoder : ConsumerProducer, Shared /// The pipeline to add the component to. /// The depth image encoder to use. - public DepthImageEncoder(Pipeline pipeline, IDepthImageToStreamEncoder encoder) - : base(pipeline) + /// An optional name for the component. + public DepthImageEncoder(Pipeline pipeline, IDepthImageToStreamEncoder encoder, string name = null) + : base(pipeline, name ?? $"{nameof(DepthImageEncoder)}({encoder.Description})") { this.encoder = encoder; } @@ -28,7 +29,10 @@ public DepthImageEncoder(Pipeline pipeline, IDepthImageToStreamEncoder encoder) protected override void Receive(Shared sharedDepthImage, Envelope e) { using var sharedEncodedDepthImage = EncodedDepthImagePool.GetOrCreate( - sharedDepthImage.Resource.Width, sharedDepthImage.Resource.Height); + sharedDepthImage.Resource.Width, + sharedDepthImage.Resource.Height, + sharedDepthImage.Resource.DepthValueSemantics, + sharedDepthImage.Resource.DepthValueToMetersScaleFactor); sharedEncodedDepthImage.Resource.EncodeFrom(sharedDepthImage.Resource, this.encoder); this.Out.Post(sharedEncodedDepthImage, e.OriginatingTime); } diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/DepthImagePool.cs b/Sources/Imaging/Microsoft.Psi.Imaging/DepthImagePool.cs index 9a2f7b0d9..d729208d4 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/DepthImagePool.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/DepthImagePool.cs @@ -11,26 +11,31 @@ namespace Microsoft.Psi.Imaging /// public static class DepthImagePool { - private static readonly KeyedSharedPool Instance = - new KeyedSharedPool(key => new DepthImage(key.width, key.height)); + private static readonly KeyedSharedPool Instance = + new KeyedSharedPool(key => + new DepthImage(key.width, key.height, key.depthValueSemantics, key.depthValueToMetersScaleFactor)); /// /// Gets or creates a depth image from the pool. /// /// The requested image width. /// The requested image height. + /// Optional requested depth value semantics. + /// Optional scale factor to convert from depth values to meters. /// A shared depth image from the pool. - public static Shared GetOrCreate(int width, int height) + public static Shared GetOrCreate(int width, int height, DepthValueSemantics depthValueSemantics = DepthValueSemantics.DistanceToPlane, double depthValueToMetersScaleFactor = 0.001) { - return Instance.GetOrCreate((width, height)); + return Instance.GetOrCreate((width, height, depthValueSemantics, depthValueToMetersScaleFactor)); } /// /// Gets or creates a depth image from the pool and initializes it with a managed object. /// /// A bitmap from which to copy the image data. + /// Optional depth value semantics. + /// Optional scale factor to convert from depth values to meters. /// A shared depth image from the pool containing a copy of the image data from . - public static Shared GetOrCreateFrom(Bitmap bitmap) + public static Shared GetOrCreateFrom(Bitmap bitmap, DepthValueSemantics depthValueSemantics = DepthValueSemantics.DistanceToPlane, double depthValueToMetersScaleFactor = 0.001) { BitmapData sourceData = bitmap.LockBits( new Rectangle(0, 0, bitmap.Width, bitmap.Height), @@ -39,7 +44,7 @@ public static Shared GetOrCreateFrom(Bitmap bitmap) Shared sharedDepthImage = null; try { - sharedDepthImage = GetOrCreate(bitmap.Width, bitmap.Height); + sharedDepthImage = GetOrCreate(bitmap.Width, bitmap.Height, depthValueSemantics, depthValueToMetersScaleFactor); sharedDepthImage.Resource.CopyFrom(sourceData); } finally diff --git a/Sources/Calibration/Microsoft.Psi.Calibration/DepthPixelSemantics.cs b/Sources/Imaging/Microsoft.Psi.Imaging/DepthValueSemantics.cs similarity index 79% rename from Sources/Calibration/Microsoft.Psi.Calibration/DepthPixelSemantics.cs rename to Sources/Imaging/Microsoft.Psi.Imaging/DepthValueSemantics.cs index fb6715e05..66110dcab 100644 --- a/Sources/Calibration/Microsoft.Psi.Calibration/DepthPixelSemantics.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/DepthValueSemantics.cs @@ -1,12 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -namespace Microsoft.Psi.Calibration +namespace Microsoft.Psi.Imaging { /// - /// Defines how depth pixel values should be interpreted. + /// Defines how depth values should be interpreted. /// - public enum DepthPixelSemantics + public enum DepthValueSemantics { /// /// The depth value indicates the distance to a plane perpendicular diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/EncodedDepthImage.cs b/Sources/Imaging/Microsoft.Psi.Imaging/EncodedDepthImage.cs index 672988d29..9c739c733 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/EncodedDepthImage.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/EncodedDepthImage.cs @@ -5,12 +5,19 @@ namespace Microsoft.Psi.Imaging { using System; using System.IO; + using System.Runtime.Serialization; /// /// Defines an encoded depth image. /// - public class EncodedDepthImage : IDisposable + public class EncodedDepthImage : IDepthImage, IDisposable { + [OptionalField] + private readonly DepthValueSemantics? depthValueSemantics; + + [OptionalField] + private readonly double? depthValueToMetersScaleFactor; + /// /// The memory stream storing the encoded bytes. /// @@ -21,10 +28,14 @@ public class EncodedDepthImage : IDisposable /// /// Width of encoded depth image in pixels. /// Height of encoded depth image in pixels. - public EncodedDepthImage(int width, int height) + /// Optional depth value semantics. + /// Optional scale factor to convert from depth values to meters. + public EncodedDepthImage(int width, int height, DepthValueSemantics depthValueSemantics = DepthValueSemantics.DistanceToPlane, double depthValueToMetersScaleFactor = 0.001) { this.Width = width; this.Height = height; + this.depthValueSemantics = depthValueSemantics; + this.depthValueToMetersScaleFactor = depthValueToMetersScaleFactor; this.PixelFormat = PixelFormat.Gray_16bpp; this.stream = new MemoryStream(); } @@ -35,31 +46,35 @@ public EncodedDepthImage(int width, int height) /// Width of image in pixels. /// Height of image in pixels. /// Byte array used to initialize the image data. - public EncodedDepthImage(int width, int height, byte[] contents) + /// Optional depth value semantics. + /// Optional scale factor to convert from depth values to meters. + public EncodedDepthImage(int width, int height, byte[] contents, DepthValueSemantics depthValueSemantics = DepthValueSemantics.DistanceToPlane, double depthValueToMetersScaleFactor = 0.001) { this.Width = width; this.Height = height; + this.depthValueSemantics = depthValueSemantics; + this.depthValueToMetersScaleFactor = depthValueToMetersScaleFactor; this.PixelFormat = PixelFormat.Gray_16bpp; this.stream = new MemoryStream(); this.stream.Write(contents, 0, contents.Length); this.stream.Position = 0; } - /// - /// Gets the width of the depth image in pixels. - /// + /// public int Width { get; } - /// - /// Gets the height of the depth image in pixels. - /// + /// public int Height { get; } - /// - /// Gets the pixel format for the depth image. - /// + /// public PixelFormat PixelFormat { get; } + /// + public DepthValueSemantics DepthValueSemantics => this.depthValueSemantics ?? DepthValueSemantics.DistanceToPlane; + + /// + public double DepthValueToMetersScaleFactor => this.depthValueToMetersScaleFactor ?? 0.001; + /// /// Releases the depth image. /// @@ -114,9 +129,13 @@ public void SetBuffer(byte[] buffer, int offset, int count) /// The depth image width, height and pixel format must match. The method should not be called concurrently. public void EncodeFrom(DepthImage depthImage, IDepthImageToStreamEncoder depthImageEncoder) { - if (depthImage.Width != this.Width || depthImage.Height != this.Height || depthImage.PixelFormat != this.PixelFormat) + if (depthImage.Width != this.Width || + depthImage.Height != this.Height || + depthImage.PixelFormat != this.PixelFormat || + depthImage.DepthValueSemantics != this.DepthValueSemantics || + depthImage.DepthValueToMetersScaleFactor != this.DepthValueToMetersScaleFactor) { - throw new InvalidOperationException("Cannot encode from an image that has a different width, height, or pixel format."); + throw new InvalidOperationException("Cannot encode from an image that has a different width, height, pixel format, depth value semantics, or depth value scale factor."); } this.stream.Position = 0; @@ -130,7 +149,7 @@ public void EncodeFrom(DepthImage depthImage, IDepthImageToStreamEncoder depthIm /// A new, corresponding decoded depth image. public DepthImage Decode(IDepthImageFromStreamDecoder depthImageDecoder) { - var depthImage = new DepthImage(this.Width, this.Height); + var depthImage = new DepthImage(this.Width, this.Height, this.DepthValueSemantics, this.DepthValueToMetersScaleFactor); depthImage.DecodeFrom(this, depthImageDecoder); return depthImage; } diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/EncodedDepthImagePool.cs b/Sources/Imaging/Microsoft.Psi.Imaging/EncodedDepthImagePool.cs index be70c6a94..a02000801 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/EncodedDepthImagePool.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/EncodedDepthImagePool.cs @@ -8,8 +8,9 @@ namespace Microsoft.Psi.Imaging /// public static class EncodedDepthImagePool { - private static readonly KeyedSharedPool Instance = - new KeyedSharedPool(key => new EncodedDepthImage(key.width, key.height)); + private static readonly KeyedSharedPool Instance = + new KeyedSharedPool(key => + new EncodedDepthImage(key.width, key.height, key.depthValueSemantics, key.depthValueToMetersScaleFactor)); /// /// Gets or creates an encoded depth image from the pool. @@ -17,9 +18,11 @@ public static class EncodedDepthImagePool /// A shared encoded depth image from the pool. /// The requested encoded depth image width. /// The requested encoded depth image height. - public static Shared GetOrCreate(int width, int height) + /// Optional requested depth value semantics. + /// Optional scale factor to convert from depth values to meters. + public static Shared GetOrCreate(int width, int height, DepthValueSemantics depthValueSemantics = DepthValueSemantics.DistanceToPlane, double depthValueToMetersScaleFactor = 0.001) { - return Instance.GetOrCreate((width, height)); + return Instance.GetOrCreate((width, height, depthValueSemantics, depthValueToMetersScaleFactor)); } } } diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/EncodedImage.cs b/Sources/Imaging/Microsoft.Psi.Imaging/EncodedImage.cs index 6a6d00829..9be71a2db 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/EncodedImage.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/EncodedImage.cs @@ -5,12 +5,13 @@ namespace Microsoft.Psi.Imaging { using System; using System.IO; + using System.Runtime.InteropServices; using System.Runtime.Serialization; /// /// Defines an encoded image. /// - public class EncodedImage : IDisposable + public class EncodedImage : IImage, IDisposable { /// /// The pixel format was added as a private optional field backed property @@ -39,19 +40,13 @@ public EncodedImage(int width, int height, PixelFormat pixelFormat) this.stream = new MemoryStream(); } - /// - /// Gets the width of the image in pixels. - /// + /// public int Width { get; } - /// - /// Gets the height of the image in pixels. - /// + /// public int Height { get; } - /// - /// Gets the pixel format for the encoded image. - /// + /// public PixelFormat PixelFormat => this.pixelFormat; /// @@ -94,15 +89,28 @@ public byte[] GetBuffer() } /// - /// Sets the image data to byte array. + /// Copy the image data from a byte array. /// - /// Byte array containing the image data. + /// Byte array containing the image data. /// The offset in buffer at which to begin copying bytes. /// The maximum number of bytes to copy. - public void SetBuffer(byte[] buffer, int offset, int count) + public void CopyFrom(byte[] source, int offset, int count) { this.stream.SetLength(0); - this.stream.Write(buffer, offset, count); + this.stream.Write(source, offset, count); + } + + /// + /// Copy the image data from a memory pointer. + /// + /// Memory pointer from which to copy data. + /// The offset at which to begin copying bytes. + /// The maximum number of bytes to copy. + public unsafe void CopyFrom(IntPtr source, int offset, int count) + { + this.stream.SetLength(offset + count); + var buffer = this.stream.GetBuffer(); + Marshal.Copy(source, buffer, offset, count); } /// diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/IDepthImage.cs b/Sources/Imaging/Microsoft.Psi.Imaging/IDepthImage.cs new file mode 100644 index 000000000..efa4d597b --- /dev/null +++ b/Sources/Imaging/Microsoft.Psi.Imaging/IDepthImage.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Imaging +{ + /// + /// Interface that defines a depth image. + /// + public interface IDepthImage : IImage + { + /// + /// Gets depth value semantics. + /// + DepthValueSemantics DepthValueSemantics { get; } + + /// + /// Gets the scale factor to convert depth values to meters. + /// + double DepthValueToMetersScaleFactor { get; } + } +} diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/IDepthImageToStreamEncoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging/IDepthImageToStreamEncoder.cs index d4b8dd405..a6e57c13e 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/IDepthImageToStreamEncoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/IDepthImageToStreamEncoder.cs @@ -10,6 +10,11 @@ namespace Microsoft.Psi.Imaging /// public interface IDepthImageToStreamEncoder { + /// + /// Gets the description of the depth image stream encoder. + /// + public string Description { get; } + /// /// Encodes a depth image into a stream. /// diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/IImage.cs b/Sources/Imaging/Microsoft.Psi.Imaging/IImage.cs new file mode 100644 index 000000000..d00f6cc26 --- /dev/null +++ b/Sources/Imaging/Microsoft.Psi.Imaging/IImage.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Imaging +{ + /// + /// Interface that defines an image. + /// + public interface IImage + { + /// + /// Gets the width of the image in pixels. + /// + public int Width { get; } + + /// + /// Gets the height of the image in pixels. + /// + public int Height { get; } + + /// + /// Gets image pixel format. + /// + public PixelFormat PixelFormat { get; } + } +} diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/IImageToStreamEncoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging/IImageToStreamEncoder.cs index d6205616f..561ba46bf 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/IImageToStreamEncoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/IImageToStreamEncoder.cs @@ -10,6 +10,11 @@ namespace Microsoft.Psi.Imaging /// public interface IImageToStreamEncoder { + /// + /// Gets the description of the encoder. + /// + string Description { get; } + /// /// Encodes an image into a stream. /// diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/ImageBase.cs b/Sources/Imaging/Microsoft.Psi.Imaging/ImageBase.cs index 4a4b9e194..eeb246de4 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/ImageBase.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/ImageBase.cs @@ -15,7 +15,7 @@ namespace Microsoft.Psi.Imaging /// /// Using this class it is possible as to allocate a new image in unmanaged memory, /// as to just wrap provided pointer to unmanaged memory, where an image is stored. - public abstract class ImageBase : IDisposable + public abstract class ImageBase : IImage, IDisposable { /// /// Exception message when unexpected pixel format is encountered. @@ -134,20 +134,14 @@ public ImageBase(BitmapData bitmapData, bool makeCopy = false) /// public IntPtr ImageData => this.image.Data; - /// - /// Gets image width in pixels. - /// + /// public int Width => this.width; - /// - /// Gets image height in pixels. - /// + /// public int Height => this.height; - /// - /// Gets image stride (line size in bytes). - /// - public int Stride => this.stride; + /// + public PixelFormat PixelFormat => this.pixelFormat; /// /// Gets the size of the image in bytes (stride times height). @@ -155,14 +149,14 @@ public ImageBase(BitmapData bitmapData, bool makeCopy = false) public int Size => this.stride * this.height; /// - /// Gets the bits per pixel in the image. + /// Gets image stride (line size in bytes). /// - public int BitsPerPixel => this.pixelFormat.GetBitsPerPixel(); + public int Stride => this.stride; /// - /// Gets image pixel format. + /// Gets the bits per pixel in the image. /// - public PixelFormat PixelFormat => this.pixelFormat; + public int BitsPerPixel => this.pixelFormat.GetBitsPerPixel(); /// /// Disposes the image. @@ -653,15 +647,18 @@ private void CopyImageSlow(IntPtr sourceIntPtr, PixelFormat sourceFormat, IntPtr public abstract class CustomSerializer : ISerializer where TImage : ImageBase { - private const int Version = 4; + /// + /// Gets the schema version for custom image serialization. + /// + protected const int Version = 5; /// public bool? IsClearRequired => true; /// - /// Gets the type schema. + /// Gets or sets the type schema. /// - protected TypeSchema Schema { get; private set; } + protected TypeSchema Schema { get; set; } /// /// Initialize custom serializer. diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/ImageDecoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging/ImageDecoder.cs index 5ba7bfa34..1b9b12cdf 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/ImageDecoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/ImageDecoder.cs @@ -19,8 +19,9 @@ public class ImageDecoder : ConsumerProducer, Shared /// /// The pipeline to add the component to. /// The image decoder to use. - public ImageDecoder(Pipeline pipeline, IImageFromStreamDecoder decoder) - : base(pipeline) + /// An optional name for the component. + public ImageDecoder(Pipeline pipeline, IImageFromStreamDecoder decoder, string name = nameof(ImageDecoder)) + : base(pipeline, name) { this.decoder = decoder; } diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/ImageEncoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging/ImageEncoder.cs index 1b97c3467..056812224 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/ImageEncoder.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/ImageEncoder.cs @@ -18,8 +18,9 @@ public class ImageEncoder : ConsumerProducer, Shared /// /// The pipeline to add the component to. /// The image encoder to use. - public ImageEncoder(Pipeline pipeline, IImageToStreamEncoder encoder) - : base(pipeline) + /// An optional name for the component. + public ImageEncoder(Pipeline pipeline, IImageToStreamEncoder encoder, string name = null) + : base(pipeline, name ?? $"{nameof(ImageEncoder)}({encoder.Description})") { this.encoder = encoder; } diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/ImageExtensions.cs b/Sources/Imaging/Microsoft.Psi.Imaging/ImageExtensions.cs index 9d49fe298..213ebb927 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/ImageExtensions.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/ImageExtensions.cs @@ -488,6 +488,28 @@ public static bool Compare(this ImageBase image1, ImageBase image2, double toler return errorMetrics.NumberOutliers <= percentOutliersAllowed * image1.Width * image1.Height; } + /// + /// Compares two depth images to see if they are identical (within some specified tolerance). + /// + /// First image in comparison. + /// Second image in comparison. + /// Maximum allowable distance between pixels in Grayscale space. + /// Percetange of pixels allowed to be outside tolerance. + /// Error metrics across all pixels. + /// True if images are considered identical. False otherwise. + public static bool Compare(this DepthImage depthImage1, DepthImage depthImage2, double tolerance, double percentOutliersAllowed, ref ImageError errorMetrics) + { + if (depthImage1.DepthValueSemantics != depthImage2.DepthValueSemantics || + depthImage1.DepthValueToMetersScaleFactor != depthImage2.DepthValueToMetersScaleFactor) + { + return false; + } + else + { + return Compare(depthImage1 as ImageBase, depthImage2 as ImageBase, tolerance, percentOutliersAllowed, ref errorMetrics); + } + } + /// /// Resizes an image by the specified scale factors using the specified sampling mode. /// @@ -843,7 +865,7 @@ public static void Crop(this Image image, Image croppedImage, Rectangle rectangl public static DepthImage Crop(this DepthImage depthImage, Rectangle rectangle, bool clip = false) { var actualRectangle = clip ? GetImageSizeClippedRectangle(rectangle, depthImage.Width, depthImage.Height) : rectangle; - var croppedDepthImage = new DepthImage(actualRectangle.Width, actualRectangle.Height, depthImage.Stride); + var croppedDepthImage = new DepthImage(actualRectangle.Width, actualRectangle.Height, depthImage.Stride, depthImage.DepthValueSemantics, depthImage.DepthValueToMetersScaleFactor); depthImage.Crop(croppedDepthImage, actualRectangle, clip: false); return croppedDepthImage; } @@ -875,6 +897,16 @@ public static void Crop(this DepthImage depthImage, DepthImage croppedDepthImage throw new ArgumentOutOfRangeException($"{nameof(croppedDepthImage)}.PixelFormat", "destination image pixel format doesn't match source image pixel format"); } + if (croppedDepthImage.DepthValueSemantics != depthImage.DepthValueSemantics) + { + throw new ArgumentOutOfRangeException($"{nameof(croppedDepthImage)}.DepthValueSemantics", "destination image depth value semantics doesn't match source depth value semantics."); + } + + if (croppedDepthImage.DepthValueToMetersScaleFactor != depthImage.DepthValueToMetersScaleFactor) + { + throw new ArgumentOutOfRangeException($"{nameof(croppedDepthImage)}.DepthValueToMetersScaleFactor", "destination image depth value scale factor doesn't match source depth value scale factor."); + } + var actualRectangle = clip ? GetImageSizeClippedRectangle(rectangle, depthImage.Width, depthImage.Height) : rectangle; if (actualRectangle.IsEmpty) diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/ImageFromGZipStreamDecoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging/ImageFromGZipStreamDecoder.cs new file mode 100644 index 000000000..b24caf40b --- /dev/null +++ b/Sources/Imaging/Microsoft.Psi.Imaging/ImageFromGZipStreamDecoder.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Imaging +{ + using System; + using System.IO; + using System.IO.Compression; + + /// + /// Implements a GZip image decoder. + /// + public class ImageFromGZipStreamDecoder : IImageFromStreamDecoder + { + /// + /// Determine whether stream has a GZip header. + /// + /// Stream containing image data. + /// A value indicating whether the stream has a GZip header. + public bool HasGZipHeader(Stream stream) + { + var isGZip = stream.Length >= 2 && stream.ReadByte() == 0x1f && stream.ReadByte() == 0x8b; + stream.Position = 0; + return isGZip; + } + + /// + public void DecodeFromStream(Stream stream, Image image) + { + if (!this.HasGZipHeader(stream)) + { + throw new ArgumentException("Stream does not appear to be GZip-encoded (missing header)."); + } + + // decode GZip + var size = image.Stride * image.Height; + using var decompressor = new GZipStream(stream, CompressionMode.Decompress); + unsafe + { + decompressor.CopyTo(new UnmanagedMemoryStream((byte*)image.ImageData.ToPointer(), size, size, FileAccess.ReadWrite)); + } + } + + /// + public PixelFormat GetPixelFormat(Stream stream) + { + if (!this.HasGZipHeader(stream)) + { + throw new ArgumentException("Stream does not appear to be GZip-encoded (missing header)."); + } + + return PixelFormat.Undefined; // unknown (only affects images prior to PixelFormat property being introduced anyway) + } + } +} diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/ImageFromNV12StreamDecoder.cs b/Sources/Imaging/Microsoft.Psi.Imaging/ImageFromNV12StreamDecoder.cs new file mode 100644 index 000000000..a7bb6efbf --- /dev/null +++ b/Sources/Imaging/Microsoft.Psi.Imaging/ImageFromNV12StreamDecoder.cs @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Imaging +{ + using System; + using System.IO; + + /// + /// Implements an NV12 image decoder. + /// + public class ImageFromNV12StreamDecoder : IImageFromStreamDecoder + { + /// + /// Determine whether stream has an NV12 header. + /// + /// Stream containing image data. + /// A value indicating whether the stream has an NV12 header. + public bool HasNV12Header(Stream stream) + { + var isNV12 = stream.Length >= 4 && stream.ReadByte() == 'N' && stream.ReadByte() == 'V' && stream.ReadByte() == '1' && stream.ReadByte() == '2'; + stream.Position = 0; + return isNV12; + } + + /// + public void DecodeFromStream(Stream stream, Image image) + { + if (!this.HasNV12Header(stream)) + { + throw new ArgumentException("Stream does not appear to be NV12-encoded (missing header)."); + } + + stream.Position = 4; // skip header + + // decode NV12 + var width = image.Width; + var height = image.Height; + int startUV = width * height; + int strideUV = width; + + var size = (int)(width * height * 1.5 + 0.5); // 12-bit/pixel + using var sharedData = SharedArrayPool.GetOrCreate(size); + var data = sharedData.Resource; + stream.Read(data, 0, size); + + unsafe + { + var buffer = (byte*)image.UnmanagedBuffer.Data; + for (int i = 0; i < height; i++) + { + var p = buffer + (i * 4 * width); + int row = i * width; + var startUVrow = startUV + ((i / 2) * strideUV); + for (int j = 0; j < width; j++) + { + var y = data[row + j] - 16; + var uindex = startUVrow + (2 * (j / 2)); + var u = data[uindex] - 128; + var v = data[uindex + 1] - 128; + + var yy = 1.164383 * y; + var b = yy + (2.017232 * u); + var g = yy - (0.812968 * v) - (0.391762 * u); + var r = yy + (1.596027 * v); + + *p++ = (byte)Math.Max(0, Math.Min(255, b + 0.5)); + *p++ = (byte)Math.Max(0, Math.Min(255, g + 0.5)); + *p++ = (byte)Math.Max(0, Math.Min(255, r + 0.5)); + *p++ = 0xff; // alpha + } + } + } + } + + /// + public PixelFormat GetPixelFormat(Stream stream) + { + if (!this.HasNV12Header(stream)) + { + throw new ArgumentException("Stream does not appear to be NV12-encoded (missing header)."); + } + + return PixelFormat.BGRA_32bpp; + } + } +} diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/ImageTransformer.cs b/Sources/Imaging/Microsoft.Psi.Imaging/ImageTransformer.cs index 7ab8544b7..3ca669594 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/ImageTransformer.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/ImageTransformer.cs @@ -3,6 +3,7 @@ namespace Microsoft.Psi.Imaging { + using System; using Microsoft.Psi; using Microsoft.Psi.Components; @@ -20,7 +21,7 @@ public class ImageTransformer : ConsumerProducer, Shared> { private readonly TransformDelegate transformer; private readonly PixelFormat pixelFormat; - private System.Func> sharedImageAllocator; + private readonly Func> sharedImageAllocator; /// /// Initializes a new instance of the class. @@ -29,13 +30,18 @@ public class ImageTransformer : ConsumerProducer, Shared> /// Function for transforming the source image. /// Pixel format for destination image. /// Optional image allocator for creating new shared image. - public ImageTransformer(Pipeline pipeline, TransformDelegate transformer, PixelFormat pixelFormat, System.Func> sharedImageAllocator = null) - : base(pipeline) + /// An optional name for the component. + public ImageTransformer( + Pipeline pipeline, + TransformDelegate transformer, + PixelFormat pixelFormat, + Func> sharedImageAllocator = null, + string name = nameof(ImageTransformer)) + : base(pipeline, name) { this.transformer = transformer; this.pixelFormat = pixelFormat; - sharedImageAllocator ??= (width, height, pixelFormat) => ImagePool.GetOrCreate(width, height, pixelFormat); - this.sharedImageAllocator = sharedImageAllocator; + this.sharedImageAllocator = sharedImageAllocator ?? ((width, height, pixelFormat) => ImagePool.GetOrCreate(width, height, pixelFormat)); } /// diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/PixelFormatHelper.cs b/Sources/Imaging/Microsoft.Psi.Imaging/PixelFormatHelper.cs index 2a0b33237..3c0e0c1f2 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/PixelFormatHelper.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/PixelFormatHelper.cs @@ -101,31 +101,16 @@ internal static System.Drawing.Imaging.PixelFormat ToSystemPixelFormat(PixelForm /// internal static int GetBytesPerPixel(PixelFormat pixelFormat) { - switch (pixelFormat) + return pixelFormat switch { - case PixelFormat.Gray_8bpp: - return 1; - - case PixelFormat.Gray_16bpp: - return 2; - - case PixelFormat.BGR_24bpp: - case PixelFormat.RGB_24bpp: - return 3; - - case PixelFormat.BGRX_32bpp: - case PixelFormat.BGRA_32bpp: - return 4; - - case PixelFormat.RGBA_64bpp: - return 8; - - case PixelFormat.Undefined: - return 0; - - default: - throw new ArgumentException("Unknown pixel format"); - } + PixelFormat.Gray_8bpp => 1, + PixelFormat.Gray_16bpp => 2, + PixelFormat.BGR_24bpp or PixelFormat.RGB_24bpp => 3, + PixelFormat.BGRX_32bpp or PixelFormat.BGRA_32bpp => 4, + PixelFormat.RGBA_64bpp => 8, + PixelFormat.Undefined => 0, + _ => throw new ArgumentException("Unknown pixel format"), + }; } /// @@ -138,25 +123,13 @@ internal static int GetBytesPerPixel(PixelFormat pixelFormat) /// internal static int GetBitsPerChannel(PixelFormat pixelFormat) { - switch (pixelFormat) + return pixelFormat switch { - case PixelFormat.Gray_8bpp: - case PixelFormat.BGR_24bpp: - case PixelFormat.BGRX_32bpp: - case PixelFormat.BGRA_32bpp: - case PixelFormat.RGB_24bpp: - return 8; - - case PixelFormat.Gray_16bpp: - case PixelFormat.RGBA_64bpp: - return 16; - - case PixelFormat.Undefined: - return 0; - - default: - throw new ArgumentException("Unknown pixel format"); - } + PixelFormat.Gray_8bpp or PixelFormat.BGR_24bpp or PixelFormat.BGRX_32bpp or PixelFormat.BGRA_32bpp or PixelFormat.RGB_24bpp => 8, + PixelFormat.Gray_16bpp or PixelFormat.RGBA_64bpp => 16, + PixelFormat.Undefined => 0, + _ => throw new ArgumentException("Unknown pixel format"), + }; } } } diff --git a/Sources/Imaging/Microsoft.Psi.Imaging/StreamOperators.cs b/Sources/Imaging/Microsoft.Psi.Imaging/StreamOperators.cs index ddc86a954..cd162ee9d 100644 --- a/Sources/Imaging/Microsoft.Psi.Imaging/StreamOperators.cs +++ b/Sources/Imaging/Microsoft.Psi.Imaging/StreamOperators.cs @@ -1,68 +1,94 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace Microsoft.Psi.Imaging -{ - using System; +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Imaging +{ + using System; using System.Drawing; - using Microsoft.Psi.Components; - - /// - /// Implements operators for processing streams of images. - /// - public static partial class Operators - { - /// - /// Converts a stream of images into a stream of depth images. - /// - /// A producer of images. - /// An optional delivery policy. - /// Optional image allocator for creating new shared depth image. - /// A corresponding stream of depth images. - /// The images in the source stream need to be in format. - public static IProducer> ToDepthImage(this IProducer> source, DeliveryPolicy> deliveryPolicy = null, Func> sharedDepthImageAllocator = null) - { - sharedDepthImageAllocator ??= DepthImagePool.GetOrCreate; - return source.Process, Shared>( - (sharedImage, envelope, emitter) => - { - using var sharedDepthImage = sharedDepthImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height); - sharedDepthImage.Resource.CopyFrom(sharedImage.Resource); - emitter.Post(sharedDepthImage, envelope.OriginatingTime); - }, - deliveryPolicy); - } - - /// - /// Converts a stream of depth images into a stream of format images. - /// - /// A producer of depth images. - /// An optional delivery policy. - /// Optional image allocator for creating new shared depth image. - /// A corresponding stream of images. - public static IProducer> ToImage(this IProducer> source, DeliveryPolicy> deliveryPolicy = null, Func> sharedDepthImageAllocator = null) - { - sharedDepthImageAllocator ??= (width, height) => ImagePool.GetOrCreate(width, height, PixelFormat.Gray_16bpp); - return source.Process, Shared>( - (sharedDepthImage, envelope, emitter) => - { + using Microsoft.Psi.Components; + + /// + /// Implements operators for processing streams of images. + /// + public static partial class Operators + { + /// + /// Converts a stream of images into a stream of depth images. + /// + /// A producer of images. + /// Depth value semantics. + /// Scale factor to convert from depth values to meters. + /// An optional delivery policy. + /// Optional image allocator for creating new shared depth image. + /// An optional name for the stream operator. + /// A corresponding stream of depth images. + /// The images in the source stream need to be in format. + public static IProducer> ToDepthImage( + this IProducer> source, + DepthValueSemantics depthValueSemantics, + double depthValueToMetersScaleFactor, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedDepthImageAllocator = null, + string name = nameof(ToDepthImage)) + { + sharedDepthImageAllocator ??= DepthImagePool.GetOrCreate; + return source.Process, Shared>( + (sharedImage, envelope, emitter) => + { + using var sharedDepthImage = sharedDepthImageAllocator( + sharedImage.Resource.Width, + sharedImage.Resource.Height, + depthValueSemantics, + depthValueToMetersScaleFactor); + sharedDepthImage.Resource.CopyFrom(sharedImage.Resource); + emitter.Post(sharedDepthImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + /// + /// Converts a stream of depth images into a stream of format images. + /// + /// A producer of depth images. + /// An optional delivery policy. + /// Optional image allocator for creating new shared depth image. + /// An optional name for the stream operator. + /// A corresponding stream of images. + public static IProducer> ToImage( + this IProducer> source, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedDepthImageAllocator = null, + string name = nameof(ToImage)) + { + sharedDepthImageAllocator ??= (width, height) => ImagePool.GetOrCreate(width, height, PixelFormat.Gray_16bpp); + return source.Process, Shared>( + (sharedDepthImage, envelope, emitter) => + { using var sharedImage = sharedDepthImageAllocator(sharedDepthImage.Resource.Width, sharedDepthImage.Resource.Height); sharedImage.Resource.CopyFrom(sharedDepthImage.Resource); emitter.Post(sharedImage, envelope.OriginatingTime); - }, - deliveryPolicy); - } - - /// - /// Converts the source image to a different pixel format. - /// - /// The source stream. - /// The pixel format to convert to. - /// An optional delivery policy. - /// Optional image allocator for creating new shared image. - /// The resulting stream. - public static IProducer> Convert(this IProducer> source, PixelFormat pixelFormat, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { + }, + deliveryPolicy, + name); + } + + /// + /// Converts the source image to a different pixel format. + /// + /// The source stream. + /// The pixel format to convert to. + /// An optional delivery policy. + /// Optional image allocator for creating new shared image. + /// An optional name for the stream operator. + /// The resulting stream. + public static IProducer> Convert( + this IProducer> source, + PixelFormat pixelFormat, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(Convert)) + { sharedImageAllocator ??= (width, height, pixelFormat) => ImagePool.GetOrCreate(width, height, pixelFormat); return source.Process, Shared>( (sharedImage, envelope, emitter) => @@ -83,148 +109,181 @@ public static IProducer> Convert(this IProducer> sou sharedImage.Resource.CopyTo(image.Resource); emitter.Post(image, envelope.OriginatingTime); } - }, deliveryPolicy); - } - - /// - /// Converts a shared image to a different pixel format using the specified transformer. - /// - /// Source image to compress. - /// Method for converting an image sample. - /// Pixel format to use for converted image. - /// An optional delivery policy. - /// Optional image allocator for creating new shared image. - /// Returns a producer that generates the transformed images. - public static IProducer> Transform(this IProducer> source, TransformDelegate transformer, PixelFormat pixelFormat, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - return source.PipeTo(new ImageTransformer(source.Out.Pipeline, transformer, pixelFormat, sharedImageAllocator), deliveryPolicy); - } - - /// - /// Crops a shared image using the specified rectangle. - /// - /// Source of image and rectangle messages. + }, + deliveryPolicy, + name); + } + + /// + /// Converts a shared image to a different pixel format using the specified transformer. + /// + /// Source image to compress. + /// Method for converting an image sample. + /// Pixel format to use for converted image. + /// An optional delivery policy. + /// Optional image allocator for creating new shared image. + /// An optional name for the stream operator. + /// Returns a producer that generates the transformed images. + public static IProducer> Transform( + this IProducer> source, + TransformDelegate transformer, + PixelFormat pixelFormat, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(Transform)) + => source.PipeTo(new ImageTransformer(source.Out.Pipeline, transformer, pixelFormat, sharedImageAllocator, name), deliveryPolicy); + + /// + /// Crops a shared image using the specified rectangle. + /// + /// Source of image and rectangle messages. /// An optional parameter indicating whether to clip the region (by default false). - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Returns a producer generating new cropped image samples. - public static IProducer> Crop( - this IProducer<(Shared, Rectangle)> source, - bool clip = false, - DeliveryPolicy<(Shared, Rectangle)> deliveryPolicy = null, - Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process<(Shared, Rectangle), Shared>( - (tupleOfSharedImageAndRectangle, envelope, emitter) => - { - (var sharedImage, var rectangle) = tupleOfSharedImageAndRectangle; - var actualRectangle = clip ? GetImageSizeClippedRectangle(rectangle, sharedImage.Resource.Width, sharedImage.Resource.Height) : rectangle; + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Returns a producer generating new cropped image samples. + public static IProducer> Crop( + this IProducer<(Shared, Rectangle)> source, + bool clip = false, + DeliveryPolicy<(Shared, Rectangle)> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(Crop)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process<(Shared, Rectangle), Shared>( + (tupleOfSharedImageAndRectangle, envelope, emitter) => + { + (var sharedImage, var rectangle) = tupleOfSharedImageAndRectangle; + var actualRectangle = clip ? GetImageSizeClippedRectangle(rectangle, sharedImage.Resource.Width, sharedImage.Resource.Height) : rectangle; if (actualRectangle.IsEmpty) { emitter.Post(null, envelope.OriginatingTime); - } + } else { using var croppedSharedImage = sharedImageAllocator(actualRectangle.Width, actualRectangle.Height, sharedImage.Resource.PixelFormat); sharedImage.Resource.Crop(croppedSharedImage.Resource, actualRectangle, clip: false); emitter.Post(croppedSharedImage, envelope.OriginatingTime); - } - }, deliveryPolicy); - } - - /// - /// Crops a shared image using the specified nullable rectangle. When no rectangle is specified, produces a null image. - /// - /// Source of image and rectangle messages. + } + }, + deliveryPolicy, + name); + } + + /// + /// Crops a shared image using the specified nullable rectangle. When no rectangle is specified, produces a null image. + /// + /// Source of image and rectangle messages. /// An optional parameter indicating whether to clip the region (by default false). - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Returns a producer generating new cropped image samples. - public static IProducer> Crop( - this IProducer<(Shared, Rectangle?)> source, - bool clip = false, - DeliveryPolicy<(Shared, Rectangle?)> deliveryPolicy = null, - Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process<(Shared, Rectangle?), Shared>( - (tupleOfSharedImageAndRectangle, envelope, emitter) => - { - (var sharedImage, var rectangle) = tupleOfSharedImageAndRectangle; + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Returns a producer generating new cropped image samples. + public static IProducer> Crop( + this IProducer<(Shared, Rectangle?)> source, + bool clip = false, + DeliveryPolicy<(Shared, Rectangle?)> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(Crop)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process<(Shared, Rectangle?), Shared>( + (tupleOfSharedImageAndRectangle, envelope, emitter) => + { + (var sharedImage, var rectangle) = tupleOfSharedImageAndRectangle; if (rectangle.HasValue) { - var actualRectangle = clip ? GetImageSizeClippedRectangle(rectangle.Value, sharedImage.Resource.Width, sharedImage.Resource.Height) : rectangle.Value; + var actualRectangle = clip ? GetImageSizeClippedRectangle(rectangle.Value, sharedImage.Resource.Width, sharedImage.Resource.Height) : rectangle.Value; if (actualRectangle.IsEmpty) { emitter.Post(null, envelope.OriginatingTime); - } + } else { using var croppedSharedImage = sharedImageAllocator(actualRectangle.Width, actualRectangle.Height, sharedImage.Resource.PixelFormat); sharedImage.Resource.Crop(croppedSharedImage.Resource, actualRectangle, clip: false); emitter.Post(croppedSharedImage, envelope.OriginatingTime); } - } + } else { emitter.Post(null, envelope.OriginatingTime); - } - }, deliveryPolicy); - } - - /// - /// Crops a shared depth image using the specified rectangle. - /// - /// Source of depth image and rectangle messages. + } + }, + deliveryPolicy, + name); + } + + /// + /// Crops a shared depth image using the specified rectangle. + /// + /// Source of depth image and rectangle messages. + /// Depth value semantics. + /// Scale factor to convert from depth values to meters. /// An optional parameter indicating whether to clip the region (by default false). - /// An optional delivery policy. - /// Optional image allocator to create new shared depth image. - /// Returns a producer generating new cropped image samples. - public static IProducer> Crop( - this IProducer<(Shared, Rectangle)> source, - bool clip = false, - DeliveryPolicy<(Shared, Rectangle)> deliveryPolicy = null, - Func> sharedDepthImageAllocator = null) - { - sharedDepthImageAllocator ??= DepthImagePool.GetOrCreate; - return source.Process<(Shared, Rectangle), Shared>( - (tupleOfSharedDepthImageAndRectangle, envelope, emitter) => - { - (var sharedDepthImage, var rectangle) = tupleOfSharedDepthImageAndRectangle; - var actualRectangle = clip ? GetImageSizeClippedRectangle(rectangle, sharedDepthImage.Resource.Width, sharedDepthImage.Resource.Height) : rectangle; + /// An optional delivery policy. + /// Optional image allocator to create new shared depth image. + /// An optional name for the stream operator. + /// Returns a producer generating new cropped image samples. + public static IProducer> Crop( + this IProducer<(Shared, Rectangle)> source, + DepthValueSemantics depthValueSemantics, + double depthValueToMetersScaleFactor, + bool clip = false, + DeliveryPolicy<(Shared, Rectangle)> deliveryPolicy = null, + Func> sharedDepthImageAllocator = null, + string name = nameof(Crop)) + { + sharedDepthImageAllocator ??= DepthImagePool.GetOrCreate; + return source.Process<(Shared, Rectangle), Shared>( + (tupleOfSharedDepthImageAndRectangle, envelope, emitter) => + { + (var sharedDepthImage, var rectangle) = tupleOfSharedDepthImageAndRectangle; + var actualRectangle = clip ? GetImageSizeClippedRectangle(rectangle, sharedDepthImage.Resource.Width, sharedDepthImage.Resource.Height) : rectangle; if (actualRectangle.IsEmpty) { emitter.Post(null, envelope.OriginatingTime); - } + } else { - using var croppedSharedDepthImage = sharedDepthImageAllocator(actualRectangle.Width, actualRectangle.Height); + using var croppedSharedDepthImage = sharedDepthImageAllocator( + actualRectangle.Width, + actualRectangle.Height, + depthValueSemantics, + depthValueToMetersScaleFactor); sharedDepthImage.Resource.Crop(croppedSharedDepthImage.Resource, actualRectangle, clip: false); emitter.Post(croppedSharedDepthImage, envelope.OriginatingTime); - } - }, deliveryPolicy); - } - - /// - /// Crops a shared depth image using the specified nullable rectangle. When no rectangle is specified, produces a null image. - /// - /// Source of depth image and rectangle messages. + } + }, + deliveryPolicy, + name); + } + + /// + /// Crops a shared depth image using the specified nullable rectangle. When no rectangle is specified, produces a null image. + /// + /// Source of depth image and rectangle messages. + /// Depth value semantics. + /// Scale factor to convert from depth values to meters. /// An optional parameter indicating whether to clip the region (by default false). - /// An optional delivery policy. - /// Optional image allocator to create new shared depth image. - /// Returns a producer generating new cropped image samples. - public static IProducer> Crop( - this IProducer<(Shared, Rectangle?)> source, - bool clip = false, - DeliveryPolicy<(Shared, Rectangle?)> deliveryPolicy = null, - Func> sharedDepthImageAllocator = null) - { - sharedDepthImageAllocator ??= DepthImagePool.GetOrCreate; - return source.Process<(Shared, Rectangle?), Shared>( - (tupleOfSharedDepthImageAndRectangle, envelope, emitter) => - { - (var sharedDepthImage, var rectangle) = tupleOfSharedDepthImageAndRectangle; + /// An optional delivery policy. + /// Optional image allocator to create new shared depth image. + /// An optional name for the stream operator. + /// Returns a producer generating new cropped image samples. + public static IProducer> Crop( + this IProducer<(Shared, Rectangle?)> source, + DepthValueSemantics depthValueSemantics, + double depthValueToMetersScaleFactor, + bool clip = false, + DeliveryPolicy<(Shared, Rectangle?)> deliveryPolicy = null, + Func> sharedDepthImageAllocator = null, + string name = nameof(Crop)) + { + sharedDepthImageAllocator ??= DepthImagePool.GetOrCreate; + return source.Process<(Shared, Rectangle?), Shared>( + (tupleOfSharedDepthImageAndRectangle, envelope, emitter) => + { + (var sharedDepthImage, var rectangle) = tupleOfSharedDepthImageAndRectangle; if (rectangle.HasValue) { var actualRectangle = clip ? GetImageSizeClippedRectangle(rectangle.Value, sharedDepthImage.Resource.Width, sharedDepthImage.Resource.Height) : rectangle.Value; @@ -234,428 +293,598 @@ public static IProducer> Transform(this IProducer> s } else { - using var croppedSharedDepthImage = sharedDepthImageAllocator(actualRectangle.Width, actualRectangle.Height); + using var croppedSharedDepthImage = sharedDepthImageAllocator( + actualRectangle.Width, + actualRectangle.Height, + depthValueSemantics, + depthValueToMetersScaleFactor); sharedDepthImage.Resource.Crop(croppedSharedDepthImage.Resource, actualRectangle, clip: false); emitter.Post(croppedSharedDepthImage, envelope.OriginatingTime); } - } + } else { emitter.Post(null, envelope.OriginatingTime); - } - }, deliveryPolicy); - } - - /// - /// Convert a producer of depth images into pseudo-colorized images, where more distant pixels are blue, and closer pixels are reddish. - /// - /// Source producer of depth images. - /// A tuple indicating the range (MinValue, MaxValue) of the depth values in the image. + } + }, + deliveryPolicy, + name); + } + + /// + /// Convert a producer of depth images into pseudo-colorized images, where more distant pixels are blue, and closer pixels are reddish. + /// + /// Source producer of depth images. + /// A tuple indicating the range (MinValue, MaxValue) of the depth values in the image. /// Indicates invalid depth values. These values are left black, or set to transparent based on the invalidAsTransparent parameter. /// Indicates whether to render invalid values as transparent in the image. - /// An optional delivery policy. - /// Optional image allocator to create new shared images (in format). - /// A producer of pseudo-colorized images. - public static IProducer> PseudoColorize( - this IProducer> source, - (ushort MinValue, ushort MaxValue) range, - ushort? invalidValue = null, - bool invalidAsTransparent = false, - DeliveryPolicy> deliveryPolicy = null, - Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= (width, height) => ImagePool.GetOrCreate(width, height, PixelFormat.BGRA_32bpp); - return source.Process, Shared>( - (sharedDepthImage, envelope, emitter) => - { - using var colorizedImage = sharedImageAllocator(sharedDepthImage.Resource.Width, sharedDepthImage.Resource.Height); - sharedDepthImage.Resource.PseudoColorize(colorizedImage.Resource, range, invalidValue, invalidAsTransparent); - emitter.Post(colorizedImage, envelope.OriginatingTime); - }, deliveryPolicy); - } - - /// - /// Resizes a shared image. - /// - /// Image to scale. - /// Final width of desired output. - /// Final height of desired output. - /// Method for sampling pixels when rescaling. - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Returns a producer that generates resized images. - public static IProducer> Resize(this IProducer> source, float finalWidth, float finalHeight, SamplingMode samplingMode = SamplingMode.Bilinear, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process, Shared>( - (sharedImage, envelope, emitter) => - { - using var resizedSharedImage = sharedImageAllocator((int)finalWidth, (int)finalHeight, sharedImage.Resource.PixelFormat); - sharedImage.Resource.Resize(resizedSharedImage.Resource, finalWidth, finalHeight, samplingMode); - emitter.Post(resizedSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); - } - - /// - /// Scales a shared by the specified scale factors. - /// - /// Image to scale. - /// Scale factor for X. - /// Scale factor for Y. - /// Method for sampling pixels when rescaling. - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Returns a producer that generates resized images. - public static IProducer> Scale(this IProducer> source, float scaleX, float scaleY, SamplingMode samplingMode = SamplingMode.Bilinear, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process, Shared>( - (sharedImage, envelope, emitter) => - { - int finalWidth = (int)(sharedImage.Resource.Width * scaleX); - int finalHeight = (int)(sharedImage.Resource.Height * scaleY); - using var scaledSharedImage = sharedImageAllocator(finalWidth, finalHeight, sharedImage.Resource.PixelFormat); - sharedImage.Resource.Scale(scaledSharedImage.Resource, scaleX, scaleY, samplingMode); - emitter.Post(scaledSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); - } - - /// - /// Flips a shared image about the horizontal or vertical axis. - /// - /// Image to flip. - /// Axis about which to flip. - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// A producer that generates flip images. - public static IProducer> Flip(this IProducer> source, FlipMode mode, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - if (mode == FlipMode.None) - { - // just post original image in the case of a no-op - return source; - } - else - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process, Shared>( - (sharedImage, envelope, emitter) => - { - using var flippedSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); - sharedImage.Resource.Flip(flippedSharedImage.Resource, mode); - emitter.Post(flippedSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); - } - } - - /// - /// Rotates a shared image by the specified angle. - /// - /// Image to rotate. - /// Angle for rotation specified in degrees. - /// Sampling mode to use when sampling pixels. - /// Used to describe the fit of the output image. Tight=output image is cropped to match exactly the required size. Loose=output image will be maximum size possible (i.e. length of source image diagonal). - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Returns a producer that generates rotated images. - public static IProducer> Rotate(this IProducer> source, float angleInDegrees, SamplingMode samplingMode, RotationFitMode fit = RotationFitMode.Tight, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process, Shared>( - (sharedImage, envelope, emitter) => - { - DetermineRotatedWidthHeight( - sharedImage.Resource.Width, - sharedImage.Resource.Height, - angleInDegrees, - fit, - out int rotatedWidth, - out int rotateHeight, - out float originx, - out float originy); - using var rotatedSharedImage = sharedImageAllocator(rotatedWidth, rotateHeight, sharedImage.Resource.PixelFormat); - rotatedSharedImage.Resource.Clear(Color.Black); - sharedImage.Resource.Rotate(rotatedSharedImage.Resource, angleInDegrees, samplingMode, fit); - emitter.Post(rotatedSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); - } - - /// - /// Draws a rectangle over a shared image. - /// - /// Image to draw rectangle on. - /// Pixel coordinates for rectangle. - /// Color to use when drawing the rectangle. - /// Line width (in pixels) of each side of the rectangle. - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Returns a producer that generates images overdrawn with a rectangle. - public static IProducer> DrawRectangle(this IProducer> source, Rectangle rect, Color color, int width, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process, Shared>( - (sharedImage, envelope, emitter) => - { - using var drawRectSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); - drawRectSharedImage.Resource.CopyFrom(sharedImage.Resource); - drawRectSharedImage.Resource.DrawRectangle(rect, color, width); - emitter.Post(drawRectSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); - } - - /// - /// Draws a line over a shared image. - /// - /// Image to draw line on. - /// Pixel coordinates for one end of the line. - /// Pixel coordinates for the other end of the line. - /// Color to use when drawing the line. - /// Line width (in pixels). - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Returns a producer that generates images overdrawn with a line. - public static IProducer> DrawLine(this IProducer> source, Point p0, Point p1, Color color, int width, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process, Shared>( - (sharedImage, envelope, emitter) => - { - using var drawLineSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); - drawLineSharedImage.Resource.CopyFrom(sharedImage.Resource); - drawLineSharedImage.Resource.DrawLine(p0, p1, color, width); - emitter.Post(drawLineSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); - } - - /// - /// Draws a circle over a shared image. - /// - /// Image to draw circle on. - /// Center of circle (in pixels). - /// Radius of circle (in pixels). - /// Color to use when drawing the circle. - /// Line width (in pixels). - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Returns a producer that generates images overdrawn with a circle. - public static IProducer> DrawCircle(this IProducer> source, Point p0, int radius, Color color, int width, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process, Shared>( - (sharedImage, envelope, emitter) => - { - using var drawCircleSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); - drawCircleSharedImage.Resource.CopyFrom(sharedImage.Resource); - drawCircleSharedImage.Resource.DrawCircle(p0, radius, color, width); - emitter.Post(drawCircleSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); - } - - /// - /// Draws a piece of text over a shared image. - /// - /// Image to draw text on. - /// Text to render. - /// Coordinates for start of text (in pixels). - /// Color to use while drawing text. - /// Name of font to use. Optional. - /// Size of font. Optional. - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Returns a producer that generates images overdrawn with text. - public static IProducer> DrawText(this IProducer> source, string text, Point p0, Color color, string font = null, float fontSize = 24.0f, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process, Shared>( - (sharedImage, envelope, emitter) => - { - using var drawTextSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); - drawTextSharedImage.Resource.CopyFrom(sharedImage.Resource); - drawTextSharedImage.Resource.DrawText(text, p0, color, font, fontSize); - emitter.Post(drawTextSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); + /// An optional delivery policy. + /// Optional image allocator to create new shared images (in format). + /// An optional name for the stream operator. + /// A producer of pseudo-colorized images. + public static IProducer> PseudoColorize( + this IProducer> source, + (ushort MinValue, ushort MaxValue) range, + ushort? invalidValue = null, + bool invalidAsTransparent = false, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(PseudoColorize)) + { + sharedImageAllocator ??= (width, height) => ImagePool.GetOrCreate(width, height, PixelFormat.BGRA_32bpp); + return source.Process, Shared>( + (sharedDepthImage, envelope, emitter) => + { + using var colorizedImage = sharedImageAllocator(sharedDepthImage.Resource.Width, sharedDepthImage.Resource.Height); + sharedDepthImage.Resource.PseudoColorize(colorizedImage.Resource, range, invalidValue, invalidAsTransparent); + emitter.Post(colorizedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + /// + /// Resizes a shared image. + /// + /// Image to scale. + /// Final width of desired output. + /// Final height of desired output. + /// Method for sampling pixels when rescaling. + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Returns a producer that generates resized images. + public static IProducer> Resize( + this IProducer> source, + float finalWidth, + float finalHeight, + SamplingMode samplingMode = SamplingMode.Bilinear, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(Resize)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process, Shared>( + (sharedImage, envelope, emitter) => + { + using var resizedSharedImage = sharedImageAllocator((int)finalWidth, (int)finalHeight, sharedImage.Resource.PixelFormat); + sharedImage.Resource.Resize(resizedSharedImage.Resource, finalWidth, finalHeight, samplingMode); + emitter.Post(resizedSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + /// + /// Scales a shared by the specified scale factors. + /// + /// Image to scale. + /// Scale factor for X. + /// Scale factor for Y. + /// Method for sampling pixels when rescaling. + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Returns a producer that generates resized images. + public static IProducer> Scale( + this IProducer> source, + float scaleX, + float scaleY, + SamplingMode samplingMode = SamplingMode.Bilinear, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(Scale)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process, Shared>( + (sharedImage, envelope, emitter) => + { + int finalWidth = (int)(sharedImage.Resource.Width * scaleX); + int finalHeight = (int)(sharedImage.Resource.Height * scaleY); + using var scaledSharedImage = sharedImageAllocator(finalWidth, finalHeight, sharedImage.Resource.PixelFormat); + sharedImage.Resource.Scale(scaledSharedImage.Resource, scaleX, scaleY, samplingMode); + emitter.Post(scaledSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + /// + /// Flips a shared image about the horizontal or vertical axis. + /// + /// Image to flip. + /// Axis about which to flip. + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// A producer that generates flip images. + public static IProducer> Flip( + this IProducer> source, + FlipMode mode, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(Flip)) + { + if (mode == FlipMode.None) + { + // just post original image in the case of a no-op + return source; + } + else + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process, Shared>( + (sharedImage, envelope, emitter) => + { + using var flippedSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); + sharedImage.Resource.Flip(flippedSharedImage.Resource, mode); + emitter.Post(flippedSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + } + + /// + /// Rotates a shared image by the specified angle. + /// + /// Image to rotate. + /// Angle for rotation specified in degrees. + /// Sampling mode to use when sampling pixels. + /// Used to describe the fit of the output image. Tight=output image is cropped to match exactly the required size. Loose=output image will be maximum size possible (i.e. length of source image diagonal). + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Returns a producer that generates rotated images. + public static IProducer> Rotate( + this IProducer> source, + float angleInDegrees, + SamplingMode samplingMode, + RotationFitMode fit = RotationFitMode.Tight, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(Rotate)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process, Shared>( + (sharedImage, envelope, emitter) => + { + DetermineRotatedWidthHeight( + sharedImage.Resource.Width, + sharedImage.Resource.Height, + angleInDegrees, + fit, + out int rotatedWidth, + out int rotateHeight, + out float originx, + out float originy); + using var rotatedSharedImage = sharedImageAllocator(rotatedWidth, rotateHeight, sharedImage.Resource.PixelFormat); + rotatedSharedImage.Resource.Clear(Color.Black); + sharedImage.Resource.Rotate(rotatedSharedImage.Resource, angleInDegrees, samplingMode, fit); + emitter.Post(rotatedSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + /// + /// Draws a rectangle over a shared image. + /// + /// Image to draw rectangle on. + /// Pixel coordinates for rectangle. + /// Color to use when drawing the rectangle. + /// Line width (in pixels) of each side of the rectangle. + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Returns a producer that generates images overdrawn with a rectangle. + public static IProducer> DrawRectangle( + this IProducer> source, + Rectangle rect, + Color color, + int width, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(DrawRectangle)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process, Shared>( + (sharedImage, envelope, emitter) => + { + using var drawRectSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); + drawRectSharedImage.Resource.CopyFrom(sharedImage.Resource); + drawRectSharedImage.Resource.DrawRectangle(rect, color, width); + emitter.Post(drawRectSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + /// + /// Draws a line over a shared image. + /// + /// Image to draw line on. + /// Pixel coordinates for one end of the line. + /// Pixel coordinates for the other end of the line. + /// Color to use when drawing the line. + /// Line width (in pixels). + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Returns a producer that generates images overdrawn with a line. + public static IProducer> DrawLine( + this IProducer> source, + Point p0, + Point p1, + Color color, + int width, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(DrawLine)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process, Shared>( + (sharedImage, envelope, emitter) => + { + using var drawLineSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); + drawLineSharedImage.Resource.CopyFrom(sharedImage.Resource); + drawLineSharedImage.Resource.DrawLine(p0, p1, color, width); + emitter.Post(drawLineSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + /// + /// Draws a circle over a shared image. + /// + /// Image to draw circle on. + /// Center of circle (in pixels). + /// Radius of circle (in pixels). + /// Color to use when drawing the circle. + /// Line width (in pixels). + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Returns a producer that generates images overdrawn with a circle. + public static IProducer> DrawCircle( + this IProducer> source, + Point p0, + int radius, + Color color, + int width, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(DrawCircle)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process, Shared>( + (sharedImage, envelope, emitter) => + { + using var drawCircleSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); + drawCircleSharedImage.Resource.CopyFrom(sharedImage.Resource); + drawCircleSharedImage.Resource.DrawCircle(p0, radius, color, width); + emitter.Post(drawCircleSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + /// + /// Draws a piece of text over a shared image. + /// + /// Image to draw text on. + /// Text to render. + /// Coordinates for start of text (in pixels). + /// Color to use while drawing text. + /// Name of font to use. Optional. + /// Size of font. Optional. + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Returns a producer that generates images overdrawn with text. + public static IProducer> DrawText( + this IProducer> source, + string text, + Point p0, + Color color, + string font = null, + float fontSize = 24.0f, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(DrawText)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process, Shared>( + (sharedImage, envelope, emitter) => + { + using var drawTextSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); + drawTextSharedImage.Resource.CopyFrom(sharedImage.Resource); + drawTextSharedImage.Resource.DrawText(text, p0, color, font, fontSize); + emitter.Post(drawTextSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + /// + /// Draws a piece of text with background over a shared image. + /// + /// Image to draw text on. + /// Text to render. + /// Coordinates for start of text (in pixels). + /// Background color to use when drawing text. + /// Color to use to draw the text. + /// Name of font to use. Optional. + /// Size of font. Optional. + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Returns a producer that generates images overdrawn with text. + public static IProducer> DrawText( + this IProducer> source, + string text, + Point p0, + Color backgroundColor, + Color textColor, + string font = null, + float fontSize = 24.0f, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(DrawText)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process, Shared>( + (sharedImage, envelope, emitter) => + { + using var drawTextSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); + drawTextSharedImage.Resource.CopyFrom(sharedImage.Resource); + drawTextSharedImage.Resource.DrawText(text, p0, backgroundColor, textColor, font, fontSize); + emitter.Post(drawTextSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + /// + /// Fills a rectangle over a shared image. + /// + /// Image to draw rectangle on. + /// Pixel coordinates for rectangle. + /// Color to use when drawing the rectangle. + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Returns a producer that generates images overdrawn with a rectangle. + public static IProducer> FillRectangle( + this IProducer> source, + Rectangle rect, + Color color, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(FillRectangle)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process, Shared>( + (sharedImage, envelope, emitter) => + { + using var drawRectSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); + drawRectSharedImage.Resource.CopyFrom(sharedImage.Resource); + drawRectSharedImage.Resource.FillRectangle(rect, color); + emitter.Post(drawRectSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + /// + /// Fills a circle over a shared image. + /// + /// Image to draw circle on. + /// Center of circle (in pixels). + /// Radius of circle (in pixels). + /// Color to use when drawing the circle. + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Returns a producer that generates images overdrawn with a circle. + public static IProducer> FillCircle( + this IProducer> source, + Point p0, + int radius, + Color color, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(FillCircle)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process, Shared>( + (sharedImage, envelope, emitter) => + { + using var drawCircleSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); + drawCircleSharedImage.Resource.CopyFrom(sharedImage.Resource); + drawCircleSharedImage.Resource.FillCircle(p0, radius, color); + emitter.Post(drawCircleSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); } - /// - /// Draws a piece of text with background over a shared image. - /// - /// Image to draw text on. - /// Text to render. - /// Coordinates for start of text (in pixels). - /// Background color to use when drawing text. - /// Color to use to draw the text. - /// Name of font to use. Optional. - /// Size of font. Optional. - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Returns a producer that generates images overdrawn with text. - public static IProducer> DrawText(this IProducer> source, string text, Point p0, Color backgroundColor, Color textColor, string font = null, float fontSize = 24.0f, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process, Shared>( - (sharedImage, envelope, emitter) => - { - using var drawTextSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); - drawTextSharedImage.Resource.CopyFrom(sharedImage.Resource); - drawTextSharedImage.Resource.DrawText(text, p0, backgroundColor, textColor, font, fontSize); - emitter.Post(drawTextSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); + /// + /// Inverts each color channel in a shared image. + /// + /// Images to invert. + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Producer that returns the inverted image. + public static IProducer> Invert( + this IProducer> source, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(Invert)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process, Shared>( + (sourceImage, envelope, emitter) => + { + using var invertedSharedImage = sharedImageAllocator(sourceImage.Resource.Width, sourceImage.Resource.Height, sourceImage.Resource.PixelFormat); + sourceImage.Resource.Invert(invertedSharedImage.Resource); + emitter.Post(invertedSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); } - /// - /// Fills a rectangle over a shared image. - /// - /// Image to draw rectangle on. - /// Pixel coordinates for rectangle. - /// Color to use when drawing the rectangle. - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Returns a producer that generates images overdrawn with a rectangle. - public static IProducer> FillRectangle(this IProducer> source, Rectangle rect, Color color, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process, Shared>( - (sharedImage, envelope, emitter) => - { - using var drawRectSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); - drawRectSharedImage.Resource.CopyFrom(sharedImage.Resource); - drawRectSharedImage.Resource.FillRectangle(rect, color); - emitter.Post(drawRectSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); + /// + /// Clears a shared image to the specified color. + /// + /// Images to clear. + /// Color to set image to. + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Producer that returns the cleared image. + public static IProducer> Clear( + this IProducer> source, + Color color, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(Clear)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process, Shared>( + (sourceImage, envelope, emitter) => + { + using var clearedSharedImage = sharedImageAllocator(sourceImage.Resource.Width, sourceImage.Resource.Height, sourceImage.Resource.PixelFormat); + clearedSharedImage.Resource.Clear(color); + emitter.Post(clearedSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); } - /// - /// Fills a circle over a shared image. - /// - /// Image to draw circle on. - /// Center of circle (in pixels). - /// Radius of circle (in pixels). - /// Color to use when drawing the circle. - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Returns a producer that generates images overdrawn with a circle. - public static IProducer> FillCircle(this IProducer> source, Point p0, int radius, Color color, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process, Shared>( - (sharedImage, envelope, emitter) => - { - using var drawCircleSharedImage = sharedImageAllocator(sharedImage.Resource.Width, sharedImage.Resource.Height, sharedImage.Resource.PixelFormat); - drawCircleSharedImage.Resource.CopyFrom(sharedImage.Resource); - drawCircleSharedImage.Resource.FillCircle(p0, radius, color); - emitter.Post(drawCircleSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); - } - - /// - /// Inverts each color channel in a shared image. - /// - /// Images to invert. - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Producer that returns the inverted image. - public static IProducer> Invert(this IProducer> source, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process, Shared>( - (sourceImage, envelope, emitter) => - { - using var invertedSharedImage = sharedImageAllocator(sourceImage.Resource.Width, sourceImage.Resource.Height, sourceImage.Resource.PixelFormat); - sourceImage.Resource.Invert(invertedSharedImage.Resource); - emitter.Post(invertedSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); - } - - /// - /// Clears a shared image to the specified color. - /// - /// Images to clear. - /// Color to set image to. - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Producer that returns the cleared image. - public static IProducer> Clear(this IProducer> source, Color clr, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process, Shared>( - (sourceImage, envelope, emitter) => - { - using var clearedSharedImage = sharedImageAllocator(sourceImage.Resource.Width, sourceImage.Resource.Height, sourceImage.Resource.PixelFormat); - clearedSharedImage.Resource.Clear(clr); - emitter.Post(clearedSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); - } - - /// - /// Extracts a color channel from a shared image. Returned image is of type Gray_8bpp. - /// - /// Images to extract channel from. - /// Index of which channel to extract. - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Producer that returns the extracted channel as a gray scale image. - public static IProducer> ExtractChannel(this IProducer> source, int channel, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return source.Process, Shared>( - (sourceImage, envelope, emitter) => - { - using var channelSharedImage = sharedImageAllocator(sourceImage.Resource.Width, sourceImage.Resource.Height, PixelFormat.Gray_8bpp); - sourceImage.Resource.ExtractChannel(channelSharedImage.Resource, channel); - emitter.Post(channelSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); - } - - /// - /// Computes the absolute difference between two images. - /// - /// Images to diff. - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Producer that returns the difference image. - public static IProducer> AbsDiff(this IProducer<(Shared, Shared)> sources, DeliveryPolicy<(Shared, Shared)> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return sources.Process<(Shared, Shared), Shared>( - (tupleOfSharedImages, envelope, emitter) => - { - using var absdiffSharedImage = sharedImageAllocator(tupleOfSharedImages.Item1.Resource.Width, tupleOfSharedImages.Item1.Resource.Height, tupleOfSharedImages.Item1.Resource.PixelFormat); - tupleOfSharedImages.Item1.Resource.AbsDiff(tupleOfSharedImages.Item2.Resource, absdiffSharedImage.Resource); - emitter.Post(absdiffSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); - } - - /// - /// Thresholds the image. See Threshold for what modes of thresholding are available. - /// - /// Images to threshold. - /// Threshold value. - /// Maximum value. - /// Type of thresholding to perform. - /// An optional delivery policy. - /// Optional image allocator to create new shared image. - /// Producer that returns the difference image. - public static IProducer> Threshold(this IProducer> image, int threshold, int maxvalue, Threshold thresholdType, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) - { - sharedImageAllocator ??= ImagePool.GetOrCreate; - return image.Process, Shared>( - (sharedSourceImage, envelope, emitter) => - { - using var thresholdSharedImage = sharedImageAllocator(sharedSourceImage.Resource.Width, sharedSourceImage.Resource.Height, sharedSourceImage.Resource.PixelFormat); - sharedSourceImage.Resource.Threshold(thresholdSharedImage.Resource, threshold, maxvalue, thresholdType); - emitter.Post(thresholdSharedImage, envelope.OriginatingTime); - }, deliveryPolicy); - } - - /// + /// + /// Extracts a color channel from a shared image. Returned image is of type Gray_8bpp. + /// + /// Images to extract channel from. + /// Index of which channel to extract. + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Producer that returns the extracted channel as a gray scale image. + public static IProducer> ExtractChannel( + this IProducer> source, + int channel, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(ExtractChannel)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return source.Process, Shared>( + (sourceImage, envelope, emitter) => + { + using var channelSharedImage = sharedImageAllocator(sourceImage.Resource.Width, sourceImage.Resource.Height, PixelFormat.Gray_8bpp); + sourceImage.Resource.ExtractChannel(channelSharedImage.Resource, channel); + emitter.Post(channelSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + /// + /// Computes the absolute difference between two images. + /// + /// Images to diff. + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Producer that returns the difference image. + public static IProducer> AbsDiff( + this IProducer<(Shared, Shared)> sources, + DeliveryPolicy<(Shared, Shared)> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(AbsDiff)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return sources.Process<(Shared, Shared), Shared>( + (tupleOfSharedImages, envelope, emitter) => + { + using var absdiffSharedImage = sharedImageAllocator(tupleOfSharedImages.Item1.Resource.Width, tupleOfSharedImages.Item1.Resource.Height, tupleOfSharedImages.Item1.Resource.PixelFormat); + tupleOfSharedImages.Item1.Resource.AbsDiff(tupleOfSharedImages.Item2.Resource, absdiffSharedImage.Resource); + emitter.Post(absdiffSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + /// + /// Thresholds the image. See for what modes of thresholding are available. + /// + /// Images to threshold. + /// Threshold value. + /// Maximum value. + /// Type of thresholding to perform. + /// An optional delivery policy. + /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. + /// Producer that returns the difference image. + public static IProducer> Threshold( + this IProducer> image, + int threshold, + int maxvalue, + Threshold thresholdType, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(Threshold)) + { + sharedImageAllocator ??= ImagePool.GetOrCreate; + return image.Process, Shared>( + (sharedSourceImage, envelope, emitter) => + { + using var thresholdSharedImage = sharedImageAllocator(sharedSourceImage.Resource.Width, sharedSourceImage.Resource.Height, sharedSourceImage.Resource.PixelFormat); + sharedSourceImage.Resource.Threshold(thresholdSharedImage.Resource, threshold, maxvalue, thresholdType); + emitter.Post(thresholdSharedImage, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + /// /// Convolves the image with a specified kernel. /// /// The stream of images. /// The kernel to use. /// An optional delivery policy. /// Optional image allocator to create new shared image. + /// An optional name for the stream operator. /// A stream containing the results of the convolution. - public static IProducer> Convolve(this IProducer> image, int[,] kernel, DeliveryPolicy> deliveryPolicy = null, Func> sharedImageAllocator = null) + public static IProducer> Convolve( + this IProducer> image, + int[,] kernel, + DeliveryPolicy> deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(Convolve)) { sharedImageAllocator ??= ImagePool.GetOrCreate; return image.Process, Shared>( @@ -671,127 +900,183 @@ public static IProducer> Convolve(this IProducer> im sharedSourceImage.Resource.Convolve(destinationImage.Resource, kernel); emitter.Post(destinationImage, envelope.OriginatingTime); } - }, deliveryPolicy); + }, + deliveryPolicy, + name); } /// - /// Encodes a shared image using a specified encoder component. - /// - /// A producer of images to encode. - /// Constructor function that returns an encoder component given a pipeline. - /// An optional delivery policy. - /// A producer that generates the encoded images. - public static IProducer> Encode( - this IProducer> source, - Func, Shared>> encoderConstructor, - DeliveryPolicy> deliveryPolicy = null) - { - return source.PipeTo(encoderConstructor(source.Out.Pipeline), deliveryPolicy); - } - - /// - /// Encodes a shared image using a specified image-to-stream encoder. - /// - /// A producer of images to encode. - /// The image-to-stream encoder to use when encoding images. - /// An optional delivery policy. - /// A producer that generates the encoded images. - public static IProducer> Encode( - this IProducer> source, - IImageToStreamEncoder encoder, - DeliveryPolicy> deliveryPolicy = null) - { - return source.Encode(p => new ImageEncoder(p, encoder), deliveryPolicy); - } - - /// - /// Decodes an encoded image using a specified decoder component. - /// - /// A producer of images to decode. - /// Constructor function that returns a decoder component given a pipeline. - /// An optional delivery policy. - /// A producer that generates the decoded images. - public static IProducer> Decode( - this IProducer> source, - Func, Shared>> decoderConstructor, - DeliveryPolicy> deliveryPolicy = null) - { - return source.PipeTo(decoderConstructor(source.Out.Pipeline), deliveryPolicy); - } - - /// - /// Decodes an encoded image using a specified image-from-stream decoder. - /// - /// A producer of images to decode. - /// The image-from-stream decoder to use when decoding images. - /// An optional delivery policy. - /// A producer that generates the decoded images. - public static IProducer> Decode( - this IProducer> source, - IImageFromStreamDecoder decoder, - DeliveryPolicy> deliveryPolicy = null) - { - return source.Decode(p => new ImageDecoder(p, decoder), deliveryPolicy); - } - - /// - /// Encodes a depth image using a specified encoder component. - /// - /// A producer of depth images to encode. - /// Constructor function that returns an encoder component given a pipeline. - /// An optional delivery policy. - /// A producer that generates the encoded depth images. - public static IProducer> Encode( - this IProducer> source, - Func, Shared>> encoderConstructor, - DeliveryPolicy> deliveryPolicy = null) - { - return source.PipeTo(encoderConstructor(source.Out.Pipeline), deliveryPolicy); - } - - /// - /// Encodes a depth image using a specified depth-image-to-stream encoder. - /// - /// A producer of depth images to encode. - /// The depth image encoder to use. - /// An optional delivery policy. - /// A producer that generates the encoded depth images. - public static IProducer> Encode( - this IProducer> source, - IDepthImageToStreamEncoder encoder, - DeliveryPolicy> deliveryPolicy = null) - { - return source.Encode(p => new DepthImageEncoder(p, encoder), deliveryPolicy); - } - - /// - /// Decodes a depth image using a specified decoder component. - /// - /// A producer of depth images to decode. - /// Constructor function that returns a decoder component given a pipeline. - /// An optional delivery policy. - /// A producer that generates the decoded depth images. - public static IProducer> Decode( - this IProducer> source, - Func, Shared>> decoderConstructor, - DeliveryPolicy> deliveryPolicy = null) - { - return source.PipeTo(decoderConstructor(source.Out.Pipeline), deliveryPolicy); - } - - /// - /// Decodes a depth image using a specified depth-image-from-stream decoder. - /// - /// A producer of depth images to decode. - /// The depth image decoder to use. - /// An optional delivery policy. - /// A producer that generates the decoded depth images. - public static IProducer> Decode( - this IProducer> source, - IDepthImageFromStreamDecoder decoder, - DeliveryPolicy> deliveryPolicy = null) - { - return source.Decode(p => new DepthImageDecoder(p, decoder), deliveryPolicy); - } - } + /// Encodes a shared image using a specified encoder component. + /// + /// A producer of images to encode. + /// Constructor function that returns an encoder component given a pipeline. + /// An optional delivery policy. + /// A producer that generates the encoded images. + public static IProducer> Encode( + this IProducer> source, + Func, Shared>> encoderConstructor, + DeliveryPolicy> deliveryPolicy = null) + => source.PipeTo(encoderConstructor(source.Out.Pipeline), deliveryPolicy); + + /// + /// Encodes a shared image using a specified named encoder component. + /// + /// A producer of images to encode. + /// Constructor function that returns a named encoder component given a pipeline and a component name. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A producer that generates the encoded images. + public static IProducer> Encode( + this IProducer> source, + Func, Shared>> encoderConstructor, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(Encode)) + => source.PipeTo(encoderConstructor(source.Out.Pipeline, name), deliveryPolicy); + + /// + /// Encodes a shared image using a specified image-to-stream encoder. + /// + /// A producer of images to encode. + /// The image-to-stream encoder to use when encoding images. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A producer that generates the encoded images. + public static IProducer> Encode( + this IProducer> source, + IImageToStreamEncoder encoder, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(Encode)) + => source.Encode(p => new ImageEncoder(p, encoder, name), deliveryPolicy); + + /// + /// Decodes an encoded image using a specified decoder component. + /// + /// A producer of images to decode. + /// Constructor function that returns a decoder component given a pipeline. + /// An optional delivery policy. + /// A producer that generates the decoded images. + public static IProducer> Decode( + this IProducer> source, + Func, Shared>> decoderConstructor, + DeliveryPolicy> deliveryPolicy = null) + => source.PipeTo(decoderConstructor(source.Out.Pipeline), deliveryPolicy); + + /// + /// Decodes an encoded image using a specified named decoder component. + /// + /// A producer of images to decode. + /// Constructor function that returns a named decoder component given a pipeline and a component name. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A producer that generates the decoded images. + public static IProducer> Decode( + this IProducer> source, + Func, Shared>> decoderConstructor, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(Decode)) + => source.PipeTo(decoderConstructor(source.Out.Pipeline, name), deliveryPolicy); + + /// + /// Decodes an encoded image using a specified image-from-stream decoder. + /// + /// A producer of images to decode. + /// The image-from-stream decoder to use when decoding images. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A producer that generates the decoded images. + public static IProducer> Decode( + this IProducer> source, + IImageFromStreamDecoder decoder, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(Decode)) + => source.Decode(p => new ImageDecoder(p, decoder, name), deliveryPolicy); + + /// + /// Encodes a depth image using a specified encoder component. + /// + /// A producer of depth images to encode. + /// Constructor function that returns an encoder component given a pipeline. + /// An optional delivery policy. + /// A producer that generates the encoded depth images. + public static IProducer> Encode( + this IProducer> source, + Func, Shared>> encoderConstructor, + DeliveryPolicy> deliveryPolicy = null) + => source.PipeTo(encoderConstructor(source.Out.Pipeline), deliveryPolicy); + + /// + /// Encodes a depth image using a specified named encoder component. + /// + /// A producer of depth images to encode. + /// Constructor function that returns a named encoder component given a pipeline and a component name. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A producer that generates the encoded depth images. + public static IProducer> Encode( + this IProducer> source, + Func, Shared>> encoderConstructor, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(Encode)) + => source.PipeTo(encoderConstructor(source.Out.Pipeline, name), deliveryPolicy); + + /// + /// Encodes a depth image using a specified depth-image-to-stream encoder. + /// + /// A producer of depth images to encode. + /// The depth image encoder to use. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A producer that generates the encoded depth images. + public static IProducer> Encode( + this IProducer> source, + IDepthImageToStreamEncoder encoder, + DeliveryPolicy> deliveryPolicy = null, + string name = null) + => source.Encode(p => new DepthImageEncoder(p, encoder, name ?? $"{nameof(Encode)}({encoder.Description})"), deliveryPolicy); + + /// + /// Decodes a depth image using a specified decoder component. + /// + /// A producer of depth images to decode. + /// Constructor function that returns a decoder component given a pipeline. + /// An optional delivery policy. + /// A producer that generates the decoded depth images. + public static IProducer> Decode( + this IProducer> source, + Func, Shared>> decoderConstructor, + DeliveryPolicy> deliveryPolicy = null) + { + return source.PipeTo(decoderConstructor(source.Out.Pipeline), deliveryPolicy); + } + + /// + /// Decodes a depth image using a specified named decoder component. + /// + /// A producer of depth images to decode. + /// Constructor function that returns a named decoder component given a pipeline and a component name. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A producer that generates the decoded depth images. + public static IProducer> Decode( + this IProducer> source, + Func, Shared>> decoderConstructor, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(Decode)) + => source.PipeTo(decoderConstructor(source.Out.Pipeline, name), deliveryPolicy); + + /// + /// Decodes a depth image using a specified depth-image-from-stream decoder. + /// + /// A producer of depth images to decode. + /// The depth image decoder to use. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A producer that generates the decoded depth images. + public static IProducer> Decode( + this IProducer> source, + IDepthImageFromStreamDecoder decoder, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(Decode)) + => source.Decode(p => new DepthImageDecoder(p, decoder, name), deliveryPolicy); + } } \ No newline at end of file diff --git a/Sources/Imaging/Test.Psi.Imaging.Windows/Properties/AssemblyInfo.cs b/Sources/Imaging/Test.Psi.Imaging.Windows/Properties/AssemblyInfo.cs index 1222c1262..d3c9b752d 100644 --- a/Sources/Imaging/Test.Psi.Imaging.Windows/Properties/AssemblyInfo.cs +++ b/Sources/Imaging/Test.Psi.Imaging.Windows/Properties/AssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyCopyright("Copyright (c) Microsoft Corporation. All rights reserved.")] [assembly: ComVisible(false)] [assembly: Guid("191df615-3d8f-45a3-b763-dd4a604a712a")] -[assembly: AssemblyVersion("0.16.92.1")] -[assembly: AssemblyFileVersion("0.16.92.1")] -[assembly: AssemblyInformationalVersion("0.16.92.1-beta")] +[assembly: AssemblyVersion("0.17.52.1")] +[assembly: AssemblyFileVersion("0.17.52.1")] +[assembly: AssemblyInformationalVersion("0.17.52.1-beta")] diff --git a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Face/FaceRecognizer.cs b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Face/FaceRecognizer.cs index 9cafd07af..98ea622a3 100644 --- a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Face/FaceRecognizer.cs +++ b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Face/FaceRecognizer.cs @@ -24,7 +24,7 @@ namespace Microsoft.Psi.CognitiveServices.Face /// passed to the component via the configuration. For more information, and to see how to create person groups, see the full direct API for. /// Microsoft Cognitive Services Face API /// - public sealed class FaceRecognizer : AsyncConsumerProducer, IList>>, IDisposable + public sealed class FaceRecognizer : AsyncConsumerProducer, IList>>, IDisposable { /// /// Empty results. @@ -51,8 +51,9 @@ public sealed class FaceRecognizer : AsyncConsumerProducer, IList< /// /// The pipeline to add the component to. /// The component configuration. - public FaceRecognizer(Pipeline pipeline, FaceRecognizerConfiguration configuration) - : base(pipeline) + /// An optional name for the component. + public FaceRecognizer(Pipeline pipeline, FaceRecognizerConfiguration configuration, string name = nameof(FaceRecognizer)) + : base(pipeline, name) { this.configuration = configuration; this.RateLimitExceeded = pipeline.CreateEmitter(this, nameof(this.RateLimitExceeded)); @@ -84,39 +85,37 @@ public void Dispose() /// protected override async Task ReceiveAsync(Shared data, Envelope e) { - using (Stream imageFileStream = new MemoryStream()) + using Stream imageFileStream = new MemoryStream(); + try { - try - { - // convert image to a Stream and send to Cog Services - data.Resource.ToBitmap(false).Save(imageFileStream, ImageFormat.Jpeg); - imageFileStream.Seek(0, SeekOrigin.Begin); + // convert image to a Stream and send to Cog Services + data.Resource.ToBitmap(false).Save(imageFileStream, ImageFormat.Jpeg); + imageFileStream.Seek(0, SeekOrigin.Begin); - var detected = (await this.client.Face.DetectWithStreamAsync(imageFileStream, recognitionModel: this.configuration.RecognitionModelName)).Select(d => d.FaceId.Value).ToList(); + var detected = (await this.client.Face.DetectWithStreamAsync(imageFileStream, recognitionModel: this.configuration.RecognitionModelName)).Select(d => d.FaceId.Value).ToList(); - // Identify each face - if (detected.Count > 0) - { - var identified = await this.client.Face.IdentifyAsync(detected, this.configuration.PersonGroupId); - var results = identified.Select(p => (IList<(string, double)>)p.Candidates.Select(c => (this.people[c.PersonId].Name, c.Confidence)).ToList()).ToList(); - this.Out.Post(results, e.OriginatingTime); - } - else - { - this.Out.Post(Empty, e.OriginatingTime); - } + // Identify each face + if (detected.Count > 0) + { + var identified = await this.client.Face.IdentifyAsync(detected, this.configuration.PersonGroupId); + var results = identified.Select(p => (IList<(string, double)>)p.Candidates.Select(c => (this.people[c.PersonId].Name, c.Confidence)).ToList()).ToList(); + this.Out.Post(results, e.OriginatingTime); + } + else + { + this.Out.Post(Empty, e.OriginatingTime); + } + } + catch (APIErrorException exception) + { + // swallow exceptions unless it's a rate limit exceeded + if (exception.Body.Error.Code == "RateLimitExceeded") + { + this.RateLimitExceeded.Post(true, e.OriginatingTime); } - catch (APIErrorException exception) + else { - // swallow exceptions unless it's a rate limit exceeded - if (exception.Body.Error.Code == "RateLimitExceeded") - { - this.RateLimitExceeded.Post(true, e.OriginatingTime); - } - else - { - throw; - } + throw; } } } diff --git a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Face/Operators.cs b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Face/Operators.cs index c5ef71021..55b4c7771 100644 --- a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Face/Operators.cs +++ b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Face/Operators.cs @@ -17,6 +17,7 @@ public static class Operators /// The source stream of images. /// The face recognizer configuration. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of messages containing detected faces and candidate identities of each person in the image. /// /// A Microsoft Cognitive Services Face API @@ -24,11 +25,11 @@ public static class Operators /// passed to the operator via the configuration. For more information, and to see how to create person groups, see the full direct API for. /// Microsoft Cognitive Services Face API /// - public static IProducer>> RecognizeFace(this IProducer> source, FaceRecognizerConfiguration configuration, DeliveryPolicy> deliveryPolicy = null) - { - var faceRecognizer = new FaceRecognizer(source.Out.Pipeline, configuration); - source.PipeTo(faceRecognizer, deliveryPolicy); - return faceRecognizer.Out; - } + public static IProducer>> RecognizeFace( + this IProducer> source, + FaceRecognizerConfiguration configuration, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(RecognizeFace)) + => source.PipeTo(new FaceRecognizer(source.Out.Pipeline, configuration, name), deliveryPolicy); } } diff --git a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Language.Windows/PersonalityChat.cs b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Language.Windows/PersonalityChat.cs index 1ae55002c..ad9c262b2 100644 --- a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Language.Windows/PersonalityChat.cs +++ b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Language.Windows/PersonalityChat.cs @@ -31,8 +31,9 @@ public sealed class PersonalityChat : ConsumerProducer /// /// The pipeline to add the component to. /// The configuration parameters. - public PersonalityChat(Pipeline pipeline, PersonalityChatConfiguration configuration) - : base(pipeline) + /// An optional name for the component. + public PersonalityChat(Pipeline pipeline, PersonalityChatConfiguration configuration, string name = nameof(PersonalityChat)) + : base(pipeline, name) { this.configuration = configuration; var personalityChatOptions = new PersonalityChatOptions(configuration.PersonalityChatSubscriptionID, PersonalityChatPersona.Professional); diff --git a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Language/LUISIntentDetector.cs b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Language/LUISIntentDetector.cs index f874051ab..e60ef0b07 100644 --- a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Language/LUISIntentDetector.cs +++ b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Language/LUISIntentDetector.cs @@ -34,8 +34,9 @@ public sealed class LUISIntentDetector : AsyncConsumerProducer /// The pipeline to add the component to. /// The component configuration. - public LUISIntentDetector(Pipeline pipeline, LUISIntentDetectorConfiguration configuration) - : base(pipeline) + /// An optional name for the component. + public LUISIntentDetector(Pipeline pipeline, LUISIntentDetectorConfiguration configuration, string name = nameof(LUISIntentDetector)) + : base(pipeline, name) { this.configuration = configuration; } diff --git a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Speech/AzureSpeechRecognizer.cs b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Speech/AzureSpeechRecognizer.cs index 5beaef4e8..d481a4b9a 100644 --- a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Speech/AzureSpeechRecognizer.cs +++ b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Speech/AzureSpeechRecognizer.cs @@ -24,10 +24,23 @@ namespace Microsoft.Psi.CognitiveServices.Speech /// and requires a subscription key in order to use. For more information, see the complete documentation for the /// Microsoft Cognitive Services Azure Speech API. /// - public sealed class AzureSpeechRecognizer : AsyncConsumerProducer, IStreamingSpeechRecognitionResult>, IDisposable + public sealed class AzureSpeechRecognizer : AsyncConsumerProducer<(AudioBuffer, bool), IStreamingSpeechRecognitionResult>, IDisposable { // For canceling any pending recognition tasks before disposal - private readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + private readonly CancellationTokenSource cancellationTokenSource = new (); + + /// + /// The last originating time that was recorded for each output stream. + /// + private readonly Dictionary lastPostedOriginatingTimes; + + /// + /// Queue of current audio buffers for the pending recognition task. + /// + private readonly Queue<(AudioBuffer, bool)> currentQueue = new (); + + // The queue of pending recognition tasks + private readonly ConcurrentQueue pendingRecognitionTasks = new (); /// /// The client that communicates with the cloud speech recognition service. @@ -37,9 +50,6 @@ public sealed class AzureSpeechRecognizer : AsyncConsumerProducer pendingRecognitionTasks = new ConcurrentQueue(); - /// /// The time the last audio input contained speech. /// @@ -50,21 +60,11 @@ public sealed class AzureSpeechRecognizer : AsyncConsumerProducer private DateTime lastAudioOriginatingTime; - /// - /// The last originating time that was recorded for each output stream. - /// - private Dictionary lastPostedOriginatingTimes; - /// /// A flag indicating whether the last audio packet received contained speech. /// private bool lastAudioContainedSpeech = false; - /// - /// Queue of current audio buffers for the pending recognition task. - /// - private Queue> currentQueue = new Queue>(); - /// /// The last conversation error. /// @@ -80,8 +80,9 @@ public sealed class AzureSpeechRecognizer : AsyncConsumerProducer /// The pipeline to add the component to. /// The component configuration. - public AzureSpeechRecognizer(Pipeline pipeline, AzureSpeechRecognizerConfiguration configuration) - : base(pipeline) + /// An optional name for the component. + public AzureSpeechRecognizer(Pipeline pipeline, AzureSpeechRecognizerConfiguration configuration, string name = nameof(AzureSpeechRecognizer)) + : base(pipeline, name) { this.Configuration = configuration ?? new AzureSpeechRecognizerConfiguration(); this.PartialRecognitionResults = pipeline.CreateEmitter(this, nameof(this.PartialRecognitionResults)); @@ -161,7 +162,7 @@ public void Dispose() /// A message containing the combined VAD signal and audio data. /// The message envelope. /// The representing the asynchronous operation. - protected override async Task ReceiveAsync(ValueTuple data, Envelope e) + protected override async Task ReceiveAsync((AudioBuffer, bool) data, Envelope e) { byte[] audioData = data.Item1.Data; bool hasSpeech = data.Item2; @@ -220,21 +221,21 @@ protected override async Task ReceiveAsync(ValueTuple data, E // update the latest in-progress recognition // Allocate a buffer large enough to hold the buffered audio - BufferWriter bw = new BufferWriter(this.currentQueue.Sum(b => b.Item1.Length)); + var bufferWriter = new BufferWriter(this.currentQueue.Sum(b => b.Item1.Length)); // Get the audio associated with the recognized text from the current queue. - ValueTuple buffer; + (AudioBuffer, bool) buffer; while (this.currentQueue.Count > 0) { buffer = this.currentQueue.Dequeue(); - bw.Write(buffer.Item1.Data); + bufferWriter.Write(buffer.Item1.Data); // We are done with this buffer so enqueue it for recycling this.In.Recycle(buffer); } // Save the buffered audio - this.currentRecognitionTask.Audio = new AudioBuffer(bw.Buffer, this.Configuration.InputFormat); + this.currentRecognitionTask.Audio = new AudioBuffer(bufferWriter.Buffer, this.Configuration.InputFormat); this.currentRecognitionTask.SpeechEndTime = lastVADSpeechEndTime; // Call EndAudio to signal that this is the last packet diff --git a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Speech/BingSpeechRecognizer.cs b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Speech/BingSpeechRecognizer.cs index a38864747..4ad20f2a2 100644 --- a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Speech/BingSpeechRecognizer.cs +++ b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Speech/BingSpeechRecognizer.cs @@ -34,10 +34,20 @@ namespace Microsoft.Psi.CognitiveServices.Speech /// Microsoft Cognitive Services Bing Speech API. /// [Obsolete("The Bing Speech service will be retired soon. Please use the AzureSpeechRecognizer instead.", false)] - public sealed class BingSpeechRecognizer : AsyncConsumerProducer, IStreamingSpeechRecognitionResult>, ISourceComponent, IDisposable + public sealed class BingSpeechRecognizer : AsyncConsumerProducer<(AudioBuffer, bool), IStreamingSpeechRecognitionResult>, ISourceComponent, IDisposable { // For canceling any pending recognition tasks before disposal - private readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + private readonly CancellationTokenSource cancellationTokenSource = new (); + + /// + /// The last originating time that was recorded for each output stream. + /// + private readonly Dictionary lastPostedOriginatingTimes; + + /// + /// Queue of current audio buffers for the pending recognition task. + /// + private readonly ConcurrentQueue<(AudioBuffer, bool)> currentQueue = new (); /// /// The client that communicates with the cloud speech recognition service. @@ -67,28 +77,18 @@ public sealed class BingSpeechRecognizer : AsyncConsumerProducer /// The time interval of the last detected speech segment. /// - private TimeInterval lastVADSpeechTimeInterval = new TimeInterval(DateTime.UtcNow, DateTime.UtcNow); + private TimeInterval lastVADSpeechTimeInterval = new (DateTime.UtcNow, DateTime.UtcNow); /// /// The originating time of the most recently received audio packet. /// private DateTime lastAudioOriginatingTime; - /// - /// The last originating time that was recorded for each output stream. - /// - private Dictionary lastPostedOriginatingTimes; - /// /// A flag indicating whether the last audio packet received contained speech. /// private bool lastAudioContainedSpeech = false; - /// - /// Queue of current audio buffers for the pending recognition task. - /// - private ConcurrentQueue> currentQueue = new ConcurrentQueue>(); - /// /// Last contiguous audio buffer collected pending recognition. /// @@ -109,8 +109,9 @@ public sealed class BingSpeechRecognizer : AsyncConsumerProducer /// The pipeline to add the component to. /// The component configuration. - public BingSpeechRecognizer(Pipeline pipeline, BingSpeechRecognizerConfiguration configuration) - : base(pipeline) + /// An optional name for the component. + public BingSpeechRecognizer(Pipeline pipeline, BingSpeechRecognizerConfiguration configuration, string name = nameof(BingSpeechRecognizer)) + : base(pipeline, name) { this.Configuration = configuration ?? new BingSpeechRecognizerConfiguration(); this.PartialRecognitionResults = pipeline.CreateEmitter(this, nameof(this.PartialRecognitionResults)); @@ -203,7 +204,7 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) /// A message containing the combined VAD signal and audio data. /// The message envelope. /// The representing the asynchronous operation. - protected override async Task ReceiveAsync(ValueTuple data, Envelope e) + protected override async Task ReceiveAsync((AudioBuffer, bool) data, Envelope e) { byte[] audioData = data.Item1.Data; bool hasSpeech = data.Item2; @@ -246,20 +247,20 @@ protected override async Task ReceiveAsync(ValueTuple data, E this.lastVADSpeechTimeInterval = new TimeInterval(this.lastVADSpeechStartTime, this.lastVADSpeechEndTime); // Allocate a buffer large enough to hold the buffered audio - BufferWriter bw = new BufferWriter(this.currentQueue.Sum(b => b.Item1.Length)); + var bufferWriter = new BufferWriter(this.currentQueue.Sum(b => b.Item1.Length)); // Get the audio associated with the recognized text from the current queue. - ValueTuple buffer; + (AudioBuffer, bool) buffer; while (this.currentQueue.TryDequeue(out buffer)) { - bw.Write(buffer.Item1.Data); + bufferWriter.Write(buffer.Item1.Data); // We are done with this buffer so enqueue it for recycling this.In.Recycle(buffer); } // Save the buffered audio - this.lastAudioBuffer = bw.Buffer; + this.lastAudioBuffer = bufferWriter.Buffer; // Call EndAudio to signal that this is the last packet await this.speechRecognitionClient.SendEndAudioAsync(this.cancellationTokenSource.Token); diff --git a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/ImageAnalyzer.cs b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/ImageAnalyzer.cs index 255a65a79..5a64436c0 100644 --- a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/ImageAnalyzer.cs +++ b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/ImageAnalyzer.cs @@ -20,6 +20,7 @@ namespace Microsoft.Psi.CognitiveServices.Vision /// Microsoft Cognitive Services Vision API public sealed class ImageAnalyzer : IConsumer>, IProducer { + private readonly string name; private readonly ComputerVisionClient computerVisionClient; private readonly ImageAnalyzerConfiguration configuration; @@ -28,8 +29,10 @@ public sealed class ImageAnalyzer : IConsumer>, IProducer /// The pipeline to add the component to. /// The image analyzer configuration. - public ImageAnalyzer(Pipeline pipeline, ImageAnalyzerConfiguration configuration = null) + /// An optional name for the component. + public ImageAnalyzer(Pipeline pipeline, ImageAnalyzerConfiguration configuration = null, string name = nameof(ImageAnalyzer)) { + this.name = name; this.configuration = configuration ?? new ImageAnalyzerConfiguration(); this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); this.In = pipeline.CreateAsyncReceiver>(this, this.ReceiveAsync, nameof(this.In)); @@ -47,6 +50,9 @@ public ImageAnalyzer(Pipeline pipeline, ImageAnalyzerConfiguration configuration /// public Emitter Out { get; } + /// + public override string ToString() => this.name; + #region Static methods for parsing results to strings /// diff --git a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/Operators.cs b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/Operators.cs index ad6efcf7e..794deeaca 100644 --- a/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/Operators.cs +++ b/Sources/Integrations/CognitiveServices/Microsoft.Psi.CognitiveServices.Vision/Operators.cs @@ -3,8 +3,10 @@ namespace Microsoft.Psi.CognitiveServices.Vision { + using System; using System.Collections.Generic; using Microsoft.Azure.CognitiveServices.Vision.ComputerVision.Models; + using Microsoft.Psi.Imaging; /// /// Stream operators and extension methods for Microsoft.Psi.CognitiveServices.Vision. @@ -12,43 +14,215 @@ namespace Microsoft.Psi.CognitiveServices.Vision public static class Operators { /// - /// Gets the tags associated with an images via Microsoft Cognitive Services Vision API.. + /// Gets the adult info on a stream of images via the Microsoft Cognitive Services Vision API.. /// /// The source stream of images. /// The Azure subscription key to use. /// The region for the Azure subscription. /// An optional delivery policy. - /// A stream of tags for each image. + /// An optional name for the stream operator. + /// A stream of adult info. /// /// A Microsoft Cognitive Services Vision API /// subscription key is required to use this operator. For more information, see the full direct API for. /// Microsoft Cognitive Services Vision API /// - public static IProducer> GetTags(this IProducer> source, string subscriptionKey, string region, DeliveryPolicy> deliveryPolicy = null) - { - var imageAnalyzer = new ImageAnalyzer(source.Out.Pipeline, new ImageAnalyzerConfiguration(subscriptionKey, region, VisualFeatureTypes.Tags)); - source.PipeTo(imageAnalyzer.In, deliveryPolicy); - return imageAnalyzer.Out.Select(ia => ia?.Tags); - } + public static IProducer GetAdult( + this IProducer> source, + string subscriptionKey, + string region, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(GetAdult)) + => source.GetImageAnalyzerInfo(subscriptionKey, region, VisualFeatureTypes.Adult, ia => ia?.Adult, deliveryPolicy, name); + + /// + /// Gets the brands info on a stream of images via the Microsoft Cognitive Services Vision API.. + /// + /// The source stream of images. + /// The Azure subscription key to use. + /// The region for the Azure subscription. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A stream of brands info. + /// + /// A Microsoft Cognitive Services Vision API + /// subscription key is required to use this operator. For more information, see the full direct API for. + /// Microsoft Cognitive Services Vision API + /// + public static IProducer> GetBrands( + this IProducer> source, + string subscriptionKey, + string region, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(GetBrands)) + => source.GetImageAnalyzerInfo(subscriptionKey, region, VisualFeatureTypes.Brands, ia => ia?.Brands, deliveryPolicy, name); + + /// + /// Gets the category info on a stream of images via the Microsoft Cognitive Services Vision API.. + /// + /// The source stream of images. + /// The Azure subscription key to use. + /// The region for the Azure subscription. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A stream of category info. + /// + /// A Microsoft Cognitive Services Vision API + /// subscription key is required to use this operator. For more information, see the full direct API for. + /// Microsoft Cognitive Services Vision API + /// + public static IProducer> GetCategories( + this IProducer> source, + string subscriptionKey, + string region, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(GetCategories)) + => source.GetImageAnalyzerInfo(subscriptionKey, region, VisualFeatureTypes.Categories, ia => ia?.Categories, deliveryPolicy, name); + + /// + /// Gets the color info on a stream of images via the Microsoft Cognitive Services Vision API.. + /// + /// The source stream of images. + /// The Azure subscription key to use. + /// The region for the Azure subscription. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A stream of color info. + /// + /// A Microsoft Cognitive Services Vision API + /// subscription key is required to use this operator. For more information, see the full direct API for. + /// Microsoft Cognitive Services Vision API + /// + public static IProducer GetColor( + this IProducer> source, + string subscriptionKey, + string region, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(GetColor)) + => source.GetImageAnalyzerInfo(subscriptionKey, region, VisualFeatureTypes.Color, ia => ia?.Color, deliveryPolicy, name); + + /// + /// Gets the image description details on a stream of images via the Microsoft Cognitive Services Vision API.. + /// + /// The source stream of images. + /// The Azure subscription key to use. + /// The region for the Azure subscription. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A stream of image description details. + /// + /// A Microsoft Cognitive Services Vision API + /// subscription key is required to use this operator. For more information, see the full direct API for. + /// Microsoft Cognitive Services Vision API + /// + public static IProducer GetDescription( + this IProducer> source, + string subscriptionKey, + string region, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(GetDescription)) + => source.GetImageAnalyzerInfo(subscriptionKey, region, VisualFeatureTypes.Description, ia => ia?.Description, deliveryPolicy, name); + + /// + /// Gets the face description results on a stream of images via the Microsoft Cognitive Services Vision API.. + /// + /// The source stream of images. + /// The Azure subscription key to use. + /// The region for the Azure subscription. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A stream of face descriptions. + /// + /// A Microsoft Cognitive Services Vision API + /// subscription key is required to use this operator. For more information, see the full direct API for. + /// Microsoft Cognitive Services Vision API + /// + public static IProducer> GetFaces( + this IProducer> source, + string subscriptionKey, + string region, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(GetFaces)) + => source.GetImageAnalyzerInfo(subscriptionKey, region, VisualFeatureTypes.Faces, ia => ia?.Faces, deliveryPolicy, name); + + /// + /// Gets the image type info on a stream of images via the Microsoft Cognitive Services Vision API.. + /// + /// The source stream of images. + /// The Azure subscription key to use. + /// The region for the Azure subscription. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A stream of image image type info. + /// + /// A Microsoft Cognitive Services Vision API + /// subscription key is required to use this operator. For more information, see the full direct API for. + /// Microsoft Cognitive Services Vision API + /// + public static IProducer GetImageType( + this IProducer> source, + string subscriptionKey, + string region, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(GetImageType)) + => source.GetImageAnalyzerInfo(subscriptionKey, region, VisualFeatureTypes.ImageType, ia => ia?.ImageType, deliveryPolicy, name); + + /// + /// Gets the object detection results on a stream of images via the Microsoft Cognitive Services Vision API.. + /// + /// The source stream of images. + /// The Azure subscription key to use. + /// The region for the Azure subscription. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A stream of detected objects. + /// + /// A Microsoft Cognitive Services Vision API + /// subscription key is required to use this operator. For more information, see the full direct API for. + /// Microsoft Cognitive Services Vision API + /// + public static IProducer> GetObjects( + this IProducer> source, + string subscriptionKey, + string region, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(GetObjects)) + => source.GetImageAnalyzerInfo(subscriptionKey, region, VisualFeatureTypes.Objects, ia => ia?.Objects, deliveryPolicy, name); /// - /// Performs object detection on a stream of images via the Microsoft Cognitive Services Vision API.. + /// Gets the tags on a stream of images via Microsoft Cognitive Services Vision API.. /// /// The source stream of images. /// The Azure subscription key to use. /// The region for the Azure subscription. /// An optional delivery policy. - /// A stream of detected objects for each image. + /// An optional name for the stream operator. + /// A stream of tags. /// /// A Microsoft Cognitive Services Vision API /// subscription key is required to use this operator. For more information, see the full direct API for. /// Microsoft Cognitive Services Vision API /// - public static IProducer> DetectObjects(this IProducer> source, string subscriptionKey, string region, DeliveryPolicy> deliveryPolicy = null) + public static IProducer> GetTags( + this IProducer> source, + string subscriptionKey, + string region, + DeliveryPolicy> deliveryPolicy = null, + string name = nameof(GetTags)) + => source.GetImageAnalyzerInfo(subscriptionKey, region, VisualFeatureTypes.Tags, ia => ia?.Tags, deliveryPolicy, name); + + private static IProducer GetImageAnalyzerInfo( + this IProducer> source, + string subscriptionKey, + string region, + VisualFeatureTypes visualFeatureType, + Func selector, + DeliveryPolicy> deliveryPolicy = null, + string name = null) { - var imageAnalyzer = new ImageAnalyzer(source.Out.Pipeline, new ImageAnalyzerConfiguration(subscriptionKey, region, VisualFeatureTypes.Objects)); + var imageAnalyzer = new ImageAnalyzer(source.Out.Pipeline, new ImageAnalyzerConfiguration(subscriptionKey, region, visualFeatureType), name); source.PipeTo(imageAnalyzer.In, deliveryPolicy); - return imageAnalyzer.Out.Select(ia => ia?.Objects); + return imageAnalyzer.Out.Select(selector, DeliveryPolicy.SynchronousOrThrottle); } } } \ No newline at end of file diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunner.cs b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunner.cs index c550fc70b..8c9c8b808 100644 --- a/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunner.cs +++ b/Sources/Integrations/Onnx/Common/ModelRunners/ImageNet/ImageNetModelRunner.cs @@ -38,13 +38,14 @@ public class ImageNetModelRunner : ConsumerProducer, List class. /// /// The pipeline to add the component to. - /// The configuration for the compoinent. + /// The configuration for the component. + /// An optional name for the component. /// /// To run on a GPU, use the Microsoft.Psi.Onnx.ModelRunners.Gpu library instead of Microsoft.Psi.Onnx.ModelRunners.Cpu, and set /// the value of the parameter to a valid non-negative integer. Typical device ID values are 0 or 1. /// - public ImageNetModelRunner(Pipeline pipeline, ImageNetModelRunnerConfiguration configuration) - : base(pipeline) + public ImageNetModelRunner(Pipeline pipeline, ImageNetModelRunnerConfiguration configuration, string name = nameof(ImageNetModelRunner)) + : base(pipeline, name) { this.configuration = configuration; diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNDetection.cs b/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNDetection.cs new file mode 100644 index 000000000..b1a8e74ad --- /dev/null +++ b/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNDetection.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Onnx +{ + using System.Drawing; + + /// + /// Represents a masked object detection result from the . + /// + public class MaskRCNNDetection + { + /// + /// Initializes a new instance of the class. + /// + /// Classification label. + /// Confidence level. + /// Bounding box. + /// Mask within bounding box. + public MaskRCNNDetection(string label, float confidence, RectangleF bounds, float[] mask) + { + this.Label = label; + this.Confidence = confidence; + this.Bounds = bounds; + this.Mask = mask; + } + + /// + /// Gets the label. + /// + public string Label { get; } + + /// + /// Gets the confidence level. + /// + public float Confidence { get; } + + /// + /// Gets bounding box. + /// + public RectangleF Bounds { get; } + + /// + /// Gets the mask. + /// + public float[] Mask { get; } + } +} diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNDetectionResults.cs b/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNDetectionResults.cs new file mode 100644 index 000000000..bae698fda --- /dev/null +++ b/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNDetectionResults.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Onnx +{ + using System.Collections.Generic; + using System.Linq; + + /// + /// Represents a set of detection results from the . + /// + public class MaskRCNNDetectionResults + { + /// + /// Initializes a new instance of the class. + /// + /// Set of detections. + /// Original image width. + /// Original image height. + public MaskRCNNDetectionResults(IEnumerable detections, int imageWidth, int imageHeight) + { + this.Detections = detections.ToList(); + this.ImageWidth = imageWidth; + this.ImageHeight = imageHeight; + } + + /// + /// Gets the set of detections. + /// + public List Detections { get; } + + /// + /// Gets the original image width. + /// + public int ImageWidth { get; } + + /// + /// Gets the original image height. + /// + public int ImageHeight { get; } + } +} diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNModelConfiguration.cs b/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNModelConfiguration.cs new file mode 100644 index 000000000..28b02d3b4 --- /dev/null +++ b/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNModelConfiguration.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Onnx +{ + /// + /// Represents the configuration for the class. + /// + public class MaskRCNNModelConfiguration + { + /// + /// Gets or sets the image width. + /// + public int ImageWidth { get; set; } + + /// + /// Gets or sets the image height. + /// + public int ImageHeight { get; set; } + + /// + /// Gets or sets the model file name (see remarks on ). + /// + public string ModelFileName { get; set; } + + /// + /// Gets or sets the classes file name (see remarks on ). + /// + public string ClassesFileName { get; set; } + + /// + /// Gets or sets the GPU device ID to run execution on, or null to run on CPU. + /// + /// + /// To run on a GPU, use the Microsoft.Psi.Onnx.Gpu library instead of Microsoft.Psi.Onnx.Cpu, and set the value of + /// the property to a valid non-negative integer. Typical device ID values are 0 or 1. + /// + public int? GpuDeviceId { get; set; } + } +} diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNModelOutputParser.cs b/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNModelOutputParser.cs new file mode 100644 index 000000000..42dae52f7 --- /dev/null +++ b/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNModelOutputParser.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Onnx +{ + using System; + using System.Collections.Generic; + using System.Drawing; + + /// + /// Internal static class that parses the outputs from the Mask R-CNN model into + /// a set of masked object detection results. + /// + internal static class MaskRCNNModelOutputParser + { + /// + /// Parses the model outputs into a list of object masks and detection results. + /// + /// Confidence level scores. + /// Bounding boxes. + /// Classification labels. + /// Masks within bounding box. + /// Classes corresponding to label indexes. + /// The confidence threshold to use in filtering results. + /// The list of detection results. + internal static IEnumerable Extract(float[] scores, float[] boxes, long[] labels, float[] masks, string[] classes, float confidenceThreshold = .3F) + { + for (var i = 0; i < scores.Length; i++) + { + var confidence = scores[i]; + if (confidence > confidenceThreshold) + { + var label = classes[(int)labels[i]]; + + var box = i * 4; + var x0 = boxes[box]; + var y0 = boxes[box + 1]; + var x1 = boxes[box + 2]; + var y1 = boxes[box + 3]; + var bounds = new RectangleF(x0, y0, x1 - x0, y1 - y0); + + const int MASK_SIZE = 28 * 28; + var mask = new float[MASK_SIZE]; + Array.Copy(masks, i * MASK_SIZE, mask, 0, MASK_SIZE); + + yield return new MaskRCNNDetection(label, confidence, bounds, mask); + } + } + } + } +} diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNModelRunner.cs b/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNModelRunner.cs new file mode 100644 index 000000000..7b2649b79 --- /dev/null +++ b/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNModelRunner.cs @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Onnx +{ + using System.IO; + using Microsoft.Psi; + using Microsoft.Psi.Components; + using Microsoft.Psi.Imaging; + + /// + /// Component that runs the Mask R-CNN object masking and detection model. + /// + /// + /// This class implements a \psi component that runs the Mask R-CNN ONNX model. + /// It uses an input stream of \psi images, which should be of sizes matching + /// the configured width/height. For best performance, images width and height + /// should be between 800 and 1312, inclusive, and divisible by 32. This + /// component parses the outputs into a list of + /// instances, corresponding to each object detection result. The component + /// requires the filename where the Mask R-CNN model can be found. The model + /// is available here in the ONNX Model Zoo at: + /// https://github.com/onnx/models/blob/main/vision/object_detection_segmentation/mask-rcnn/model/MaskRCNN-10.onnx . + /// and the classes file is available here: + /// https://github.com/onnx/models/blob/main/vision/object_detection_segmentation/mask-rcnn/dependencies/coco_classes.txt . + /// + public class MaskRCNNModelRunner : ConsumerProducer, MaskRCNNDetectionResults> + { + private readonly MaskRCNNOnnxModel onnxModel; + private readonly string[] classes; + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// The name of the model file (see remarks on ). + /// The name of the file containing class names (see remarks on ). + /// Image wigth (should be between 800 and 1312, inclusive, and divisible by 32). + /// Image height (should be between 800 and 1312, inclusive, and divisible by 32). + /// The GPU device ID to run execution on, or null to run on CPU. + /// An optional name for the component. + /// + /// To run on a GPU, use the Microsoft.Psi.Onnx.ModelRunners.Gpu library instead of Microsoft.Psi.Onnx.ModelRunners.Cpu, and set + /// the value of the parameter to a valid non-negative integer. Typical device ID values are 0 or 1. + /// + public MaskRCNNModelRunner(Pipeline pipeline, string modelFileName, string classesFileName, int imageWidth, int imageHeight, int? gpuDeviceId = null, string name = nameof(MaskRCNNModelRunner)) + : base(pipeline, name) + { + this.onnxModel = new MaskRCNNOnnxModel(imageWidth, imageHeight, modelFileName, gpuDeviceId); + this.classes = File.ReadAllLines(classesFileName); + } + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// The configuration for the component. + /// An optional name for the component. + public MaskRCNNModelRunner(Pipeline pipeline, MaskRCNNModelConfiguration configuration, string name = nameof(MaskRCNNModelRunner)) + : this(pipeline, configuration.ModelFileName, configuration.ClassesFileName, configuration.ImageWidth, configuration.ImageHeight, configuration.GpuDeviceId) + { + } + + /// + protected override void Receive(Shared data, Envelope envelope) + { + var input = this.ConstructOnnxInput(data); + var output = this.onnxModel.GetPrediction(input); + var detections = MaskRCNNModelOutputParser.Extract( + output.Scores, + output.Boxes, + output.Labels, + output.Masks, + this.classes); + var results = new MaskRCNNDetectionResults(detections, data.Resource.Width, data.Resource.Height); + this.Out.Post(results, envelope.OriginatingTime); + } + + /// + /// Constructs the input vectors for the Mask R-CNN model for a specified image. + /// + /// The shared image to construct the input vector for. + private float[] ConstructOnnxInput(Shared sharedImage) + { + var inputImage = sharedImage.Resource; + var inputWidth = sharedImage.Resource.Width; + var inputHeight = sharedImage.Resource.Height; + var size = 3 * inputWidth * inputHeight; + + byte[] ExtractBytes() + { + if (sharedImage.Resource.PixelFormat != PixelFormat.BGR_24bpp) + { + // convert before extracting bytes + using var reformattedImage = ImagePool.GetOrCreate(inputWidth, inputHeight, PixelFormat.BGR_24bpp); + inputImage.CopyTo(reformattedImage.Resource); + return reformattedImage.Resource.ReadBytes(size); + } + else + { + return inputImage.ReadBytes(size); + } + } + + var bytes = ExtractBytes(); + + var inputVector = new float[size]; + int j = 0; + + void CopyChannel(int offset, float normalization) + { + for (int i = offset; i < bytes.Length; i += 3) + { + inputVector[j++] = bytes[i] - normalization; + } + } + + CopyChannel(2, 102.9801f); // blue + CopyChannel(1, 115.9465f); // green + CopyChannel(0, 122.7717f); // red + + return inputVector; + } + } +} diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNOnnxModel.cs b/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNOnnxModel.cs new file mode 100644 index 000000000..c963fd4ad --- /dev/null +++ b/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/MaskRCNNOnnxModel.cs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Onnx +{ + using System.Collections.Generic; + using System.Linq; + using Microsoft.ML; + using Microsoft.ML.Data; + using Microsoft.ML.Transforms.Onnx; + + /// + /// Implements an ONNX model for Mask R-CNN. + /// + public class MaskRCNNOnnxModel + { + private const string BOXES = "6568"; + private const string LABELS = "6570"; + private const string SCORES = "6572"; + private const string MASKS = "6887"; + + private readonly MLContext context = new MLContext(); + private readonly SchemaDefinition schemaDefinition; + private readonly OnnxTransformer onnxTransformer; + + /// + /// Initializes a new instance of the class. + /// + /// Input image width. + /// Input image height. + /// Model file name. + /// GPU device ID to run execution on, or null to run on CPU. + /// + /// To run on a GPU, use the Microsoft.Psi.Onnx.Gpu library instead of Microsoft.Psi.Onnx.Cpu, and pass + /// a valid non-negative integer. Typical device ID values are 0 or 1. + /// + public MaskRCNNOnnxModel(int imageWidth, int imageHeight, string modelFileName, int? gpuDeviceId = null) + { + //// REVIEW: note the ColumnType below is (float, 3, height, width), while the plain OnnxModel is simply (float, size) + //// the rest is very similar to OnnxModel + + this.schemaDefinition = SchemaDefinition.Create(typeof(OnnxInputVector)); + this.schemaDefinition[nameof(OnnxInputVector.Vector)].ColumnType = new VectorDataViewType(NumberDataViewType.Single, 3, imageHeight, imageWidth); + this.schemaDefinition[nameof(OnnxInputVector.Vector)].ColumnName = "image"; + + var onnxEmptyInputDataView = this.context.Data.LoadFromEnumerable(new List(), this.schemaDefinition); + var shapeDictionary = new Dictionary() { { "image", new int[] { 3, imageHeight, imageWidth } } }; + + var scoringEstimator = + this.context.Transforms.ApplyOnnxModel( + modelFile: modelFileName, + inputColumnNames: new[] { "image" }, + outputColumnNames: new[] { BOXES, LABELS, SCORES, MASKS }, + shapeDictionary: shapeDictionary, + gpuDeviceId: gpuDeviceId, + fallbackToCpu: false); + this.onnxTransformer = scoringEstimator.Fit(onnxEmptyInputDataView); + } + + /// + /// Runs the ONNX model on an input vector. + /// + /// The input vector. + /// The set of output vectors produced by the ONNX model. + public (float[] Scores, float[] Boxes, long[] Labels, float[] Masks) GetPrediction(float[] input) + { + //// REVIEW: note that OnnxModel is merely float[] -> float[] + + // construct a data view over the input + var onnxInput = new List { new OnnxInputVector { Vector = input } }; + var onnxInputDataView = this.context.Data.LoadFromEnumerable(onnxInput, this.schemaDefinition); + + // apply the onnxTransformer and extract the results + var prediction = this.onnxTransformer.Transform(onnxInputDataView); + var scores = prediction.GetColumn(SCORES).ToArray()[0]; + var boxes = prediction.GetColumn(BOXES).ToArray()[0]; + var labels = prediction.GetColumn(LABELS).ToArray()[0]; + var masks = prediction.GetColumn(MASKS).ToArray()[0]; + return (scores, boxes, labels, masks); + } + + /// + /// Represents a vector input for an ONNX model. + /// + internal class OnnxInputVector + { + /// + /// Gets or sets the vector data. + /// + public float[] Vector { get; set; } + } + } +} diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/Operators.cs b/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/Operators.cs new file mode 100644 index 000000000..385ffebb2 --- /dev/null +++ b/Sources/Integrations/Onnx/Common/ModelRunners/MaskRCNN/Operators.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Onnx +{ + using System; + using System.Drawing; + using System.Linq; + using Microsoft.Psi.Imaging; + using Image = Microsoft.Psi.Imaging.Image; + + /// + /// Implements operators for processing Mask R-CNN types. + /// + public static partial class Operators + { + private const float FONTSIZE = 9f; + private static Font font = new Font(FontFamily.GenericSansSerif, FONTSIZE, FontStyle.Regular); + private static Bitmap mask = new Bitmap(28, 28); + private static Brush labelBrush = Brushes.White; + private static (Pen, Color)[] rectangleAndMaskColors = new[] + { + (Pens.Red, Color.Red), + (Pens.Blue, Color.Blue), + (Pens.Green, Color.Green), + (Pens.Orange, Color.Orange), + (Pens.Purple, Color.Purple), + (Pens.Cyan, Color.Cyan), + (Pens.Magenta, Color.Magenta), + (Pens.Yellow, Color.Yellow), + (Pens.White, Color.White), + }; + + /// + /// Render to a transparent image with object + /// bounding boxes, labels and masks. + /// + /// to render. + /// Optional image allocator for creating new shared image. + /// Rendered results. + public static Shared Render( + MaskRCNNDetectionResults results, + Func> sharedImageAllocator = null) + { + // create image and preprare to draw on it + var sharedImage = (sharedImageAllocator ?? ImagePool.GetOrCreate)(results.ImageWidth, results.ImageHeight, PixelFormat.BGRA_32bpp); + sharedImage.Resource.Clear(Color.Transparent); + using var bmp = sharedImage.Resource.ToBitmap(false); + using var graphics = Graphics.FromImage(bmp); + graphics.SmoothingMode = System.Drawing.Drawing2D.SmoothingMode.None; // no antialiasing + graphics.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.NearestNeighbor; + + var colorIndex = -1; + foreach (var detection in results.Detections.OrderBy(d => d.Confidence)) + { + (var pen, var color) = rectangleAndMaskColors[++colorIndex % rectangleAndMaskColors.Length]; + var rect = new Rectangle( + (int)(detection.Bounds.X + 0.5f), + (int)(detection.Bounds.Y + 0.5f), + (int)(detection.Bounds.Width + 0.5f), + (int)(detection.Bounds.Height + 0.5f)); + + for (var y = 0; y < 28; y++) + { + for (var x = 0; x < 28; x++) + { + var m = (int)(128 * detection.Mask[x + y * 28]); + mask.SetPixel(x, y, Color.FromArgb(m, color)); + } + } + + graphics.DrawImage(mask, rect); + graphics.DrawRectangle(pen, rect); + graphics.DrawString($"{detection.Label} {(int)(detection.Confidence * 100)}", font, labelBrush, rect.X, rect.Y - 1.8f * FONTSIZE); + + sharedImage.Resource.CopyFrom(bmp); + } + + return sharedImage; + } + + /// + /// Render to a stream of transparent images + /// with object bounding boxes, labels and masks. + /// + /// Stream of . + /// An optional delivery policy. + /// Optional image allocator for creating new shared image. + /// An optional name for this stream operator. + /// Stream of rendered results. + public static IProducer> Render( + this IProducer source, + DeliveryPolicy deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(Render)) + { + return source.Select(r => Render(r, sharedImageAllocator), deliveryPolicy, name); + } + } +} diff --git a/Sources/Integrations/Onnx/Common/ModelRunners/TinyYoloV2/TinyYoloV2OnnxModelRunner.cs b/Sources/Integrations/Onnx/Common/ModelRunners/TinyYoloV2/TinyYoloV2OnnxModelRunner.cs index 1e6a50799..a32d03f40 100644 --- a/Sources/Integrations/Onnx/Common/ModelRunners/TinyYoloV2/TinyYoloV2OnnxModelRunner.cs +++ b/Sources/Integrations/Onnx/Common/ModelRunners/TinyYoloV2/TinyYoloV2OnnxModelRunner.cs @@ -35,12 +35,13 @@ public class TinyYoloV2OnnxModelRunner : ConsumerProducer, ListThe pipeline to add the component to. /// The name of the model. /// The GPU device ID to run execution on, or null to run on CPU. + /// An optional name for the component. /// /// To run on a GPU, use the Microsoft.Psi.Onnx.ModelRunners.Gpu library instead of Microsoft.Psi.Onnx.ModelRunners.Cpu, and set /// the value of the parameter to a valid non-negative integer. Typical device ID values are 0 or 1. /// - public TinyYoloV2OnnxModelRunner(Pipeline pipeline, string modelFileName, int? gpuDeviceId = null) - : base(pipeline) + public TinyYoloV2OnnxModelRunner(Pipeline pipeline, string modelFileName, int? gpuDeviceId = null, string name = nameof(TinyYoloV2OnnxModelRunner)) + : base(pipeline, name) { // create an ONNX model, with a configuration that matches the structure // of the Tiny Yolo V2 model diff --git a/Sources/Integrations/Onnx/Common/OnnxModel.cs b/Sources/Integrations/Onnx/Common/OnnxModel.cs index 073b1f809..2484f8703 100644 --- a/Sources/Integrations/Onnx/Common/OnnxModel.cs +++ b/Sources/Integrations/Onnx/Common/OnnxModel.cs @@ -33,7 +33,7 @@ public OnnxModel(OnnxModelConfiguration configuration) // The schemaDefinition is a ML.NET construct that allows us to specify the form // of the inputs. In this case we construct a schema definition programmatically - // to reflect that the input is a vector of floats, of the size specified in the + // to reflect that the input is a vector of floats, of the sizes specified in the // configuration this.schemaDefinition = SchemaDefinition.Create(typeof(OnnxInputVector)); this.schemaDefinition[nameof(OnnxInputVector.Vector)].ColumnType = new VectorDataViewType(NumberDataViewType.Single, this.configuration.InputVectorSize); @@ -46,7 +46,9 @@ public OnnxModel(OnnxModelConfiguration configuration) modelFile: configuration.ModelFileName, outputColumnNames: new[] { configuration.OutputVectorName }, inputColumnNames: new[] { configuration.InputVectorName }, - gpuDeviceId: configuration.GpuDeviceId); + shapeDictionary: configuration.ShapeDictionary, + gpuDeviceId: configuration.GpuDeviceId, + fallbackToCpu: false); this.onnxTransformer = scoringEstimator.Fit(onnxEmptyInputDataView); } diff --git a/Sources/Integrations/Onnx/Common/OnnxModelConfiguration.cs b/Sources/Integrations/Onnx/Common/OnnxModelConfiguration.cs index 47a220d1f..7a776256d 100644 --- a/Sources/Integrations/Onnx/Common/OnnxModelConfiguration.cs +++ b/Sources/Integrations/Onnx/Common/OnnxModelConfiguration.cs @@ -3,6 +3,8 @@ namespace Microsoft.Psi.Onnx { + using System.Collections.Generic; + /// /// Represents the configuration for the class. /// @@ -38,6 +40,11 @@ public OnnxModelConfiguration() /// public string OutputVectorName { get; set; } + /// + /// Gets or sets an optional shape dictionary describing the shape of input vector. + /// + public IDictionary ShapeDictionary { get; set; } + /// /// Gets or sets the GPU device ID to run execution on, or null to run on CPU. /// diff --git a/Sources/Integrations/Onnx/Common/OnnxModelRunner.cs b/Sources/Integrations/Onnx/Common/OnnxModelRunner.cs index e66a508e2..bcbfe16e7 100644 --- a/Sources/Integrations/Onnx/Common/OnnxModelRunner.cs +++ b/Sources/Integrations/Onnx/Common/OnnxModelRunner.cs @@ -34,11 +34,12 @@ public class OnnxModelRunner : ConsumerProducer /// /// The pipeline to add the component to. /// The component configuration. + /// An optional name for the component. /// The configuration parameter specifies the model filename, the /// name of the input and output vectors in that ONNX model, as well as /// the input vector size. - public OnnxModelRunner(Pipeline pipeline, OnnxModelConfiguration configuration) - : base(pipeline) + public OnnxModelRunner(Pipeline pipeline, OnnxModelConfiguration configuration, string name = nameof(OnnxModelRunner)) + : base(pipeline, name) { this.inputVectorSize = configuration.InputVectorSize; this.onnxModel = new OnnxModel(configuration); diff --git a/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Cpu/Microsoft.Psi.Onnx.Cpu.csproj b/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Cpu/Microsoft.Psi.Onnx.Cpu.csproj index 1c955c94e..96f6f1442 100644 --- a/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Cpu/Microsoft.Psi.Onnx.Cpu.csproj +++ b/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Cpu/Microsoft.Psi.Onnx.Cpu.csproj @@ -32,8 +32,8 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + diff --git a/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Gpu/Microsoft.Psi.Onnx.Gpu.csproj b/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Gpu/Microsoft.Psi.Onnx.Gpu.csproj index 8be2df523..f2da54e3e 100644 --- a/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Gpu/Microsoft.Psi.Onnx.Gpu.csproj +++ b/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Gpu/Microsoft.Psi.Onnx.Gpu.csproj @@ -32,8 +32,8 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - - + + diff --git a/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Visualization.Windows/MaskRCNNDetectionAdapter.cs b/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Visualization.Windows/MaskRCNNDetectionAdapter.cs new file mode 100644 index 000000000..93fb40905 --- /dev/null +++ b/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Visualization.Windows/MaskRCNNDetectionAdapter.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Onnx.Visualization +{ + using System.Collections.Generic; + using Microsoft.Psi.Visualization.Adapters; + using Microsoft.Psi.Visualization.Data; + using Image = Microsoft.Psi.Imaging.Image; + + /// + /// Implements a stream adapter from to with containing masks, bounding boxes and labels. + /// + [StreamAdapter] + public class MaskRCNNDetectionAdapter : StreamAdapter> + { + /// + public override Shared GetAdaptedValue(MaskRCNNDetectionResults source, Envelope envelope) + { + if (source == null) + { + return null; + } + + return Onnx.Operators.Render(source); + } + + /// + public override void Dispose(Shared destination) => destination?.Dispose(); + } +} diff --git a/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Visualization.Windows/Microsoft.Psi.Onnx.Visualization.Windows.csproj b/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Visualization.Windows/Microsoft.Psi.Onnx.Visualization.Windows.csproj new file mode 100644 index 000000000..f1296fa4d --- /dev/null +++ b/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Visualization.Windows/Microsoft.Psi.Onnx.Visualization.Windows.csproj @@ -0,0 +1,45 @@ + + + + net472 + Microsoft.Psi.Onnx.Visualization + Provides visualizers for ONNX model runner output types defined in Microsoft.Psi.Onnx.ModelRunners. + + + + DEBUG;TRACE + bin\Debug\net472\Microsoft.Psi.Onnx.Visualization.Windows.xml + true + ..\..\..\..\Build\Microsoft.Psi.ruleset + true + + + + + bin\Release\net472\Microsoft.Psi.Onnx.Visualization.Windows.xml + true + ..\..\..\..\Build\Microsoft.Psi.ruleset + true + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Visualization.Windows/stylecop.json b/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Visualization.Windows/stylecop.json new file mode 100644 index 000000000..6f09427eb --- /dev/null +++ b/Sources/Integrations/Onnx/Microsoft.Psi.Onnx.Visualization.Windows/stylecop.json @@ -0,0 +1,16 @@ +{ + // ACTION REQUIRED: This file was automatically added to your project, but it + // will not take effect until additional steps are taken to enable it. See the + // following page for additional information: + // + // https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/EnableConfiguration.md + + "$schema": "https://raw.githubusercontent.com/DotNetAnalyzers/StyleCopAnalyzers/master/StyleCop.Analyzers/StyleCop.Analyzers/Settings/stylecop.schema.json", + "settings": { + "documentationRules": { + "companyName": "Microsoft Corporation", + "copyrightText": "Copyright (c) Microsoft Corporation. All rights reserved.\nLicensed under the MIT license.", + "xmlHeader": false + } + } +} \ No newline at end of file diff --git a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectBodyTracker.cs b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectBodyTracker.cs index 5da2b380c..b99795967 100644 --- a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectBodyTracker.cs +++ b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectBodyTracker.cs @@ -34,8 +34,9 @@ public sealed class AzureKinectBodyTracker : ConsumerProducer<(Shared /// The pipeline to add the component to. /// An optional configuration to use for the body tracker. - public AzureKinectBodyTracker(Pipeline pipeline, AzureKinectBodyTrackerConfiguration configuration = null) - : base(pipeline) + /// An optional name for the component. + public AzureKinectBodyTracker(Pipeline pipeline, AzureKinectBodyTrackerConfiguration configuration = null, string name = nameof(AzureKinectBodyTracker)) + : base(pipeline, name) { this.configuration = configuration ?? new AzureKinectBodyTrackerConfiguration(); this.AzureKinectSensorCalibration = pipeline.CreateReceiver(this, this.ReceiveCalibration, nameof(this.AzureKinectSensorCalibration)); @@ -122,6 +123,7 @@ private void InitializeTracker(Calibration calibration) { SensorOrientation = this.configuration.SensorOrientation, ProcessingMode = this.configuration.CpuOnlyMode ? TrackerProcessingMode.Cpu : TrackerProcessingMode.Gpu, + ModelPath = this.configuration.UseLiteModel ? "dnn_model_2_0_lite_op11.onnx" : "dnn_model_2_0_op11.onnx", }); } diff --git a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectBodyTrackerConfiguration.cs b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectBodyTrackerConfiguration.cs index cf85b40dd..90cdd1ce2 100644 --- a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectBodyTrackerConfiguration.cs +++ b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectBodyTrackerConfiguration.cs @@ -21,15 +21,26 @@ public class AzureKinectBodyTrackerConfiguration public float TemporalSmoothing { get; set; } = 0f; /// - /// Gets or sets a value indicating whether to perform body tracking computation only - /// on the CPU. + /// Gets or sets a value indicating whether to use CPU only mode to run the tracker. + /// Defaults to false (GPU mode). /// - /// If false, the tracker requires CUDA hardware and drivers. + /// + /// The CPU only mode doesn't require the machine to have a GPU to run the tracker, + /// but it will be much slower than the GPU mode. public bool CpuOnlyMode { get; set; } = false; /// /// Gets or sets the sensor orientation used by body tracking. /// public SensorOrientation SensorOrientation { get; set; } = SensorOrientation.Default; + + /// + /// Gets or sets a value indicating whether to use the "lite" model for pose estimation. + /// Defaults to false (standard model). + /// + /// + /// The lite model trades ~2x performance increase for ~5% accuracy decrease compared to the standard model. + /// + public bool UseLiteModel { get; set; } = false; } } diff --git a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectCore.cs b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectCore.cs index 574ee6985..c4ab8d1f9 100644 --- a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectCore.cs +++ b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectCore.cs @@ -23,6 +23,7 @@ internal sealed class AzureKinectCore : ISourceComponent, IDisposable { private static readonly object CameraOpenLock = new object(); private readonly Pipeline pipeline; + private readonly string name; private readonly AzureKinectSensorConfiguration configuration; /// @@ -43,9 +44,11 @@ internal sealed class AzureKinectCore : ISourceComponent, IDisposable /// /// The pipeline to add the component to. /// Configuration to use for the device. - public AzureKinectCore(Pipeline pipeline, AzureKinectSensorConfiguration config = null) + /// An optional name for the component. + public AzureKinectCore(Pipeline pipeline, AzureKinectSensorConfiguration config = null, string name = nameof(AzureKinectCore)) { this.pipeline = pipeline; + this.name = name; this.configuration = config ?? new AzureKinectSensorConfiguration(); if (this.configuration.OutputColor) @@ -241,6 +244,9 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) notifyCompleted(); } + /// + public override string ToString() => this.name; + private void CaptureThreadProc() { if (this.configuration.ColorResolution == ColorResolution.Off && @@ -393,7 +399,11 @@ private void CaptureThreadProc() if (this.configuration.OutputDepth && capture.Depth != null) { - sharedDepthImage = DepthImagePool.GetOrCreate(this.depthImageWidth, this.depthImageHeight); + sharedDepthImage = DepthImagePool.GetOrCreate( + this.depthImageWidth, + this.depthImageHeight, + DepthValueSemantics.DistanceToPlane, + 0.001); sharedDepthImage.Resource.CopyFrom(capture.Depth.Memory.ToArray()); this.DepthImage.Post(sharedDepthImage, currentTime); diff --git a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectSensor.cs b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectSensor.cs index 35e8b1418..a5b197c75 100644 --- a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectSensor.cs +++ b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/AzureKinectSensor.cs @@ -26,12 +26,14 @@ public class AzureKinectSensor : Subpipeline /// Configuration to use for the sensor. /// An optional default delivery policy for the subpipeline (defaults is LatestMessage). /// An optional delivery policy for sending the depth-and-IR images stream to the body tracker (default is LatestMessage). + /// An optional name for the component. public AzureKinectSensor( Pipeline pipeline, AzureKinectSensorConfiguration configuration = null, DeliveryPolicy defaultDeliveryPolicy = null, - DeliveryPolicy bodyTrackerDeliveryPolicy = null) - : base(pipeline, nameof(AzureKinectSensor), defaultDeliveryPolicy ?? DeliveryPolicy.LatestMessage) + DeliveryPolicy bodyTrackerDeliveryPolicy = null, + string name = nameof(AzureKinectSensor)) + : base(pipeline, name, defaultDeliveryPolicy ?? DeliveryPolicy.LatestMessage) { this.Configuration = configuration ?? new AzureKinectSensorConfiguration(); @@ -88,7 +90,7 @@ public static IEnumerable AllDevices int numDevices = Device.GetInstalledCount(); for (int i = 0; i < numDevices; i++) { - CameraDeviceInfo di = new CameraDeviceInfo + var di = new CameraDeviceInfo { FriendlyName = $"AzureKinect-{i}", DeviceName = $"AzureKinect-{i}", @@ -110,7 +112,7 @@ public static IEnumerable AllDevices di.Sensors = new List(); for (int k = 0; k < 3; k++) { - CameraDeviceInfo.Sensor sensor = new CameraDeviceInfo.Sensor(); + var sensor = new CameraDeviceInfo.Sensor(); uint[,] resolutions = null; switch (k) { @@ -156,7 +158,7 @@ public static IEnumerable AllDevices continue; // Mode doesn't support 30fps } - CameraDeviceInfo.Sensor.ModeInfo mi = new CameraDeviceInfo.Sensor.ModeInfo + var mi = new CameraDeviceInfo.Sensor.ModeInfo { Format = Imaging.PixelFormat.BGRA_32bpp, FrameRateNumerator = fr, diff --git a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/Microsoft.Psi.AzureKinect.x64.csproj b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/Microsoft.Psi.AzureKinect.x64.csproj index fb688c4dd..43c68265c 100644 --- a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/Microsoft.Psi.AzureKinect.x64.csproj +++ b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/Microsoft.Psi.AzureKinect.x64.csproj @@ -29,8 +29,8 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/Microsoft.Psi.AzureKinect.x64.nuspec b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/Microsoft.Psi.AzureKinect.x64.nuspec index 372b59b4d..4ef31f855 100644 --- a/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/Microsoft.Psi.AzureKinect.x64.nuspec +++ b/Sources/Kinect/Microsoft.Psi.AzureKinect.x64/Microsoft.Psi.AzureKinect.x64.nuspec @@ -20,8 +20,8 @@ - - + + diff --git a/Sources/Kinect/Microsoft.Psi.Kinect.Windows/KinectSensor.cs b/Sources/Kinect/Microsoft.Psi.Kinect.Windows/KinectSensor.cs index 3466342bf..63d22b1e0 100644 --- a/Sources/Kinect/Microsoft.Psi.Kinect.Windows/KinectSensor.cs +++ b/Sources/Kinect/Microsoft.Psi.Kinect.Windows/KinectSensor.cs @@ -407,7 +407,11 @@ private void DepthFrameReader_FrameArrived(DepthFrame depthFrame) FrameDescription depthFrameDescription = depthFrame.FrameDescription; using (KinectBuffer depthBuffer = depthFrame.LockImageBuffer()) { - using (var dest = DepthImagePool.GetOrCreate(depthFrameDescription.Width, depthFrameDescription.Height)) + using (var dest = DepthImagePool.GetOrCreate( + depthFrameDescription.Width, + depthFrameDescription.Height, + DepthValueSemantics.DistanceToPlane, + 0.001)) { depthFrame.CopyFrameDataToIntPtr(dest.Resource.ImageData, (uint)(depthFrameDescription.Width * depthFrameDescription.Height * 2)); var time = this.pipeline.GetCurrentTimeFromElapsedTicks(depthFrame.RelativeTime.Ticks); diff --git a/Sources/Kinect/Test.Psi.Kinect.Windows.x64/Mesh.cs b/Sources/Kinect/Test.Psi.Kinect.Windows.x64/Mesh.cs index e8632956b..f8f10799a 100644 --- a/Sources/Kinect/Test.Psi.Kinect.Windows.x64/Mesh.cs +++ b/Sources/Kinect/Test.Psi.Kinect.Windows.x64/Mesh.cs @@ -78,7 +78,7 @@ public static Mesh MeshFromDepthMap(Shared depthImage, Shared { // Determine vertex position+color via new calibration Point2D newpt = new Point2D(pt.X, calib.DepthIntrinsics.ImageHeight - pt.Y); - Point3D p = calib.DepthIntrinsics.GetCameraSpacePosition(newpt, depth, true); + Point3D p = calib.DepthIntrinsics.GetCameraSpacePosition(newpt, depth, depthImage.Resource.DepthValueSemantics, true); mesh.Vertices[count].Pos = new Point3D(p.X / 1000.0, p.Y / 1000.0, p.Z / 1000.0); Vector pos = Vector.Build.Dense(4); diff --git a/Sources/Media/Microsoft.Psi.Media.Linux/MediaCapture.cs b/Sources/Media/Microsoft.Psi.Media.Linux/MediaCapture.cs index 1531735b1..2cedaada9 100644 --- a/Sources/Media/Microsoft.Psi.Media.Linux/MediaCapture.cs +++ b/Sources/Media/Microsoft.Psi.Media.Linux/MediaCapture.cs @@ -19,6 +19,7 @@ namespace Microsoft.Psi.Media public class MediaCapture : IProducer>, ISourceComponent, IDisposable { private readonly Pipeline pipeline; + private readonly string name; private readonly MediaCaptureConfiguration configuration; private MediaCaptureInternal camera; @@ -28,8 +29,9 @@ public class MediaCapture : IProducer>, ISourceComponent, IDisposa /// /// The pipeline to add the component to. /// Name of file containing media capture device configuration. - public MediaCapture(Pipeline pipeline, string configurationFilename) - : this(pipeline) + /// An optional name for the component. + public MediaCapture(Pipeline pipeline, string configurationFilename, string name = nameof(MediaCapture)) + : this(pipeline, name) { var configurationHelper = new ConfigurationHelper(configurationFilename); this.configuration = configurationHelper.Configuration; @@ -40,8 +42,9 @@ public MediaCapture(Pipeline pipeline, string configurationFilename) /// /// The pipeline to add the component to. /// Describes how to configure the media capture device. - public MediaCapture(Pipeline pipeline, MediaCaptureConfiguration configuration) - : this(pipeline) + /// An optional name for the component. + public MediaCapture(Pipeline pipeline, MediaCaptureConfiguration configuration, string name = nameof(MediaCapture)) + : this(pipeline, name) { this.configuration = configuration; } @@ -54,8 +57,9 @@ public MediaCapture(Pipeline pipeline, MediaCaptureConfiguration configuration) /// Height of output image in pixels. /// Device ID. /// Pixel format. - public MediaCapture(Pipeline pipeline, int width, int height, string deviceId = "/dev/video0", PixelFormatId pixelFormat = PixelFormatId.BGR24) - : this(pipeline) + /// An optional name for the component. + public MediaCapture(Pipeline pipeline, int width, int height, string deviceId = "/dev/video0", PixelFormatId pixelFormat = PixelFormatId.BGR24, string name = nameof(MediaCapture)) + : this(pipeline, name) { if (pixelFormat != PixelFormatId.BGR24 && pixelFormat != PixelFormatId.YUYV && pixelFormat != PixelFormatId.MJPEG) { @@ -71,9 +75,10 @@ public MediaCapture(Pipeline pipeline, int width, int height, string deviceId = }; } - private MediaCapture(Pipeline pipeline) + private MediaCapture(Pipeline pipeline, string name) { this.pipeline = pipeline; + this.name = name; this.Out = pipeline.CreateEmitter>(this, nameof(this.Out)); this.Raw = pipeline.CreateEmitter>(this, nameof(this.Raw)); } @@ -141,78 +146,68 @@ public unsafe void Start(Action notifyCompletionTime) if (this.Raw.HasSubscribers) { var len = frame.Length; - using (Shared shared = SharedArrayPool.GetOrCreate(len)) - { - Marshal.Copy(frame.Start, shared.Resource, 0, len); - this.Raw.Post(shared, originatingTime); - } + using Shared shared = SharedArrayPool.GetOrCreate(len); + Marshal.Copy(frame.Start, shared.Resource, 0, len); + this.Raw.Post(shared, originatingTime); } if (this.Out.HasSubscribers) { if (this.configuration.PixelFormat == PixelFormatId.BGR24) { - using (var sharedImage = ImagePool.GetOrCreate(this.configuration.Width, this.configuration.Height, PixelFormat.BGR_24bpp)) - { - sharedImage.Resource.CopyFrom((IntPtr)frame.Start); - this.Out.Post(sharedImage, this.pipeline.GetCurrentTime()); - } + using var sharedImage = ImagePool.GetOrCreate(this.configuration.Width, this.configuration.Height, PixelFormat.BGR_24bpp); + sharedImage.Resource.CopyFrom((IntPtr)frame.Start); + this.Out.Post(sharedImage, this.pipeline.GetCurrentTime()); } else if (this.configuration.PixelFormat == PixelFormatId.YUYV) { // convert YUYV -> BGR24 (see https://msdn.microsoft.com/en-us/library/ms893078.aspx) - using (var sharedImage = ImagePool.GetOrCreate(this.configuration.Width, this.configuration.Height, PixelFormat.BGR_24bpp)) + using var sharedImage = ImagePool.GetOrCreate(this.configuration.Width, this.configuration.Height, PixelFormat.BGR_24bpp); + var len = (int)(frame.Length * 1.5); + using Shared shared = SharedArrayPool.GetOrCreate(len); + var bytes = shared.Resource; + var pY = (byte*)frame.Start.ToPointer(); + var pU = pY + 1; + var pV = pY + 3; + for (var i = 0; i < len;) { - var len = (int)(frame.Length * 1.5); - using (Shared shared = SharedArrayPool.GetOrCreate(len)) - { - var bytes = shared.Resource; - var pY = (byte*)frame.Start.ToPointer(); - var pU = pY + 1; - var pV = pY + 3; - for (var i = 0; i < len;) - { - int y = (*pY - 16) * 298; - int u = *pU - 128; - int v = *pV - 128; - int r = (y + (409 * v) + 128) >> 8; - int g = (y - (100 * u) - (208 * v) + 128) >> 8; - int b = (y + (516 * u) + 128) >> 8; - - bytes[i++] = (byte)((r < 0) ? 0 : ((r > 255) ? 255 : r)); - bytes[i++] = (byte)((g < 0) ? 0 : ((g > 255) ? 255 : g)); - bytes[i++] = (byte)((b < 0) ? 0 : ((b > 255) ? 255 : b)); - - pY += 2; - - y = (*pY - 16) * 298; - r = (y + (409 * v) + 128) >> 8; - g = (y - (100 * u) - (208 * v) + 128) >> 8; - b = (y + (516 * u) + 128) >> 8; - bytes[i++] = (byte)((r < 0) ? 0 : ((r > 255) ? 255 : r)); - bytes[i++] = (byte)((g < 0) ? 0 : ((g > 255) ? 255 : g)); - bytes[i++] = (byte)((b < 0) ? 0 : ((b > 255) ? 255 : b)); - - pY += 2; - pU += 4; - pV += 4; - } - - sharedImage.Resource.CopyFrom(bytes); - this.Out.Post(sharedImage, originatingTime); - } + int y = (*pY - 16) * 298; + int u = *pU - 128; + int v = *pV - 128; + int r = (y + (409 * v) + 128) >> 8; + int g = (y - (100 * u) - (208 * v) + 128) >> 8; + int b = (y + (516 * u) + 128) >> 8; + + bytes[i++] = (byte)((r < 0) ? 0 : ((r > 255) ? 255 : r)); + bytes[i++] = (byte)((g < 0) ? 0 : ((g > 255) ? 255 : g)); + bytes[i++] = (byte)((b < 0) ? 0 : ((b > 255) ? 255 : b)); + + pY += 2; + + y = (*pY - 16) * 298; + r = (y + (409 * v) + 128) >> 8; + g = (y - (100 * u) - (208 * v) + 128) >> 8; + b = (y + (516 * u) + 128) >> 8; + bytes[i++] = (byte)((r < 0) ? 0 : ((r > 255) ? 255 : r)); + bytes[i++] = (byte)((g < 0) ? 0 : ((g > 255) ? 255 : g)); + bytes[i++] = (byte)((b < 0) ? 0 : ((b > 255) ? 255 : b)); + + pY += 2; + pU += 4; + pV += 4; } + + sharedImage.Resource.CopyFrom(bytes); + this.Out.Post(sharedImage, originatingTime); } else if (this.configuration.PixelFormat == PixelFormatId.MJPEG) { var decoded = SKBitmap.Decode(new UnmanagedMemoryStream((byte*)frame.Start, frame.Length)); if (decoded != null) { - using (var sharedImage = ImagePool.GetOrCreate(this.configuration.Width, this.configuration.Height, PixelFormat.BGRA_32bpp)) - { - sharedImage.Resource.CopyFrom(decoded.Bytes); - this.Out.Post(sharedImage, originatingTime); - } + using var sharedImage = ImagePool.GetOrCreate(this.configuration.Width, this.configuration.Height, PixelFormat.BGRA_32bpp); + sharedImage.Resource.CopyFrom(decoded.Bytes); + this.Out.Post(sharedImage, originatingTime); } } } @@ -239,6 +234,9 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) notifyCompleted(); } + + /// + public override string ToString() => this.name; } } diff --git a/Sources/Media/Microsoft.Psi.Media.Native.x64/Microsoft.Psi.Media.Native.x64.vcxproj b/Sources/Media/Microsoft.Psi.Media.Native.x64/Microsoft.Psi.Media.Native.x64.vcxproj index bc61d71f3..8d8ce72d4 100644 --- a/Sources/Media/Microsoft.Psi.Media.Native.x64/Microsoft.Psi.Media.Native.x64.vcxproj +++ b/Sources/Media/Microsoft.Psi.Media.Native.x64/Microsoft.Psi.Media.Native.x64.vcxproj @@ -15,7 +15,7 @@ {C50F7F21-BEB0-4366-B73F-859EEBC3ED42} Win32Proj MicrosoftPsiMediaNativex64 - 10.0.18362.0 + 10.0.19041.0 4.7.2 diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaCapture.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaCapture.cs index 461bcdfd7..bfd45c6d5 100644 --- a/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaCapture.cs +++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaCapture.cs @@ -17,6 +17,7 @@ namespace Microsoft.Psi.Media public class MediaCapture : IProducer>, ISourceComponent, IDisposable, IMediaCapture { private readonly Pipeline pipeline; + private readonly string name; /// /// The video camera configuration. @@ -40,9 +41,11 @@ public class MediaCapture : IProducer>, ISourceComponent, IDisposa /// /// The pipeline to add the component to. /// Name of file containing media capture device configuration. - public MediaCapture(Pipeline pipeline, string configurationFilename) + /// An optional name for the component. + public MediaCapture(Pipeline pipeline, string configurationFilename, string name = nameof(MediaCapture)) : this(pipeline) { + this.name = name; var configurationHelper = new ConfigurationHelper(configurationFilename); this.configuration = configurationHelper.Configuration; if (this.configuration.CaptureAudio) @@ -56,8 +59,9 @@ public MediaCapture(Pipeline pipeline, string configurationFilename) /// /// The pipeline to add the component to. /// Describes how to configure the media capture device. - public MediaCapture(Pipeline pipeline, MediaCaptureConfiguration configuration = null) - : this(pipeline) + /// An optional name for the component. + public MediaCapture(Pipeline pipeline, MediaCaptureConfiguration configuration = null, string name = nameof(MediaCapture)) + : this(pipeline, name) { this.configuration = configuration ?? new MediaCaptureConfiguration(); if (this.configuration.CaptureAudio) @@ -76,8 +80,17 @@ public MediaCapture(Pipeline pipeline, MediaCaptureConfiguration configuration = /// Should we create an audio capture device. /// Device ID. /// Indicates whether camera is shared amongst multiple applications. - public MediaCapture(Pipeline pipeline, int width, int height, double framerate = 30, bool captureAudio = false, string deviceId = null, bool useInSharedMode = false) - : this(pipeline) + /// An optional name for the component. + public MediaCapture( + Pipeline pipeline, + int width, + int height, + double framerate = 30, + bool captureAudio = false, + string deviceId = null, + bool useInSharedMode = false, + string name = nameof(MediaCapture)) + : this(pipeline, name) { this.configuration = new MediaCaptureConfiguration() { @@ -94,8 +107,9 @@ public MediaCapture(Pipeline pipeline, int width, int height, double framerate = } } - private MediaCapture(Pipeline pipeline) + private MediaCapture(Pipeline pipeline, string name) { + this.name = name; this.pipeline = pipeline; this.Out = pipeline.CreateEmitter>(this, nameof(this.Out)); } @@ -299,6 +313,9 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) notifyCompleted(); } + /// + public override string ToString() => this.name; + private void SetDeviceProperty(VideoProperty prop, MediaCaptureInfo.PropertyInfo propInfo, MediaCaptureConfiguration.PropertyValue value) { if (propInfo.Supported) diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaSource.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaSource.cs index ba9ce61a4..0396e9981 100644 --- a/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaSource.cs +++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/MediaSource.cs @@ -17,12 +17,14 @@ namespace Microsoft.Psi.Media /// public class MediaSource : Generator, IDisposable { + private readonly Stream input; + private readonly DateTime start; + private readonly bool dropOutOfOrderPackets = false; + private bool disposed = false; - private Stream input; private short videoWidth; private short videoHeight; private SourceReader sourceReader; - private DateTime start; private int imageStreamIndex = -1; private int audioStreamIndex = -1; private WaveFormat waveFormat; @@ -37,19 +39,15 @@ public class MediaSource : Generator, IDisposable /// private DateTime lastPostedAudioTime = DateTime.MinValue; - /// - /// Whether to drop out of order packets from media foundation. - /// - private bool dropOutOfOrderPackets = false; - /// /// Initializes a new instance of the class. /// /// The pipeline to add the component to. /// Name of media file to play. /// Optional flag specifying whether to drop out of order packets (defaults to false). - public MediaSource(Pipeline pipeline, string filename, bool dropOutOfOrderPackets = false) - : this(pipeline, File.OpenRead(filename), new FileInfo(filename).CreationTime, dropOutOfOrderPackets) + /// An optional name for the stream operator. + public MediaSource(Pipeline pipeline, string filename, bool dropOutOfOrderPackets = false, string name = nameof(MediaSource)) + : this(pipeline, File.OpenRead(filename), new FileInfo(filename).CreationTime, dropOutOfOrderPackets, name) { } @@ -60,8 +58,9 @@ public MediaSource(Pipeline pipeline, string filename, bool dropOutOfOrderPacket /// Source stream of the media to consume. /// Optional date/time that the media started. /// Optional flag specifying whether to drop out of order packets (defaults to false). - public MediaSource(Pipeline pipeline, Stream input, DateTime? startTime = null, bool dropOutOfOrderPackets = false) - : base(pipeline) + /// An optional name for the component. + public MediaSource(Pipeline pipeline, Stream input, DateTime? startTime = null, bool dropOutOfOrderPackets = false, string name = nameof(MediaSource)) + : base(pipeline, name: name) { var proposedReplayTime = startTime ?? DateTime.UtcNow; pipeline.ProposeReplayTime(new TimeInterval(proposedReplayTime, DateTime.MaxValue)); @@ -104,30 +103,23 @@ public void Dispose() /// The originating time at which to capture the next sample. protected override DateTime GenerateNext(DateTime currentTime) { - DateTime originatingTime = default(DateTime); - int streamIndex = 0; - SourceReaderFlags flags = SourceReaderFlags.None; - long timestamp = 0; - Sample sample = this.sourceReader.ReadSample(SourceReaderIndex.AnyStream, 0, out streamIndex, out flags, out timestamp); + DateTime originatingTime = default; + var sample = this.sourceReader.ReadSample(SourceReaderIndex.AnyStream, 0, out int streamIndex, out SourceReaderFlags flags, out long timestamp); if (sample != null) { originatingTime = this.start + TimeSpan.FromTicks(timestamp); - MediaBuffer buffer = sample.ConvertToContiguousBuffer(); - int currentByteCount = 0; - int maxByteCount = 0; - IntPtr data = buffer.Lock(out maxByteCount, out currentByteCount); + var buffer = sample.ConvertToContiguousBuffer(); + var data = buffer.Lock(out _, out int currentByteCount); if (streamIndex == this.imageStreamIndex) { // Detect out of order originating times if (originatingTime > this.lastPostedImageTime) { - using (var sharedImage = ImagePool.GetOrCreate(this.videoWidth, this.videoHeight, Imaging.PixelFormat.BGR_24bpp)) - { - sharedImage.Resource.CopyFrom(data); - this.Image.Post(sharedImage, originatingTime); - this.lastPostedImageTime = originatingTime; - } + using var sharedImage = ImagePool.GetOrCreate(this.videoWidth, this.videoHeight, Imaging.PixelFormat.BGR_24bpp); + sharedImage.Resource.CopyFrom(data); + this.Image.Post(sharedImage, originatingTime); + this.lastPostedImageTime = originatingTime; } else if (!this.dropOutOfOrderPackets) { @@ -144,7 +136,7 @@ protected override DateTime GenerateNext(DateTime currentTime) // Detect out of order originating times if (originatingTime > this.lastPostedAudioTime) { - AudioBuffer audioBuffer = new AudioBuffer(currentByteCount, this.waveFormat); + var audioBuffer = new AudioBuffer(currentByteCount, this.waveFormat); Marshal.Copy(data, audioBuffer.Data, 0, currentByteCount); this.Audio.Post(audioBuffer, originatingTime); this.lastPostedAudioTime = originatingTime; @@ -185,7 +177,7 @@ private static void DumpMediaType(MediaType mediaType) private void InitializeMediaPipeline() { MediaManager.Startup(false); - MediaAttributes sourceReaderAttributes = new MediaAttributes(); + var sourceReaderAttributes = new MediaAttributes(); sourceReaderAttributes.Set(SourceReaderAttributeKeys.EnableAdvancedVideoProcessing, true); this.sourceReader = new SourceReader(this.input, sourceReaderAttributes); this.sourceReader.SetStreamSelection(SourceReaderIndex.AllStreams, false); diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/Mpeg4Writer.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/Mpeg4Writer.cs index 30c37e642..c798f4da9 100644 --- a/Sources/Media/Microsoft.Psi.Media.Windows.x64/Mpeg4Writer.cs +++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/Mpeg4Writer.cs @@ -15,8 +15,9 @@ namespace Microsoft.Psi.Media public class Mpeg4Writer : IConsumer>, IDisposable { private readonly Pipeline pipeline; + private readonly string name; + private readonly string filename; private readonly Mpeg4WriterConfiguration configuration; - private string filename; private MP4Writer writer; /// @@ -25,8 +26,9 @@ public class Mpeg4Writer : IConsumer>, IDisposable /// The pipeline to add the component to. /// Name of output file to write to. /// Name of file containing media capture device configuration. - public Mpeg4Writer(Pipeline pipeline, string filename, string configurationFilename) - : this(pipeline, filename) + /// An optional name for the component. + public Mpeg4Writer(Pipeline pipeline, string filename, string configurationFilename, string name = nameof(Mpeg4Writer)) + : this(pipeline, filename, name) { var configurationHelper = new ConfigurationHelper(configurationFilename); this.configuration = configurationHelper.Configuration; @@ -38,8 +40,9 @@ public Mpeg4Writer(Pipeline pipeline, string filename, string configurationFilen /// The pipeline to add the component to. /// Name of output file to write to. /// Describes how to configure the media capture device. - public Mpeg4Writer(Pipeline pipeline, string filename, Mpeg4WriterConfiguration configuration) - : this(pipeline, filename) + /// An optional name for the component. + public Mpeg4Writer(Pipeline pipeline, string filename, Mpeg4WriterConfiguration configuration, string name = nameof(Mpeg4Writer)) + : this(pipeline, filename, name) { this.configuration = configuration; } @@ -52,8 +55,9 @@ public Mpeg4Writer(Pipeline pipeline, string filename, Mpeg4WriterConfiguration /// Width of output image in pixels. /// Height of output image in pixels. /// Format of input images. - public Mpeg4Writer(Pipeline pipeline, string filename, uint width, uint height, Imaging.PixelFormat pixelFormat) - : this(pipeline, filename) + /// An optional name for the component. + public Mpeg4Writer(Pipeline pipeline, string filename, uint width, uint height, PixelFormat pixelFormat, string name = nameof(Mpeg4Writer)) + : this(pipeline, filename, name) { this.configuration = Mpeg4WriterConfiguration.Default; this.configuration.ImageWidth = width; @@ -61,10 +65,11 @@ public Mpeg4Writer(Pipeline pipeline, string filename, uint width, uint height, this.configuration.PixelFormat = pixelFormat; } - private Mpeg4Writer(Pipeline pipeline, string filename) + private Mpeg4Writer(Pipeline pipeline, string filename, string name) { pipeline.PipelineRun += (s, e) => this.OnPipelineRun(); this.pipeline = pipeline; + this.name = name; this.ImageIn = pipeline.CreateReceiver>(this, this.ReceiveImage, nameof(this.ImageIn)); this.AudioIn = pipeline.CreateReceiver(this, this.ReceiveAudio, nameof(this.AudioIn)); this.filename = filename; diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCapture.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCapture.cs index 7bdabcd9e..15c95d19d 100644 --- a/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCapture.cs +++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/VisualCapture.cs @@ -29,8 +29,9 @@ public class VisualCapture : Generator, IProducer> /// Windows Media Visual to stream. /// Pixel width at which to render the visual. /// Pixel height at which to render the visual. - public VisualCapture(Pipeline pipeline, TimeSpan interval, Visual visual, int pixelWidth, int pixelHeight) - : base(pipeline, true) + /// An optional name for the component. + public VisualCapture(Pipeline pipeline, TimeSpan interval, Visual visual, int pixelWidth, int pixelHeight, string name = nameof(VisualCapture)) + : base(pipeline, true, name) { this.interval = interval; this.visual = visual; @@ -45,8 +46,9 @@ public VisualCapture(Pipeline pipeline, TimeSpan interval, Visual visual, int pi /// /// The pipeline to add the component to. /// Configuration values for the visual capture component. - public VisualCapture(Pipeline pipeline, VisualCaptureConfiguration configuration) - : this(pipeline, configuration.Interval, configuration.Visual, configuration.PixelWidth, configuration.PixelHeight) + /// An optional name for the component. + public VisualCapture(Pipeline pipeline, VisualCaptureConfiguration configuration, string name = nameof(VisualCapture)) + : this(pipeline, configuration.Interval, configuration.Visual, configuration.PixelWidth, configuration.PixelHeight, name) { } diff --git a/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCapture.cs b/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCapture.cs index 9ad099bd5..86b4481b4 100644 --- a/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCapture.cs +++ b/Sources/Media/Microsoft.Psi.Media.Windows.x64/WindowCapture.cs @@ -31,8 +31,9 @@ public class WindowCapture : Generator, IProducer> /// The pipeline to add the component to. /// Interval at which to render and emit frames of the window. /// Window handle to capture (default=desktop window/primary screen). - public WindowCapture(Pipeline pipeline, TimeSpan interval, IntPtr hWnd) - : base(pipeline, true) + /// An optional name for the component. + public WindowCapture(Pipeline pipeline, TimeSpan interval, IntPtr hWnd, string name = nameof(WindowCapture)) + : base(pipeline, true, name) { this.interval = interval; this.hWnd = hWnd == IntPtr.Zero ? GetDesktopWindow() : hWnd; @@ -44,8 +45,9 @@ public WindowCapture(Pipeline pipeline, TimeSpan interval, IntPtr hWnd) /// /// The pipeline to add the component to. /// Interval at which to render and emit frames of the window. - public WindowCapture(Pipeline pipeline, TimeSpan interval) - : this(pipeline, interval, IntPtr.Zero) + /// An optional name for the component. + public WindowCapture(Pipeline pipeline, TimeSpan interval, string name = nameof(WindowCapture)) + : this(pipeline, interval, IntPtr.Zero, name) { } @@ -54,8 +56,9 @@ public WindowCapture(Pipeline pipeline, TimeSpan interval) /// /// The pipeline to add the component to. /// Configuration values for the window capture component. - public WindowCapture(Pipeline pipeline, WindowCaptureConfiguration configuration) - : this(pipeline, configuration.Interval, configuration.WindowHandle) + /// An optional name for the component. + public WindowCapture(Pipeline pipeline, WindowCaptureConfiguration configuration, string name = nameof(WindowCapture)) + : this(pipeline, configuration.Interval, configuration.WindowHandle, name) { } @@ -63,8 +66,9 @@ public WindowCapture(Pipeline pipeline, WindowCaptureConfiguration configuration /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - public WindowCapture(Pipeline pipeline) - : this(pipeline, WindowCaptureConfiguration.Default) + /// An optional name for the component. + public WindowCapture(Pipeline pipeline, string name = nameof(WindowCapture)) + : this(pipeline, WindowCaptureConfiguration.Default, name) { } diff --git a/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.cpp b/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.cpp index 94bcd67ec..a8edf8299 100644 --- a/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.cpp +++ b/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.cpp @@ -15,6 +15,6 @@ using namespace System::Security::Permissions; [assembly:AssemblyCopyrightAttribute("Copyright (c) Microsoft Corporation. All rights reserved.")]; [assembly:ComVisible(false)]; [assembly:CLSCompliantAttribute(true)]; -[assembly:AssemblyVersionAttribute("0.16.92.1")]; -[assembly:AssemblyFileVersionAttribute("0.16.92.1")]; -[assembly:AssemblyInformationalVersionAttribute("0.16.92.1-beta")]; +[assembly:AssemblyVersionAttribute("0.17.52.1")]; +[assembly:AssemblyFileVersionAttribute("0.17.52.1")]; +[assembly:AssemblyInformationalVersionAttribute("0.17.52.1-beta")]; diff --git a/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.rc b/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/AssemblyInfo.rc index cf19cf33437f0e00f787b43b1a3bf6ae52d6be45..c90806db6ef5222e0062fc5abb5f73f97d813b31 100644 GIT binary patch delta 38 scmeyM{y}|19tXEMgARi!gAtH4oNUNpJb4X=5UV+Z9)s!T!yLNY0LHutGynhq delta 38 scmeyM{y}|19tXD>gARiwgAtH4oNUNpJb4X=5UUx39)soP!yLNY0LK&wIsgCw diff --git a/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/Microsoft.Psi.Media_Interop.Windows.x64.vcxproj b/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/Microsoft.Psi.Media_Interop.Windows.x64.vcxproj index 56139a740..6d917eb5f 100644 --- a/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/Microsoft.Psi.Media_Interop.Windows.x64.vcxproj +++ b/Sources/Media/Microsoft.Psi.Media_Interop.Windows.x64/Microsoft.Psi.Media_Interop.Windows.x64.vcxproj @@ -16,7 +16,7 @@ ManagedCProj VideoCapture Microsoft.Psi.Media_Interop.Windows.x64 - 10.0.18362.0 + 10.0.19041.0 $(FFMPEGDir)\lib\avdevice.lib;$(FFMPEGDir)\lib\avfilter.lib;$(FFMPEGDir)\lib\postproc.lib;$(FFMPEGDir)\lib\swresample.lib;$(FFMPEGDir)\lib\avcodec.lib;$(FFMPEGDir)\lib\avformat.lib;$(FFMPEGDir)\lib\avutil.lib;$(FFMPEGDir)\lib\swscale.lib diff --git a/Sources/Media/Shared/FFMPEGMediaSource.cs b/Sources/Media/Shared/FFMPEGMediaSource.cs index 764d35ff7..d50520847 100644 --- a/Sources/Media/Shared/FFMPEGMediaSource.cs +++ b/Sources/Media/Shared/FFMPEGMediaSource.cs @@ -40,8 +40,9 @@ public class FFMPEGMediaSource : Generator, IDisposable /// The pipeline to add the component to. /// Name of media file to play /// Output format for images - public FFMPEGMediaSource(Pipeline pipeline, string filename, PixelFormat format = PixelFormat.BGRX_32bpp) - : base(pipeline) + /// An optional name for the component. + public FFMPEGMediaSource(Pipeline pipeline, string filename, PixelFormat format = PixelFormat.BGRX_32bpp, string name = nameof(FFMPEGMediaSource)) + : base(pipeline, name: name) { FileInfo info = new FileInfo(filename); pipeline.ProposeReplayTime(new TimeInterval(info.CreationTime, DateTime.MaxValue), new TimeInterval(info.CreationTime, DateTime.MaxValue)); diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCapture.sln b/Sources/MixedReality/HoloLensCapture/HoloLensCapture.sln new file mode 100644 index 000000000..a6cae1378 --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCapture.sln @@ -0,0 +1,141 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.2.32210.308 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HoloLensCapture", "HoloLensCapture", "{AB09B2F1-126E-47D2-A2CE-C895942544B2}" + ProjectSection(SolutionItems) = preProject + Readme.md = Readme.md + EndProjectSection +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HoloLensCaptureApp", "HoloLensCaptureApp\HoloLensCaptureApp.csproj", "{D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HoloLensCaptureExporter", "HoloLensCaptureExporter\HoloLensCaptureExporter.csproj", "{BC00A00F-C929-436E-8F82-1CF696ABFA32}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HoloLensCaptureInterop", "HoloLensCaptureInterop\HoloLensCaptureInterop.csproj", "{62D294B0-B662-4A11-96B1-486B69D72868}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "HoloLensCaptureServer", "HoloLensCaptureServer\HoloLensCaptureServer.csproj", "{2D049963-1089-475B-B74E-31321F8A17CE}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Dependencies", "Dependencies", "{18171A33-C859-41E1-9FB4-0E314C2B16E3}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "HoloLens2ResearchMode", "..\HoloLens2ResearchMode\HoloLens2ResearchMode.vcxproj", "{F50194C0-9561-40C7-B9CB-B977E3B3D76D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi", "..\..\Runtime\Microsoft.Psi\Microsoft.Psi.csproj", "{66B8EEA6-321E-40BF-8027-5BCE9B093DE3}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi.Audio", "..\..\Audio\Microsoft.Psi.Audio\Microsoft.Psi.Audio.csproj", "{42838889-787E-4DC5-9682-8FB9D923B104}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi.Calibration", "..\..\Calibration\Microsoft.Psi.Calibration\Microsoft.Psi.Calibration.csproj", "{050BBC1A-29B7-4D59-B756-038B589713CE}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi.Imaging", "..\..\Imaging\Microsoft.Psi.Imaging\Microsoft.Psi.Imaging.csproj", "{EDE9283C-ACFF-444E-8AB7-AC4D0EE3CDB1}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi.Interop", "..\..\Runtime\Microsoft.Psi.Interop\Microsoft.Psi.Interop.csproj", "{755F21A4-CB2B-46F3-85A1-86F558737AA9}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi.MixedReality", "..\Microsoft.Psi.MixedReality\Microsoft.Psi.MixedReality.csproj", "{11C17353-4485-4BF4-9AB7-37D9EBF4EE45}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Psi.MixedReality.UniversalWindows", "..\Microsoft.Psi.MixedReality.UniversalWindows\Microsoft.Psi.MixedReality.UniversalWindows.csproj", "{ECD9E150-8104-4DA3-B807-A6A4392A67C6}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi.Spatial.Euclidean", "..\..\Spatial\Microsoft.Psi.Spatial.Euclidean\Microsoft.Psi.Spatial.Euclidean.csproj", "{BE854962-52F4-4F1F-919B-1345DC97536C}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi.Imaging.Windows", "..\..\Imaging\Microsoft.Psi.Imaging.Windows\Microsoft.Psi.Imaging.Windows.csproj", "{2279401B-9B6C-4F4E-B69F-21DFF582E4C9}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Psi.Data", "..\..\Data\Microsoft.Psi.Data\Microsoft.Psi.Data.csproj", "{5318B3E0-2FEA-4DED-B041-6F7CC1E15890}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D}.Debug|Any CPU.ActiveCfg = Debug|ARM + {D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D}.Debug|Any CPU.Build.0 = Debug|ARM + {D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D}.Debug|Any CPU.Deploy.0 = Debug|ARM + {D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D}.Release|Any CPU.ActiveCfg = Release|ARM + {D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D}.Release|Any CPU.Build.0 = Release|ARM + {D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D}.Release|Any CPU.Deploy.0 = Release|ARM + {BC00A00F-C929-436E-8F82-1CF696ABFA32}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BC00A00F-C929-436E-8F82-1CF696ABFA32}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BC00A00F-C929-436E-8F82-1CF696ABFA32}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BC00A00F-C929-436E-8F82-1CF696ABFA32}.Release|Any CPU.Build.0 = Release|Any CPU + {62D294B0-B662-4A11-96B1-486B69D72868}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {62D294B0-B662-4A11-96B1-486B69D72868}.Debug|Any CPU.Build.0 = Debug|Any CPU + {62D294B0-B662-4A11-96B1-486B69D72868}.Release|Any CPU.ActiveCfg = Release|Any CPU + {62D294B0-B662-4A11-96B1-486B69D72868}.Release|Any CPU.Build.0 = Release|Any CPU + {2D049963-1089-475B-B74E-31321F8A17CE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2D049963-1089-475B-B74E-31321F8A17CE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2D049963-1089-475B-B74E-31321F8A17CE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2D049963-1089-475B-B74E-31321F8A17CE}.Release|Any CPU.Build.0 = Release|Any CPU + {F50194C0-9561-40C7-B9CB-B977E3B3D76D}.Debug|Any CPU.ActiveCfg = Debug|ARM + {F50194C0-9561-40C7-B9CB-B977E3B3D76D}.Debug|Any CPU.Build.0 = Debug|ARM + {F50194C0-9561-40C7-B9CB-B977E3B3D76D}.Release|Any CPU.ActiveCfg = Release|ARM + {F50194C0-9561-40C7-B9CB-B977E3B3D76D}.Release|Any CPU.Build.0 = Release|ARM + {66B8EEA6-321E-40BF-8027-5BCE9B093DE3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {66B8EEA6-321E-40BF-8027-5BCE9B093DE3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {66B8EEA6-321E-40BF-8027-5BCE9B093DE3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {66B8EEA6-321E-40BF-8027-5BCE9B093DE3}.Release|Any CPU.Build.0 = Release|Any CPU + {42838889-787E-4DC5-9682-8FB9D923B104}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {42838889-787E-4DC5-9682-8FB9D923B104}.Debug|Any CPU.Build.0 = Debug|Any CPU + {42838889-787E-4DC5-9682-8FB9D923B104}.Release|Any CPU.ActiveCfg = Release|Any CPU + {42838889-787E-4DC5-9682-8FB9D923B104}.Release|Any CPU.Build.0 = Release|Any CPU + {050BBC1A-29B7-4D59-B756-038B589713CE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {050BBC1A-29B7-4D59-B756-038B589713CE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {050BBC1A-29B7-4D59-B756-038B589713CE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {050BBC1A-29B7-4D59-B756-038B589713CE}.Release|Any CPU.Build.0 = Release|Any CPU + {EDE9283C-ACFF-444E-8AB7-AC4D0EE3CDB1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EDE9283C-ACFF-444E-8AB7-AC4D0EE3CDB1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EDE9283C-ACFF-444E-8AB7-AC4D0EE3CDB1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EDE9283C-ACFF-444E-8AB7-AC4D0EE3CDB1}.Release|Any CPU.Build.0 = Release|Any CPU + {755F21A4-CB2B-46F3-85A1-86F558737AA9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {755F21A4-CB2B-46F3-85A1-86F558737AA9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {755F21A4-CB2B-46F3-85A1-86F558737AA9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {755F21A4-CB2B-46F3-85A1-86F558737AA9}.Release|Any CPU.Build.0 = Release|Any CPU + {11C17353-4485-4BF4-9AB7-37D9EBF4EE45}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {11C17353-4485-4BF4-9AB7-37D9EBF4EE45}.Debug|Any CPU.Build.0 = Debug|Any CPU + {11C17353-4485-4BF4-9AB7-37D9EBF4EE45}.Release|Any CPU.ActiveCfg = Release|Any CPU + {11C17353-4485-4BF4-9AB7-37D9EBF4EE45}.Release|Any CPU.Build.0 = Release|Any CPU + {ECD9E150-8104-4DA3-B807-A6A4392A67C6}.Debug|Any CPU.ActiveCfg = Debug|ARM + {ECD9E150-8104-4DA3-B807-A6A4392A67C6}.Debug|Any CPU.Build.0 = Debug|ARM + {ECD9E150-8104-4DA3-B807-A6A4392A67C6}.Release|Any CPU.ActiveCfg = Release|ARM + {ECD9E150-8104-4DA3-B807-A6A4392A67C6}.Release|Any CPU.Build.0 = Release|ARM + {BE854962-52F4-4F1F-919B-1345DC97536C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BE854962-52F4-4F1F-919B-1345DC97536C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BE854962-52F4-4F1F-919B-1345DC97536C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BE854962-52F4-4F1F-919B-1345DC97536C}.Release|Any CPU.Build.0 = Release|Any CPU + {2279401B-9B6C-4F4E-B69F-21DFF582E4C9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2279401B-9B6C-4F4E-B69F-21DFF582E4C9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2279401B-9B6C-4F4E-B69F-21DFF582E4C9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2279401B-9B6C-4F4E-B69F-21DFF582E4C9}.Release|Any CPU.Build.0 = Release|Any CPU + {5318B3E0-2FEA-4DED-B041-6F7CC1E15890}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5318B3E0-2FEA-4DED-B041-6F7CC1E15890}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5318B3E0-2FEA-4DED-B041-6F7CC1E15890}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5318B3E0-2FEA-4DED-B041-6F7CC1E15890}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D} = {AB09B2F1-126E-47D2-A2CE-C895942544B2} + {BC00A00F-C929-436E-8F82-1CF696ABFA32} = {AB09B2F1-126E-47D2-A2CE-C895942544B2} + {62D294B0-B662-4A11-96B1-486B69D72868} = {18171A33-C859-41E1-9FB4-0E314C2B16E3} + {2D049963-1089-475B-B74E-31321F8A17CE} = {AB09B2F1-126E-47D2-A2CE-C895942544B2} + {F50194C0-9561-40C7-B9CB-B977E3B3D76D} = {18171A33-C859-41E1-9FB4-0E314C2B16E3} + {66B8EEA6-321E-40BF-8027-5BCE9B093DE3} = {18171A33-C859-41E1-9FB4-0E314C2B16E3} + {42838889-787E-4DC5-9682-8FB9D923B104} = {18171A33-C859-41E1-9FB4-0E314C2B16E3} + {050BBC1A-29B7-4D59-B756-038B589713CE} = {18171A33-C859-41E1-9FB4-0E314C2B16E3} + {EDE9283C-ACFF-444E-8AB7-AC4D0EE3CDB1} = {18171A33-C859-41E1-9FB4-0E314C2B16E3} + {755F21A4-CB2B-46F3-85A1-86F558737AA9} = {18171A33-C859-41E1-9FB4-0E314C2B16E3} + {11C17353-4485-4BF4-9AB7-37D9EBF4EE45} = {18171A33-C859-41E1-9FB4-0E314C2B16E3} + {ECD9E150-8104-4DA3-B807-A6A4392A67C6} = {18171A33-C859-41E1-9FB4-0E314C2B16E3} + {BE854962-52F4-4F1F-919B-1345DC97536C} = {18171A33-C859-41E1-9FB4-0E314C2B16E3} + {2279401B-9B6C-4F4E-B69F-21DFF582E4C9} = {18171A33-C859-41E1-9FB4-0E314C2B16E3} + {5318B3E0-2FEA-4DED-B041-6F7CC1E15890} = {18171A33-C859-41E1-9FB4-0E314C2B16E3} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {EAF15EE9-DCC5-411B-A9E5-7C2F3D132331} + EndGlobalSection + GlobalSection(Performance) = preSolution + HasPerformanceSessions = true + EndGlobalSection + GlobalSection(Performance) = preSolution + HasPerformanceSessions = true + EndGlobalSection +EndGlobal diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/LockScreenLogo.scale-200.png b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/LockScreenLogo.scale-200.png new file mode 100644 index 0000000000000000000000000000000000000000..735f57adb5dfc01886d137b4e493d7e97cf13af3 GIT binary patch literal 1430 zcmaJ>TTC2P7~aKltDttVHYH6u8Io4i*}3fO&d$gd*bA_<3j~&e7%8(eXJLfhS!M@! zKrliY>>6yT4+Kr95$!DoD(Qn-5TP|{V_KS`k~E6(LGS@#`v$hQo&^^BKsw3HIsZBT z_y6C2n`lK@apunKojRQ^(_P}Mgewt$(^BBKCTZ;*xa?J3wQ7~@S0lUvbcLeq1Bg4o zH-bvQi|wt~L7q$~a-gDFP!{&TQfc3fX*6=uHv* zT&1&U(-)L%Xp^djI2?~eBF2cxC@YOP$+9d?P&h?lPy-9M2UT9fg5jKm1t$m#iWE{M zIf%q9@;fyT?0UP>tcw-bLkz;s2LlKl2qeP0w zECS7Ate+Awk|KQ+DOk;fl}Xsy4o^CY=pwq%QAAKKl628_yNPsK>?A>%D8fQG6IgdJ ztnxttBz#NI_a@fk7SU`WtrpsfZsNs9^0(2a z@C3#YO3>k~w7?2hipBf{#b6`}Xw1hlG$yi?;1dDs7k~xDAw@jiI*+tc;t2Lflg&bM)0!Y;0_@=w%`LW^8DsYpS#-bLOklX9r?Ei}TScw|4DbpW%+7 zFgAI)f51s}{y-eWb|vrU-Ya!GuYKP)J7z#*V_k^Xo>4!1Yqj*m)x&0L^tg3GJbVAJ zJ-Pl$R=NAabouV=^z_t;^K*0AvFs!vYU>_<|I^#c?>>CR<(T?=%{;U=aI*SbZADLH z&(f2wz_Y0??Tf|g;?|1Znw6}6U43Q#qNRwv1vp9uFn1)V#*4p&%$mP9x&15^OaBiDS(XppT|z^>;B{PLVEbS3IFYV yGvCsSX*m literal 0 HcmV?d00001 diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/SplashScreen.scale-200.png b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/SplashScreen.scale-200.png new file mode 100644 index 0000000000000000000000000000000000000000..023e7f1feda78d5100569825acedfd213a0d84e9 GIT binary patch literal 7700 zcmeHLYj~4Yw%(;oxoEH#Kxq-eR|+VkP17b#Vk;?4QwkI+A{L04G+#<<(x#Un1#+h5>eArRq zTw$)ZvTWW_Y?bDho0nPVTh08+s`sp!j74rJTTtXIDww0SILedFv?sZ?yb@@}GN;#8 znk_b~Q(A0YR#uV4ef!osoV1M3;vQ8N$O|fStfgf$S5;ddUNv`tWtGjM;koG#N;7M< zP*84lnx(bn_KF&9Z5Ai$)#Cs3a|$OFw>WKCT$of*L7_CqQEinflT|W{JT+aKp-E0v zsxmYg)1(T>DROm+LN1eQw8}KCTp=C!$H7`PU!t9_Hw@TsTI2`udRZv*!a5`#A9hK6Y95L(CDUX&_@QxKV z_feX{UhA#ZWlvgpL$#w^D#lq`_A4AzDqd|Zv6y9PX&DNcN|l}_D^{q@GG&H^Pg583 z8FI6N8^H7b5WjGp;urW)d7F+_lcp%KsLX0viCmE(OHH+=%ZfD_=`voUuoUxFO^L;- z;!;2{g-YiiO6m4bs89OuF9!p{FGtH-f%8<2gY!h9s)4ciN%{Kh1+`}{^}M~+TDH9N z^Z5PlgVXMC&2&k*Hw^Lb9gny#ro$MOIxIt{+r)EA10$VR3 zanN8D{TUkl+v0CQ_>ZoHP<M-x#8@8ZiT#$Kh`(uRaX1g$Bg|qy$<#7 zSSAi{Nb8Y=lvNVeio+UGLCAtoLBfL`iOv`)yoJMDJBN>4IH@(l7YRF;61@>qq1iM9 zr@b#OC~SAxSle?5Pp8Z78{VO0YFr1x7kZU64Z23eLf2T2#6J_t;-E}DkB?NufZ0Ug zi?J&byXeaB-uTNVhuiM!UVQw}bZrJ3GtAETYp->!{q#zfN7D3AS9@Q7*V^85jGx#R z(QxYV(wW#F0XF9^^s>>H8pPlVJ>)3Oz z&_X8Sf@~?cH_O*cgi$U#`v`RRfv#y3m(ZpKk^5uLup+lVs$~}FZU$r_+}#hl%?g5m z-u-}-666ssp-xWQak~>PPy$mRc|~?pVSs1_@mBEXpPVfLF6(Ktf1S* zPPh@QZ=tFMs?LM2(5P3L2;l_6XX6s&cYsP1ip#eg0`ZEP0HGYh{UmS@o`MihLLvkU zgyAG0G`b1|qjxxh1(ODKFE%AP}Dq=3vK$P7TXP4GrM1kQ72!GUVMDl`rDC&2;TA}*nF z8$nQD&6ys_nc1*E7$*1S@R8$ymy(sQV}imGSedB@{!QR5P&N_H=-^o!?LsWs+2|mH z-e=)T^SvI)=_JIm7}j4;@*Z17=(#}m=~YF~z~CLI+vdAGlJDcdF$TM?CVI1%LhUrN zaa6DJ=Yh$)$k&Oz{-~8yw^GM^8prYxSxo zvI4k#ibryMa%%*8oI-5m61Koa_A_xg=(fwp0aBX{;X4Q;NXUhtaoJDo1>TqhWtn=_ zd5~chq#&6~c%8JZK#t_&J(9EVUU&upYeIovLt1>vaHe}UUq>#RGQj!EN#5+0@T`(@ z^g~>*c`VGRiSt;!$_4+0hk^I!@O3``5=sZ8IwlxWW7km1B&_t&E*u0_9UBa#VqwY* zz>nxv?FAsVnRaD(Bui=6i==BFUw0k4n$>`umU`F2l?7CYTD^)c2X+d9X&ddS9|gj? zM?knGkGCX&W8offw8aLC2$D{PjC3nVZwd4k?eZH8*mZ)U@3Qk8RDFOz_#WUA#vnzy zyP>KrCfKwSXea7}jgJjBc}PGY+4#6%lbZyjhy`5sZd_Vy6Wz;ixa?czkN}J9It1K6 zY!eu>|AwF^fwZlLAYyQI*lM@^>O>Iu6Vf6i>Q$?v!SeUS<{>UYMwz$*%Aq?w^`j{h z!$GZbhu=^D{&ET8;))LL%ZBDZkQqRd2;u~!d9bHGmLRhLDctNgYyjsuvoSZ#iVdoB z2!f--UUA#U;<{je#?cYt^{PIyKa%hW>}uepWMyAI{{Zo7?2>?$c9;whJae%oN|I-kpTQSx_C$Z&;f zi2i)qmEn=y4U0uvk)$m;zKfjPK@oc?I`}1Jzl$Q~aoKBd3kt7L#7gyt|A_qgz6ai< z=X%D1i!d2h?rHR^R8SUj&G||dkC?DT>{o#Yau<@uqVT{Xef&XG}5*E4aPk{}~ zplx&XhaV)&1EfI3Em;Bw#O5SV^c;{twb-1Rw)+=0!e_BLbd7tYmXCH0wrlOSS+~`7He8Iqx0{CN+DVit9;*6L~JAN zD&cyT)2?h}xnYmL?^)<7YyzZ3$FHU^Eg;DLqAV{#wv#Wj7S`Jdl1pX&{3(uZ?!uh} zDc$ZTNV*7le_W6}Hju~GMTxZQ1aWCeUc%!jv3MHAzt>Y-nQK%zfT*3ebDQA5b?iGn; zBjv3B+GhLTexd_(CzZDP4|#n5^~scvB6#Pk%Ho!kQ>yYw((Dv{6=$g3jT1!u6gORW zx5#`7Wy-ZHRa~IxGHdrp(bm%lf>2%J660nj$fCqN(epv@y!l9s7@k6EvxS{AMP>WY zX4$@F8^kayphIx-RGO$+LYl9YdoI5d|4#q9##`_F5Xnx`&GPzp2fB{-{P@ATw=X@~ z_|&^UMWAKD;jjBKTK(~o?cUFRK8EX=6>cXpfzg4ZpMB>*w_^8GSiT-Jp|xBOnzM+j z*09-@-~qJ(eqWq5@R4i^u4^{McCP(!3}C|v_WsTR*bIUxN(Nx`u##3B4{sE`Z`v8w zAwIG`?1~PkID~W{uDzmqH98Pew_1(;x2%8r^vY{)_&J2K)cN{W+h5+g)ZcjP&Ci#O zgy|8K@4kyMfwilHd&6TDlhb%++Pk!>9HRld6HT7gwyZGrxS$}CsD6`>6!!2K1@Mjf z(P0WYB7V_OFZyeWrbOFb>O54BNXf~K&?}3=^v;v_wT{DKr?jN^DtN&DXwX%u?s*c6`%8>WFz z7}YW^tp0bp^NriE)AB6M2l<7rn7fzePtR*omOevpfm9n?}2V*+0iW;S)C zhg`NAjL?D=W#k*$aR{>pGf~lD-rVtD;5jW1_*Jn1j1=es@Kcx4ySM_bwcQCT=d+DV z>Sz~L=Hj@(X%31nK$mWI@7d>}ORB`K(p=+`UD)+99YUGQc7y^bHZ1F(8|tL0 zdK*DT0kSXG_{BKTpP2*2PecdKV9;dq$^ZZDP;Nyq1kp-&GI5eAyZsK!e3V zK@rPy*{(`KIfo+lc878mDKk^V#`VT05}64kBtk%DgwLrOvLMj5-;*GNKv6c6pzMuL z6EP%ob|_0IW}lLRXCP2!9wWhEw3LA7iF#1O1mIZ@Z=6&bz41F;@S_GvYAG-#CW3z{ zP3+6vHhvP&A3$##Vo9$dT^#MoGg^|MDm=Bt1d2RRwSZ<;ZHICpLBv5Xs!D?BH^(9_ z7`H=N&^v|Z-%mP}wNzG{aiFCsRgwzwq!N6obW9+7(R; z(SZ=23`|`>qil!LMGG{_Heq!BD>(Y-zV9wD)}hz25JA37YR%39;kI4y9pgtcUass6 zP24}ZY$vvYeI`zy&)A_X#nY3017ap*0&jx|mVwyGhg3;!keU53a}Uhm3BZI$N$6Se zLWlAmy1S0xKJm4G_U@sN_Tm=`$xWJSEwKU98rZ&)1R^*$$1vA3oG#&*%SMxY_~oGP zP&PFJatFLM-Ps%84IV-+Ow)T{C7cqUAvauy4C z(FRz&?6$Rypj{xO!`y=*J5o4@U8Q-(y5(*=YoKeZ+-1YdljXxkA#B)zo=FeQH#?Le zycNUmEEHWO9a=X^pb#&cOq7-`7UA87#|S22)<7RUtZo|(zibX=w;K3qur9vy#`MNV z6UUcf9ZwEnKCCp+OoBnF@OdbvH)ANXO0o~Pi9l8=x3))}L<#vO0-~O4!~--Ket?d} zJaqsj<@CD1%S2cTW%rOP{Vto%0sGW~1RMa_j^)5nil0Yw- z0EE#bP+l4#P^%PQ+N*oxu1Zq05xZ!bXfYTg>9c{(Iw*lnjR^>kz%lAN^zFce7rppy zY8zA~3GD=A6d*hze&l4D_wA~+O!56)BZTe_rEu}Ezi<4!kG|W#amBZ5{&XS2@6R~H z{9o^y*BkH4$~yX9U&@CgbOzX1bn9xqF|zh$Dh0Y5y*E0e90*$!ObrHY3Ok0`2=O~r zCuke6KrP9KOf?V(YDsM<6pX2nVoN%M$LT^q#FmtaF?1^27F*IcNX~XRB(|hCFvdcc zc)$=S-)acdk$g4?_>jRqxpI6M3vHZk?0c^3=byamYDNf;uB{3NlKW5IhnOS3DNkMV z?tK8?kJ}pmvp%&&eTVOVjHP`q34hN1@!aK}H(K!vI`~gf|Gv+FNEQD5Yd<~yX7k_l h&G-K)@HZb3BABY{)U1?^%I#E6`MGoTtustd{~yM6srvu` literal 0 HcmV?d00001 diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/Square150x150Logo.scale-200.png b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/Square150x150Logo.scale-200.png new file mode 100644 index 0000000000000000000000000000000000000000..af49fec1a5484db1d52a7f9b5ec90a27c7030186 GIT binary patch literal 2937 zcma)84OCO-8BSud5)jwMLRVKgX(S?$n?Ld|vrsm<$CF7)&zTbyy1FE5bU`Q17MRv`9ue$;R(@8kR;#vJ*IM0>cJIAOte!d7oRgdH zd%ySjdB6L9=gX^A6)VzH7p2l@v~3zJAMw|DFy#^)F@@F*`mqUn=Il>l)8_+ab;nOW{%+iPx z+s{Eu|&pIs)Z7{La9~?xKfyl z#43?gjEL15d4WbOZo#SiP%>DB^+BcnJ=7dHEe;r#G=tuw|ka z%q@}##Uh7;tc%L_64m(kHtw74ty%BJMb)_1)#S0j`)F8_1jF7vScpsnH=0V19bO8y zR`0SjIdCUo&=>JwMQF8KHA<{ODHTiQh}0^@5QRmCA?gOH6_H3K^-_sNB^RrdNuK-R zOO*vOrKCVvDwgUck`kF(E7j{I#iiN;b*ZdCt4m@HPA`EuEqGGf4%!K<;(=I=&Vyrw z%TwcWtxa}8mCZ%Cyf&ActJ6_$ox5z6-D!0-dvnRx6t7y3d+h6QYpKWO;8OdnvERo7 zuEf>ih5`wqY)~o@OeVt-wM?Q!>QzdGRj!bz6fzYrfw$hZfAKzr2-M+D+R>}~oT574c;_3zquHcElqKIsryILt3g8n3jcMb+j?i?-L3FpZJ z2WRVBRdDPc+G5aaYg#5hpE+6nQ|(VSoxT3|biF;BUq#==-27Xi=gihDPYP$7?=9cP zYKE$jeQ|3~_L0VG-(F~2ZPyD0=k{J4Q~h(t__{-mz_w8{JDY9{`1ouzz!Vr5!ECdE z6U~O1k8c}24V7~zzXWTV-Pe4)y}wQJS&q%H5`Fo_f_JvIU489aCX$;P`u#!I-=^4ijC2{&9!O&h>mi?9oYD=GC#%)6{GzN6nQYw+Fal50!#x^asjBBR50i`+mho*ttoqV)ubM2KD9S~k7+FR4>{29?6 z{!l6kDdyTN0YJ9LgkPWeXm|gyi@zM3?0@{&pXT12w|78&W-q!RRF)&iLCEZVH<|fR zN0fr2^t8H(>L?>K#>^+jWROLral(Qy-xoBq1U7A&DV||wClb)Otd9?(gZ|8znMF}D zf<1haWz^s0qgecz;RFGt0C-B4g`jNGHsFU+;{<%t65v^sjk^h$lmWn#B0#_)9ij&d z-~lc`A)YYExi^7sBuPM^Y|wA2g*5?`K?#7tzELQYNxGo$UB$4J8RJp1k(8Jj+~hMT zlN~>M@KTTh^--8y3PK_NZ@AC!{PT=CziBzGd+wTJ^@icH!Bd}%)g8V)%K?|c&WTUk zy}qv1C%(fjRoZ4ozC3{O%@5?)XzH35zHns$pgU*Q?fj4v?fp1Qbm+j;3l;9jam9Da zXVcKjPlQ73x78QPu|Ffm6x?`~e3oD=gl=4kYK?={kD5j~QCXU)`HSdduNNENzA*2$ zOm3PzF!lN5e*06-f1Uot67wY#{o-S1!KZ7E=!~7ynnk9_iJR#kFoNbAOT#^2Gd17F zMmvU6>lndZQGd|ax9kUoXXO+$N?|j@6qpsF&_j7YXvwo_C{JpmLw5&#e6k>atv%es z5)7r*Wvv_JkUpT}M!_o!nVlEk1Zbl=a*2hQ*<|%*K1Glj^FcF`6kTzGQ3lz~2tCc@ z&x|tj;aH&1&9HwcJBcT`;{?a+pnej;M1HO(6Z{#J!cZA04hnFl;NXA+&`=7bjW_^o zfC40u3LMG?NdPtwGl>Tq6u}*QG)}-y;)lu-_>ee3kibW(69n0$0Zy!}9rQz%*v1iO zT9_H>99yIrSPYVy6^);rR}7Yo=J_T@hi+qhTZXnVWyf;JDYm5#eYLTxr*?kiNn!+Y zQ+LUkBafNJ#rH#C(?d5^;gw9o#%daEI{mA*LHPIHPU`#|H$hD zwm>0&+kahQ)E#%~k>&5@&#Vg82H?s%71=)(soi@174pi9--2{w{1$}Sz4zGn3Du&x bht0Iza^2ykEt4(epJ78uh5nDlX8(TxzDYwP literal 0 HcmV?d00001 diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/Square44x44Logo.scale-200.png b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/Square44x44Logo.scale-200.png new file mode 100644 index 0000000000000000000000000000000000000000..ce342a2ec8a61291ba76c54604aea7e9d20af11b GIT binary patch literal 1647 zcmaJ?eM}Q)7(e+G1Q(|`V9JhTI2>MkceK4;p;PR&$Pi?ejk3YQ_3o`S&|W_dsOZ8# zWPTt69g`t$ab`0cj-Y0yiBSOqmd)tG7G(}M5aP0_%&9TijB#&)I{zSE^4@#z^FF`l z`8{8`o%wlL(UI|y2!cdsuVamHH~H86F!*-15em4)NqUpCQM5?aoC_eCf@lV4wvF2a zjDQn1JBL69f&@2M3rvzJcfE!eZ8FZUBlFlC5RD)it33{mF9#B82AiyQE%w)`vlwa> zv{<1sm&kSKK$&%2jSFn7$t&P%%6Ue>R=EAnG8N7fqynWG8L3p!4801a;8{+nliO(qd(jNJ_?+9W3#hLIDLoT6~3fx9=`CC-D}-AMrpEO7HK zt3$GicGPc?GmDjy7K2P@La;eu4!$zWCZ`ym{Z$b zu-O6RM&K4JT|BIZB`E-gxqG%FzanI#+2FFmqHqXG7yxWB=w55RGOM)$xMb(>kSNR z2w=1AZi%z=AmG~yea~XaXJR!v7vLn(RUnELfiB1|6D84ICOS}^Zo2AdN}<&*h}G_u z{xZ!(%>tLT3J3<5XhWy-tg+6)0nmUUENLW8TWA{R6bgVd3X;anYFZ^IRis*_P-C-r z;i>%1^eL3UI2-{w8nuFFcs0e~7J{O2k^~Ce%+Ly4U?|=!0LH=t6()xi<^I-rs+9sF z*q{E-CxZbGPeu#a;XJwE;9S1?#R&uns>^0G3p`hEUF*v`M?@h%T%J%RChmD|EVydq zmHWh*_=S%emRC*mhxaVLzT@>Z2SX0u9v*DIJ@WC^kLVdlGV6LpK$KIrlJqc zpJ921)+3JJdTx|<`G&kXpKkjGJv=76R`yYIQ{#c-`%+`#V(7}Q;&@6U8!Td1`d;?N z_9mnI#?AA}4J!r)LN4!E-@H5eXauuB7TOawS>Y|{-P?NNx-lq+z1W-+y(;39P&&LP zL{N80?&=C*qKmdA^moMZRuPcD!B<*mq$ch=0Cnlitw#txRWhb3%TQvPqjkC`F69G4b! ze7z9MZ#+;_#l?H37UqUhDFb^l&s2{oM$3I0o^Q!yx;;V)QmCMo)Tb_ui|mit8MS?U zm##6$sZZ1$@|s%?l@>4Z<*Q}sRBSKMhb4I{e5LdEhsHIHTe8Bod5c>6QtT>$XgUBz z6MK`kO$=jmt@FqggOhJ5j~e@ygRbG;<{Vu)*+nn9aQeo0;$#j;|MS=S$&L?BeV25z xs3B`@=#`5TF{^6(A1rvdY@|-RtQ|iS5{tyX+wH?;n8E)G$kykv-D^wh{{!TZT%7;_ literal 0 HcmV?d00001 diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/Square44x44Logo.targetsize-24_altform-unplated.png b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/Square44x44Logo.targetsize-24_altform-unplated.png new file mode 100644 index 0000000000000000000000000000000000000000..f6c02ce97e0a802b85f6021e822c89f8bf57d5cd GIT binary patch literal 1255 zcmaJ>TWs4@7*5+{G#S+&C!qC#> zf>5N3P6jO*Cz>ug*(_DmW=)kea&m$gZ^+nyiF`;j%w@}y8)>p*SH}C`m?DXeieF2U zyQHecc_L%Gh!7GMt+hG06y;+|p4>m~}PjA}rKViGiEnn7G0ZO<>G|7q;2?NwGCM3s?eued6%hd$B+ z*kQJ{#~$S=DFE(%=E+UkmlEI*%3llUf~8Ja9YU1Vui0IbGBkW_gHB%Rd&!!ioX zs40O?i9I{};kle7GMvE7(rk`la=gTI)47=>%?q@^iL-nUo3}h4S}N-KHn8t5mVP8w z&bSErwp+37 zNJJ8?a|{r5Q3R0Z5s-LB1WHOwYC@7pCHWND#cL1cZ?{kJ368_*(UDWUDyb<}0y@o# zfMF016iMWPCb6obAxT$JlB6(2DrlXDTB&!0`!m??4F(qWMhjVZo?JXQmz`1*58Z=& zcDmB|S-E@j?BoFGix0flckqdS4jsPNzhfWyWIM98GxcLs89C(~dw%$_t;JjX-SD}E zfiGV;{8Q%8r}w9x>EEigW81>`kvnU@pK)4+xk9@+bNj9L!AAZ@SZ@q|)&BmY3+HZx zul~BeG4|}-;L%cHViQGQX?^zFfO0&#cHwel=d`lH9sJ-@Sl@n*(8J2>%Ac`IxyY?Q z{=GhWvC#gu-~Ia7*n{=+;qM?Ul_wy1+u7ho;=`>EwP^g~R@{unBds`!#@}tluZQpS zm)M~nYEifJWJGx?_6DcTy>#uh%>!H9=hb^(v`=m3F1{L>db=<5_tm+_&knAQ2EU$s Mu9UqpbNZeC0BbUo^Z)<= literal 0 HcmV?d00001 diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/StoreLogo.png b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/StoreLogo.png new file mode 100644 index 0000000000000000000000000000000000000000..7385b56c0e4d3c6b0efe3324aa1194157d837826 GIT binary patch literal 1451 zcmaJ>eN5D57_Z|bH;{0+1#mbl)eTU3{h)Wf7EZV?;HD@XL@{B`Ui%(2aMxQ~xdXSv z5nzWi(LW)U2=Vc-cY@s7nPt{i0hc6!7xN4NNHI#EQl>YNBy8l4%x9gr_W-j zEZMQmmTIy(>;lblRfh`dIyTgc9W5d!VP$L4(kKrN1c5G~(O_#xG zAJCNTstD^5SeXFB+&$h=ToJP2H>xr$iqPs-#O*;4(!Fjw25-!gEb*)mU}=)J;Iu>w zxK(5XoD0wrPSKQ~rbL^Cw6O_03*l*}i=ydbu7adJ6y;%@tjFeXIXT+ms30pmbOP%Q zX}S;+LBh8Tea~TSkHzvX6$rYb)+n&{kSbIqh|c7hmlxmwSiq5iVhU#iEQ<>a18|O^Sln-8t&+t`*{qBWo5M?wFM(JuimAOb5!K#D}XbslM@#1ZVz_;!9U zpfEpLAOz=0g@bd6Xj_ILi-x^!M}73h^o@}hM$1jflTs|Yuj9AL@A3<-?MV4!^4q`e z)fO@A;{9K^?W?DbnesnPr6kK>$zaKo&;FhFd(GYFCIU^T+OIMb%Tqo+P%oq(IdX7S zf6+HLO?7o0m+p>~Tp5UrXWh!UH!wZ5kv!E`_w)PTpI(#Iw{AS`gH4^b(bm^ZCq^FZ zY9DD7bH}rq9mg88+KgA$Zp!iWncuU2n1AuIa@=sWvUR-s`Qb{R*kk(SPU^`$6BXz8 zn#7yaFOIK%qGxyi`dYtm#&qqox0$h=pNi#u=M8zUG@bpiZ=3sT=1}Trr}39cC)H|v zbL?W)=&s4zrh)7>L(|cc%$1#!zfL?HjpeP%T+x_a+jZ16b^iKOHxFEX$7d|8${H-* zIrOJ5w&i$>*D>AKaIoYg`;{L@jM((Kt?$N$5OnuPqVvq**Nm}(f0wwOF%iX_Pba;V z;m@wxX&NcV3?<1+u?A{y_DIj7#m3Af1rCE)o`D&Y3}0%7E;iX1yMDiS)sh0wKi!36 zL!Wmq?P^Ku&rK~HJd97KkLTRl>ScGFYZNlYytWnhmuu|)L&ND8_PmkayQb{HOY640 bno1(wj@u8DCVuFR|31B*4ek@pZJqxCDDe1x literal 0 HcmV?d00001 diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/Wide310x150Logo.scale-200.png b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Assets/Logo/Wide310x150Logo.scale-200.png new file mode 100644 index 0000000000000000000000000000000000000000..288995b397fdbef1fb7e85afd71445d5de1952c5 GIT binary patch literal 3204 zcmbVPeQXow8NYmBd90>}0NP?GhXW~VaeThm=a0tV#EwJMI!)6M3}|c4_Bl3=Kd>G0 z(GHx1wl<7(tP?FsOQkTilSo*iIvF%uArExJ73~P zSv1xEy!U(Wd4A9D`FQV@W3@F^qJ@PEF$@z`Z!*BbFsS(^?B zyiAzJ+q})bkgiQHWqEb*jJD-coHYr1^iocg)l!Qa{Xqs-l~6J}p-|##ZHYofskQ3$ zI0;xzXyhazBeXhIsg5A=%ufo@f)1yy&ScKS0;HF^!r_2UE^lpZEom(+@duma3awTv zCrCL-%D_SvYWIcdHkmI}#50(fkUi)Qgx!80ju>g1za^}ff>JI8Z@^-iCiaCgg@TgF z+vtE?Q9{VQUX&MW9SYYmGcxA14%N2@7FwBTD4N<(2{nWgV8$e3?-F=L^&FrtWn~(U_Q~~^uYiyeY6-KoTnfh9AWz@ zIKje0)u!_Lw)E}G!#kEfwKVdNt(UAf9*f>tEL_(=xco-T%jTi@7YlC3hs2ik%Le0H ztj}RTeCF(5mwvi3_56>-yB?l;J>-1%!9~=fs|QcNG3J~a@JCu`4SB460s0ZO+##4fFUSGLcj_ja^fL4&BKALfb#$6$O?>P@qx2Agl^x0i&ugt zsy5Pyu=()`7HRMG3IB7F1@`_ z+-!J%#i6e^U$e#+C%Q>_qVRzWRsG^W_n+@OcX@vzI&z;mzHNb!GQ?LWA(wtpqHqTM z1OFw_{Zn?fD)p)`c`kOgv{de=v@suGRqY{N^U7gI1VF3*F=obwaXI6ob5__Yn zVTguS!%(NI09J8x#AO_aW!9W7k*UvB;IWDFC3srwftr{kHj%g)fvnAm;&h_dnl~

MY- zf+K}sCe8qU6Ujs`3ua{U0Of$R_gVQBuUA za0v=mu#vIOqiiAZOr&h*$WyOw&k-xr$;G4Ixa!#TJNr>95(h>l%)PUy4p+^SgR(uR zta%k*?ny-+nAr8spEk1fo{J4i!b^Fia`N{_F6@zidA2ZTTrjl#^5Z-2KfB@Cu}l9s z(*|Z2jc?p~vn2f)3y9i*7zJV1L{$?|&q)4oaT;uXi6>1GkRXVTOzAz(RHEmr=eFIi z`}<>-Q?K0GN8!IYxeP1XKXO+jsJbp~o^);Bc;%b7Flpe7;1`Ny@3r7ZR;?R)aJt8C ziNlEC<@3f_lIV4TwV}&e;D!Ee5_|e#g0LUh=5vmYWYm7&2h*M>QPKvGh9-)wfMMW3 z8J9b%1k7dzPzO0_NGQy92BZ^FR6R~6;^6?lqO;-QUP4BY%cG%3vEhbm#>4vIhPBh3 z-+pZGjh$x%Hp{?=FHsMp0&wNPlj00us{&`1ZOZTqs8%4X&xH=UDr*xyBW(Zp&Em94 zf)ZSfn#yg0N)>!1kWdkqJ^S*z0FF5|fj&qcE#Na|%OY0$uO>!&hP+1ywfD_WXk@4J(?MBftK7>$Nvqh@tDuarN%PrTLQ2Uzysx>UV=V zk^RrDSvdQ?0;=hY67EgII-f4`t=+i*yS=Y~!XlqIy_4x&%+OdfbKOFPXS2X5%4R{N z$SQMX^AK6(fA + /// Capture app used to stream sensor data to the accompanying HoloLensCaptureServer. + /// + public class HoloLensCaptureApp + { + // version number shared by capture app and server to ensure compatiblity. + private const string Version = "v1"; + + // image quality settings for JPEG encoder (0.0 - 1.0) + private const double GrayImageJpegQuality = 0.8; + private const double InfraredImageJpegQuality = 0.9; + + // frame edges + private const float FrameThickness = 0.001f; + private const float FrameDistance = 0.30f; + private const float FrameWidth = 0.285f; + private const float FrameHeight = 0.145f; + private const float FrameBottomClip = 0.023f; // clip bottom of frame to stay within camera view + private const float FrameLabelInset = 0.006f; + + // Config settings + private static readonly bool IncludeDiagnostics = true; + + private static readonly bool IncludeVideo = true; + private static readonly bool IncludePreview = false; + + // see https://docs.microsoft.com/en-us/windows/mixed-reality/develop/platform-capabilities-and-apis/locatable-camera#hololens-2 + // for different possible modes, such as: 896x504 @30/15; 960x540 @30,15; 1128x636 @30,15; 1280x720 @30/15 etc. + private static readonly int PhotoVideoFps = 30; + private static readonly int PhotoVideoImageWidth = 896; + private static readonly int PhotoVideoImageHeight = 504; + + private static readonly bool IncludeDepth = true; + private static readonly bool IncludeDepthCalibrationMap = false; + private static readonly bool IncludeAhat = false; + private static readonly bool IncludeAhatCalibrationMap = false; + private static readonly bool IncludeInfrared = false; + private static readonly bool EncodeInfrared = true; + + private static readonly bool IncludeGrayFrontCameras = true; + private static readonly bool IncludeGrayFrontCameraCalibrationMap = false; + private static readonly bool IncludeGraySideCameras = false; + private static readonly bool IncludeGraySideCameraCalibrationMap = false; + private static readonly GrayImageEncode EncodeGrayMethod = GrayImageEncode.Jpeg; + private static readonly TimeSpan GrayInterval = TimeSpan.FromMilliseconds(100); + private static readonly TimeSpan CalibrationMapInterval = TimeSpan.FromHours(1); + + private static readonly string CalibrationFolderName = "Calibration"; + + private static readonly bool IncludeImu = true; + + private static readonly bool IncludeHead = true; + private static readonly TimeSpan HeadInterval = TimeSpan.FromMilliseconds(20); + private static readonly bool IncludeEyes = true; + private static readonly TimeSpan EyesInterval = TimeSpan.FromMilliseconds(20); + private static readonly bool IncludeHands = true; + private static readonly TimeSpan HandsInterval = TimeSpan.FromMilliseconds(20); + + private static readonly bool IncludeAudio = true; + + private static readonly bool IncludeSceneUnderstanding = true; + private static readonly TimeSpan SceneUnderstandingInterval = TimeSpan.FromSeconds(60); + private static readonly SceneQuerySettings SceneUnderstandingSettings = new () + { + EnableSceneObjectMeshes = true, + EnableSceneObjectQuads = true, + EnableWorldMesh = true, + EnableOnlyObservedSceneObjects = true, + RequestedMeshLevelOfDetail = SceneMeshLevelOfDetail.Medium, + }; + + private static readonly Rectangle3D FrameRectangle = new ( + new Point3D(FrameDistance, 0, 0), + UnitVector3D.YAxis.Negate(), + UnitVector3D.ZAxis, + -(FrameWidth / 2), + -(FrameHeight / 3) * 2 + FrameBottomClip, + FrameWidth, + FrameHeight - FrameBottomClip); + + private static readonly Vec2 LabelSize = new Vec2(FrameWidth - FrameLabelInset * 2f, 0.008f); + + private static string captureServerAddress = "0.0.0.0"; + + private enum State + { + WaitingToStart, // showing menu to Start / Exit + ConstructPipeline, // construct pipeline to capture + ConstructingPipeline, // constructing the pipeline + CalibrateCameras, // calibrate the cameras + CalibratingCameras, // camera calibration in progress + ConnectToCaptureServer, // connect to the capture server + WaitingForCaptureServer, // wating for capture server to connect + Capturing, // pipeline running and capturing streams + StoppingPipeline, // stopping pipeline + Exited, // app has exited + } + + private enum GrayImageEncode + { + None, + Jpeg, + Gzip, + } + + private static async Task Main(string[] args) + { + // Initialize StereoKit + if (!SK.Initialize( + new SKSettings + { + appName = "HoloLensCaptureApp", + assetsFolder = "Assets", + })) + { + throw new Exception("StereoKit failed to initialize."); + } + + // Initialize MixedReality statics + await MixedReality.InitializeAsync(); + + // Attempt to get server address from config file + const string configFile = "CaptureServerIP.txt"; + var docs = KnownFolders.DocumentsLibrary; + try + { + var config = await docs.GetFileAsync(configFile); + captureServerAddress = await FileIO.ReadTextAsync(config); + } + catch (FileNotFoundException) + { + // save new config file only if the server address has been changed from default + if (!string.Equals(captureServerAddress, "0.0.0.0")) + { + var config = await docs.CreateFileAsync(configFile, CreationCollisionOption.FailIfExists); + await FileIO.WriteTextAsync(config, captureServerAddress); + } + } + + var pipeline = default(Pipeline); + + var accelerometer = default(Accelerometer); + var gyroscope = default(Gyroscope); + var magnetometer = default(Magnetometer); + var camera = default(PhotoVideoCamera); + var scene = default(SceneUnderstanding); + var depthCamera = default(DepthCamera); + var depthAhatCamera = default(DepthCamera); + var leftFrontCamera = default(VisibleLightCamera); + var rightFrontCamera = default(VisibleLightCamera); + var leftLeftCamera = default(VisibleLightCamera); + var rightRightCamera = default(VisibleLightCamera); + var head = default(HeadSensor); + var eyes = default(EyesSensor); + var hands = default(HandsSensor); + IProducer audio = null; + + string errorMessage = null; + var windowCoordinateSystem = default(CoordinateSystem); + var windowPose = default(Pose); + var captureServerProcess = default(Rendezvous.Process); + + var lastServerHeartBeat = DateTime.MaxValue; + var serverHeartBeatTimeout = TimeSpan.FromSeconds(5); + + var videoFps = 0f; + var depthFps = 0f; + + var state = State.WaitingToStart; + + while (SK.Step(() => + { + // Setup the window coordinate system (the startup location of the menu) + if (windowCoordinateSystem == null) + { + var head = Input.Head.ToCoordinateSystem(); + var ahead = head.Origin + head.XAxis.ScaleBy(0.7); + var origin = new Point3D(ahead.X, ahead.Y, head.Origin.Z); + var axisX = (head.Origin - origin).Normalize(); + var axisY = UnitVector3D.ZAxis.CrossProduct(axisX); + var axisZ = axisX.CrossProduct(axisY); + windowCoordinateSystem = new CoordinateSystem(origin, axisX, axisY, axisZ); + windowPose = windowCoordinateSystem.ToStereoKitPose(); + } + + // Setup the handle + var windowBounds = Bounds.FromCorner(new Vec3(0, 0, -2) * U.cm, new Vec3(4, 4, 4) * U.cm); + + switch (state) + { + case State.WaitingToStart: + UI.EnableFarInteract = true; + lastServerHeartBeat = DateTime.MaxValue; + UI.HandleBegin("Handle", ref windowPose, windowBounds, true); + UI.Label($"Server: {captureServerAddress}"); + if (UI.Button($"Start")) + { + state = State.ConstructPipeline; + } + + if (UI.Button("Exit")) + { + state = State.Exited; + Task.Run(() => SK.Shutdown()); + } + + UI.HandleEnd(); + break; + case State.ConstructPipeline: + UI.HandleBegin("Handle", ref windowPose, windowBounds, true); + UI.Label("Please wait! Constructing capture pipeline ..."); + UI.HandleEnd(); + DrawFrame(Color.Yellow.ToStereoKitColor()); + state = State.ConstructingPipeline; + Task.Run(() => + { + pipeline = Pipeline.Create( + enableDiagnostics: IncludeDiagnostics, + diagnosticsConfiguration: new DiagnosticsConfiguration() + { + SamplingInterval = TimeSpan.FromSeconds(5), + }); + + // IMU SENSORS + accelerometer = IncludeImu ? new Accelerometer(pipeline) : null; + gyroscope = IncludeImu ? new Gyroscope(pipeline) : null; + magnetometer = IncludeImu ? new Magnetometer(pipeline) : null; + + // HEAD, EYES, AND HANDS + head = IncludeHead ? new HeadSensor(pipeline, HeadInterval) : null; + eyes = IncludeEyes ? new EyesSensor(pipeline, EyesInterval) : null; + hands = IncludeHands ? new HandsSensor(pipeline, HandsInterval) : null; + + // AUDIO + audio = IncludeAudio ? new Microphone(pipeline).Reframe(16384, DeliveryPolicy.Unlimited) : null; + + // PHOTOVIDEO CAMERA + var videoStreamSettings = IncludeVideo ? new PhotoVideoCameraConfiguration.StreamSettings + { + FrameRate = PhotoVideoFps, + ImageWidth = PhotoVideoImageWidth, + ImageHeight = PhotoVideoImageHeight, + OutputEncodedImage = false, + OutputEncodedImageCameraView = true, + } + : null; + + var previewStreamSettings = IncludePreview ? new PhotoVideoCameraConfiguration.StreamSettings + { + FrameRate = PhotoVideoFps, + ImageWidth = PhotoVideoImageWidth, + ImageHeight = PhotoVideoImageHeight, + OutputEncodedImage = false, + OutputEncodedImageCameraView = true, + MixedRealityCapture = new (), + } + : null; + + camera = IncludeVideo || IncludePreview ? new PhotoVideoCamera(pipeline, new PhotoVideoCameraConfiguration + { + VideoStreamSettings = videoStreamSettings, + PreviewStreamSettings = previewStreamSettings, + }) + : null; + + // DEPTH CAMERA - LONG THROW + depthCamera = IncludeDepth ? new DepthCamera(pipeline, new DepthCameraConfiguration + { + DepthSensorType = ResearchModeSensorType.DepthLongThrow, + OutputCameraIntrinsics = false, + OutputPose = false, + OutputDepthImage = false, + OutputDepthImageCameraView = true, + OutputInfraredImage = IncludeInfrared, + OutputCalibrationPointsMap = IncludeDepthCalibrationMap, + OutputCalibrationPointsMapMinInterval = CalibrationMapInterval, + }) + : null; + + // DEPTH CAMERA - AHAT + depthAhatCamera = IncludeAhat ? new DepthCamera(pipeline, new DepthCameraConfiguration + { + DepthSensorType = ResearchModeSensorType.DepthAhat, + OutputCameraIntrinsics = false, + OutputPose = false, + OutputDepthImage = false, + OutputDepthImageCameraView = true, + OutputInfraredImage = false, + OutputCalibrationPointsMap = IncludeAhatCalibrationMap, + OutputCalibrationPointsMapMinInterval = CalibrationMapInterval, + }) + : null; + + // GRAY FRONT CAMERAS + leftFrontCamera = IncludeGrayFrontCameras ? new VisibleLightCamera(pipeline, new VisibleLightCameraConfiguration + { + VisibleLightSensorType = ResearchModeSensorType.LeftFront, + OutputMinInterval = GrayInterval, + OutputCameraIntrinsics = false, + OutputPose = false, + OutputImage = false, + OutputImageCameraView = true, + OutputCalibrationPointsMap = IncludeGrayFrontCameraCalibrationMap, + OutputCalibrationPointsMapMinInterval = CalibrationMapInterval, + }) + : null; + + rightFrontCamera = IncludeGrayFrontCameras ? new VisibleLightCamera(pipeline, new VisibleLightCameraConfiguration + { + VisibleLightSensorType = ResearchModeSensorType.RightFront, + OutputMinInterval = GrayInterval, + OutputCameraIntrinsics = false, + OutputPose = false, + OutputImage = false, + OutputImageCameraView = true, + OutputCalibrationPointsMap = IncludeGrayFrontCameraCalibrationMap, + OutputCalibrationPointsMapMinInterval = CalibrationMapInterval, + }) + : null; + + // GRAY SIDE CAMERAS + leftLeftCamera = IncludeGraySideCameras ? new VisibleLightCamera(pipeline, new VisibleLightCameraConfiguration + { + VisibleLightSensorType = ResearchModeSensorType.LeftLeft, + OutputMinInterval = GrayInterval, + OutputCameraIntrinsics = false, + OutputPose = false, + OutputImage = false, + OutputImageCameraView = true, + OutputCalibrationPointsMap = IncludeGraySideCameraCalibrationMap, + OutputCalibrationPointsMapMinInterval = CalibrationMapInterval, + }) + : null; + + rightRightCamera = IncludeGraySideCameras ? new VisibleLightCamera(pipeline, new VisibleLightCameraConfiguration + { + VisibleLightSensorType = ResearchModeSensorType.RightRight, + OutputMinInterval = GrayInterval, + OutputCameraIntrinsics = false, + OutputPose = false, + OutputImage = false, + OutputImageCameraView = true, + OutputCalibrationPointsMap = IncludeGraySideCameraCalibrationMap, + OutputCalibrationPointsMapMinInterval = CalibrationMapInterval, + }) + : null; + + // SCENE UNDERSTANDING + scene = IncludeSceneUnderstanding ? new SceneUnderstanding(pipeline, new SceneUnderstandingConfiguration + { + MinQueryInterval = SceneUnderstandingInterval, + SceneQuerySettings = SceneUnderstandingSettings, + }) : null; + + if (IncludeDepth || IncludeAhat || IncludeGrayFrontCameras || IncludeGraySideCameras) + { + state = State.CalibrateCameras; + } + else + { + state = State.ConnectToCaptureServer; + } + }); + break; + case State.ConstructingPipeline: + UI.HandleBegin("Handle", ref windowPose, windowBounds, true); + if (errorMessage != null) + { + UI.Label($"Error: {errorMessage}"); + } + + UI.Label("Please wait! Constructing capture pipeline ..."); + UI.HandleEnd(); + DrawFrame(Color.Yellow.ToStereoKitColor()); + break; + case State.CalibrateCameras: + UI.HandleBegin("Handle", ref windowPose, windowBounds, true); + UI.Label("Please wait! Calibrating cameras ..."); + UI.HandleEnd(); + DrawFrame(Color.Yellow.ToStereoKitColor()); + state = State.CalibratingCameras; + + // Calibrate all cameras with async tasks + List calibrationTasks = new (); + Task.Run(async () => + { + // Open (or create) the Documents folder containing calibration files + var calibrationFolder = await docs.CreateFolderAsync(CalibrationFolderName, CreationCollisionOption.OpenIfExists); + calibrationTasks.Add(depthCamera?.CalibrateFromFileAsync(calibrationFolder)); + calibrationTasks.Add(depthAhatCamera?.CalibrateFromFileAsync(calibrationFolder)); + calibrationTasks.Add(leftFrontCamera?.CalibrateFromFileAsync(calibrationFolder)); + calibrationTasks.Add(rightFrontCamera?.CalibrateFromFileAsync(calibrationFolder)); + calibrationTasks.Add(leftLeftCamera?.CalibrateFromFileAsync(calibrationFolder)); + calibrationTasks.Add(rightRightCamera?.CalibrateFromFileAsync(calibrationFolder)); + await Task.WhenAll(calibrationTasks.Where(t => t is not null).ToArray()); + state = State.ConnectToCaptureServer; + }); + break; + case State.CalibratingCameras: + UI.HandleBegin("Handle", ref windowPose, windowBounds, true); + if (errorMessage != null) + { + UI.Label($"Error: {errorMessage}"); + } + + UI.Label("Please wait! Calibrating cameras ..."); + UI.HandleEnd(); + DrawFrame(Color.Yellow.ToStereoKitColor()); + break; + case State.ConnectToCaptureServer: + UI.HandleBegin("Handle", ref windowPose, windowBounds, true); + UI.Label($"Please wait! Connecting to capture server: {captureServerAddress}"); + UI.HandleEnd(); + DrawFrame(Color.Yellow.ToStereoKitColor()); + state = State.WaitingForCaptureServer; + Task.Run(() => + { + // Create the rendezvous client and process + var rendezvousClient = new RendezvousClient(captureServerAddress); + var remoteClock = new RemoteClockExporter(); + + void ResetState() + { + rendezvousClient?.Rendezvous.TryRemoveProcess(nameof(HoloLensCaptureApp)); + rendezvousClient?.Dispose(); + remoteClock?.Dispose(); + captureServerProcess = null; + errorMessage = null; + state = State.WaitingToStart; + } + + AppDomain.CurrentDomain.ProcessExit += (_, _) => + { + ResetState(); + }; + + pipeline.PipelineExceptionNotHandled += (_, ex) => + { + Trace.WriteLine($"Pipeline Error: {ex.Exception.Message}"); + ResetState(); + }; + + pipeline.PipelineCompleted += (_, _) => + { + ResetState(); + }; + + try + { + rendezvousClient.Start(); + } + catch (Exception ex) + { + errorMessage = ex.Message; + } + + if (rendezvousClient.IsActive) + { + rendezvousClient.Connected.WaitOne(); + rendezvousClient.Rendezvous.TryRemoveProcess(nameof(HoloLensCaptureApp)); // in case a previous instance crashed + + var headsetAddress = rendezvousClient.ClientAddress; + + var process = new Rendezvous.Process(nameof(HoloLensCaptureApp), Version); + + // Sync clocks + process.AddEndpoint(remoteClock.ToRendezvousEndpoint(headsetAddress)); + + // Publish streams to the rendezvous process + void Write(string name, IProducer producer, int port, IFormatSerializer serializer, DeliveryPolicy deliveryPolicy) + { + var tcpWriter = new TcpWriter(pipeline, port, serializer); + producer.PipeTo(tcpWriter, deliveryPolicy); + process.AddEndpoint(tcpWriter.ToRendezvousEndpoint(headsetAddress, name)); + } + + var port = 30000; + + if (IncludeImu) + { + Write("Accelerometer", accelerometer?.Out, port++, Serializers.ImuFormat(), DeliveryPolicy.LatestMessage); + Write("Gyroscope", gyroscope?.Out, port++, Serializers.ImuFormat(), DeliveryPolicy.LatestMessage); + Write("Magnetometer", magnetometer?.Out, port++, Serializers.ImuFormat(), DeliveryPolicy.LatestMessage); + } + + if (IncludeHead) + { + Write("Head", head?.Out, port++, Serializers.CoordinateSystemFormat(), DeliveryPolicy.LatestMessage); + } + + if (IncludeEyes) + { + Write("Eyes", eyes?.Out, port++, Serializers.Ray3DFormat(), DeliveryPolicy.LatestMessage); + } + + if (IncludeHands) + { + Write("Hands", hands?.Out, port++, Serializers.HandsFormat(), DeliveryPolicy.LatestMessage); + } + + if (IncludeAudio) + { + Write("Audio", audio?.Out, port++, Serializers.AudioBufferFormat(), DeliveryPolicy.Unlimited); + } + + if (IncludeVideo) + { + Write("VideoEncodedImageCameraView", camera.VideoEncodedImageCameraView, port++, Serializers.EncodedImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + } + + if (IncludePreview) + { + Write("PreviewEncodedImageCameraView", camera.PreviewEncodedImageCameraView, port++, Serializers.EncodedImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + } + + if (IncludeDepth) + { + var depthImageCameraView = depthCamera.DepthImageCameraView; + Write("DepthImageCameraView", depthImageCameraView, port++, Serializers.DepthImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + + if (IncludeDepthCalibrationMap) + { + Write("DepthCalibrationMap", depthCamera.CalibrationPointsMap, port++, Serializers.CalibrationPointsMapFormat(), DeliveryPolicy.Unlimited); + } + + if (IncludeInfrared) + { + if (EncodeInfrared) + { + var infraredEncodedImageCameraView = depthCamera.InfraredImageCameraView + .Encode(new ImageToJpegStreamEncoder(InfraredImageJpegQuality), DeliveryPolicy.LatestMessage); + Write("InfraredEncodedImageCameraView", infraredEncodedImageCameraView, port++, Serializers.EncodedImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + } + else + { + var infraredImageCameraView = depthCamera.InfraredImageCameraView; + Write("InfraredImageCameraView", infraredImageCameraView, port++, Serializers.ImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + } + } +#if DEBUG + Write("DepthDebugOutOfOrderFrames", depthCamera?.DebugOutOfOrderFrames, port++, Serializers.Int32Format(), DeliveryPolicy.Unlimited); +#endif + } + + if (IncludeAhat) + { + var ahatDepthImageCameraView = depthAhatCamera.DepthImageCameraView; + + Write("AhatDepthImageCameraView", ahatDepthImageCameraView, port++, Serializers.DepthImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + + if (IncludeAhatCalibrationMap) + { + Write("AhatDepthCalibrationMap", depthAhatCamera?.CalibrationPointsMap, port++, Serializers.CalibrationPointsMapFormat(), DeliveryPolicy.Unlimited); + } +#if DEBUG + Write("AhatDebugOutOfOrderFrames", depthAhatCamera?.DebugOutOfOrderFrames, port++, Serializers.Int32Format(), DeliveryPolicy.Unlimited); +#endif + } + + if (IncludeGrayFrontCameras) + { + switch (EncodeGrayMethod) + { + case GrayImageEncode.Jpeg: + var leftFrontJpegEncodedImageCameraView = leftFrontCamera?.ImageCameraView + .Convert(PixelFormat.BGRA_32bpp, DeliveryPolicy.LatestMessage) + .Encode(new ImageToJpegStreamEncoder(GrayImageJpegQuality), DeliveryPolicy.SynchronousOrThrottle); + Write("LeftFrontEncodedImageCameraView", leftFrontJpegEncodedImageCameraView, port++, Serializers.EncodedImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + + var rightFrontJpegEncodedImageCameraView = rightFrontCamera?.ImageCameraView + .Convert(PixelFormat.BGRA_32bpp, DeliveryPolicy.LatestMessage) + .Encode(new ImageToJpegStreamEncoder(GrayImageJpegQuality), DeliveryPolicy.SynchronousOrThrottle); + Write("RightFrontEncodedImageCameraView", rightFrontJpegEncodedImageCameraView, port++, Serializers.EncodedImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + + break; + + case GrayImageEncode.Gzip: + var leftFrontGZipEncodedImageCameraView = leftFrontCamera?.ImageCameraView + .Convert(PixelFormat.BGRA_32bpp, DeliveryPolicy.LatestMessage) + .Encode(new ImageToGZipStreamEncoder(), DeliveryPolicy.SynchronousOrThrottle); + Write("LeftFrontGzipImageCameraView", leftFrontGZipEncodedImageCameraView, port++, Serializers.EncodedImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + + var rightFrontGZipEncodedImageCameraView = rightFrontCamera?.ImageCameraView + .Convert(PixelFormat.BGRA_32bpp, DeliveryPolicy.LatestMessage) + .Encode(new ImageToGZipStreamEncoder(), DeliveryPolicy.SynchronousOrThrottle); + Write("RightFrontGzipImageCameraView", rightFrontGZipEncodedImageCameraView, port++, Serializers.EncodedImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + + break; + + case GrayImageEncode.None: + var leftFrontImageView = leftFrontCamera.ImageCameraView; + Write("LeftFrontImageView", leftFrontImageView, port++, Serializers.EncodedImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + + var rightFrontImageView = rightFrontCamera.ImageCameraView; + Write("RightFrontImageView", rightFrontImageView, port++, Serializers.EncodedImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + + break; + } + + if (IncludeGrayFrontCameraCalibrationMap) + { + Write("LeftFrontCalibrationMap", leftFrontCamera?.CalibrationPointsMap, port++, Serializers.CalibrationPointsMapFormat(), DeliveryPolicy.Unlimited); + Write("RightFrontCalibrationMap", rightFrontCamera?.CalibrationPointsMap, port++, Serializers.CalibrationPointsMapFormat(), DeliveryPolicy.Unlimited); + } + } + + if (IncludeGraySideCameras) + { + switch (EncodeGrayMethod) + { + case GrayImageEncode.Jpeg: + var leftLeftJpegEncodedImageCameraView = leftLeftCamera?.ImageCameraView + .Convert(PixelFormat.BGRA_32bpp, DeliveryPolicy.LatestMessage) + .Encode(new ImageToJpegStreamEncoder(GrayImageJpegQuality), DeliveryPolicy.SynchronousOrThrottle); + Write("LeftLeftEncodedImageCameraView", leftLeftJpegEncodedImageCameraView, port++, Serializers.EncodedImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + + var rightRightJpegEncodedImageCameraView = rightRightCamera?.ImageCameraView + .Convert(PixelFormat.BGRA_32bpp, DeliveryPolicy.LatestMessage) + .Encode(new ImageToJpegStreamEncoder(GrayImageJpegQuality), DeliveryPolicy.SynchronousOrThrottle); + Write("RightRightEncodedImageCameraView", rightRightJpegEncodedImageCameraView, port++, Serializers.EncodedImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + + break; + + case GrayImageEncode.Gzip: + var leftLeftGZipEncodedImageCameraView = leftLeftCamera?.ImageCameraView + .Convert(PixelFormat.BGRA_32bpp, DeliveryPolicy.LatestMessage) + .Encode(new ImageToGZipStreamEncoder(), DeliveryPolicy.SynchronousOrThrottle); + Write("LeftLeftGzipImageCameraView", leftLeftGZipEncodedImageCameraView, port++, Serializers.EncodedImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + + var rightRightGZipEncodedImageCameraView = rightRightCamera?.ImageCameraView + .Convert(PixelFormat.BGRA_32bpp, DeliveryPolicy.LatestMessage) + .Encode(new ImageToGZipStreamEncoder(), DeliveryPolicy.SynchronousOrThrottle); + Write("RightRightGzipImageCameraView", rightRightGZipEncodedImageCameraView, port++, Serializers.EncodedImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + + break; + + case GrayImageEncode.None: + var leftLeftImageView = leftLeftCamera.ImageCameraView; + Write("LeftLeftImageView", leftLeftImageView, port++, Serializers.EncodedImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + + var rightRightImageView = rightRightCamera.ImageCameraView; + Write("RightRightImageView", rightRightImageView, port++, Serializers.EncodedImageCameraViewFormat(), DeliveryPolicy.LatestMessage); + + break; + } + + if (IncludeGraySideCameraCalibrationMap) + { + Write("LeftLeftCalibrationMap", leftLeftCamera?.CalibrationPointsMap, port++, Serializers.CalibrationPointsMapFormat(), DeliveryPolicy.Unlimited); + Write("RightRightCalibrationMap", rightRightCamera?.CalibrationPointsMap, port++, Serializers.CalibrationPointsMapFormat(), DeliveryPolicy.Unlimited); + } + } + + if (IncludeSceneUnderstanding) + { + Write("SceneUnderstanding", scene?.Out, port++, Serializers.SceneObjectCollectionFormat(), DeliveryPolicy.LatestMessage); + } + + if (IncludeDiagnostics) + { + Write("HoloLensDiagnostics", pipeline.Diagnostics, port++, Serializers.PipelineDiagnosticsFormat(), DeliveryPolicy.LatestMessage); + } + + rendezvousClient.Rendezvous.ProcessAdded += (_, process) => + { + if (process.Name == "HoloLensCaptureServer") + { + if (process.Version != Version) + { + throw new Exception($"Connection received from unexpected version of HoloLensCaptureServer (expected {Version}, actual {process.Version})."); + } + + captureServerProcess = process; + } + }; + + rendezvousClient.Rendezvous.ProcessRemoved += (_, process) => + { + if (process.Name == "HoloLensCaptureServer") + { + Trace.WriteLine($"Server shutdown"); + ResetState(); + } + }; + + rendezvousClient.Rendezvous.TryAddProcess(process); + } + }); + break; + case State.WaitingForCaptureServer: + UI.HandleBegin("Handle", ref windowPose, windowBounds, true); + if (errorMessage != null) + { + UI.Label($"Error: {errorMessage}"); + if (UI.Button("Exit")) + { + state = State.Exited; + Task.Run(() => SK.Shutdown()); + } + } + else + { + UI.Label($"Please wait! Connecting to capture server: {captureServerAddress}"); + if (UI.Button($"Stop")) + { + state = State.StoppingPipeline; + Task.Run(() => + { + pipeline.Dispose(); + pipeline = null; + }); + } + } + + UI.HandleEnd(); + DrawFrame(Color.Yellow.ToStereoKitColor()); + + if (captureServerProcess != default) + { + // Get the server heartbeat stream from the rendezvous server + foreach (var endpoint in captureServerProcess.Endpoints) + { + if (endpoint is Rendezvous.TcpSourceEndpoint tcpEndpoint) + { + foreach (var stream in tcpEndpoint.Streams) + { + if (stream.StreamName == $"ServerHeartbeat") + { + // note: using captureServerAddress -- ignoring tcpEndpoint.Host (0.0.0.0) + var serverHeartbeat = new TcpSource<(float, float)>(pipeline, captureServerAddress, tcpEndpoint.Port, Serializers.HeartbeatFormat()); + serverHeartbeat.Do(fps => + { + (videoFps, depthFps) = fps; + lastServerHeartBeat = DateTime.UtcNow; + }); + + // Run the pipeline + pipeline.PipelineExceptionNotHandled += (_, e) => errorMessage = e.Exception.Message; + pipeline.RunAsync(); + state = State.Capturing; + } + } + } + else + { + throw new Exception("Unexpected endpoint type."); + } + } + } + + break; + case State.Capturing: + UI.EnableFarInteract = false; + var frameColor = Color.Yellow; + var labelText = string.Empty; + UI.HandleBegin("Handle", ref windowPose, windowBounds, true); + if (errorMessage != null) + { + labelText = $"Error: {errorMessage}"; + UI.Label(labelText); + } + + if (lastServerHeartBeat == DateTime.MaxValue) + { + UI.Label("Please wait! Looking for server heartbeat ..."); + } + else if (DateTime.UtcNow - lastServerHeartBeat > serverHeartBeatTimeout) + { + labelText = "Server connection lost"; + UI.Label($"{labelText} ({lastServerHeartBeat.ToLocalTime()})"); + videoFps = 0; + depthFps = 0; + } + else if (errorMessage == null) + { + UI.Label("Capturing ..."); + labelText = $"FPS - Video:{videoFps:0.#} Depth:{depthFps:0.#}"; + frameColor = Color.DarkGray; + } + + if (UI.Button($"Stop")) + { + state = State.StoppingPipeline; + Task.Run(() => + { + pipeline.Dispose(); + pipeline = null; + }); + } + + UI.HandleEnd(); + DrawFrame(frameColor.ToStereoKitColor(), labelText); + + break; + case State.StoppingPipeline: + UI.HandleBegin("Handle", ref windowPose, windowBounds, true); + if (errorMessage != null) + { + UI.Label($"Error: {errorMessage}"); + } + + UI.Label("Please wait! Capture shutting down ..."); + UI.HandleEnd(); + DrawFrame(Color.Yellow.ToStereoKitColor()); + break; + } + })) + { + } + + pipeline?.Dispose(); + + SK.Shutdown(); + } + + private static void DrawFrame(Color32 color, string labelText = null) + { + // position relative to head pose + var head = Input.Head.ToCoordinateSystem(); + var rect = head.Transform(FrameRectangle); + + // draw border lines + var p0 = rect.TopLeft.ToVec3(); + var p1 = rect.TopRight.ToVec3(); + var p2 = rect.BottomRight.ToVec3(); + var p3 = rect.BottomLeft.ToVec3(); + Lines.Add(p0, p1, color, FrameThickness); + Lines.Add(p1, p2, color, FrameThickness); + Lines.Add(p2, p3, color, FrameThickness); + Lines.Add(p3, p0, color, FrameThickness); + + // render label + var widthAxis = (rect.BottomRight - rect.BottomLeft).Normalize(); + var heightAxis = (rect.TopLeft - rect.BottomLeft).Normalize(); + var normalAxis = widthAxis.CrossProduct(heightAxis); + var psiCoordinateSystem = new CoordinateSystem(rect.GetCenter(), normalAxis, widthAxis, heightAxis); + var labelPose = psiCoordinateSystem.ToStereoKitMatrix(); + Text.Add(labelText, labelPose, LabelSize, TextFit.Squeeze, offY: -(FrameHeight / 2 - FrameLabelInset)); + } + } +} diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/HoloLensCaptureApp.csproj b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/HoloLensCaptureApp.csproj new file mode 100644 index 000000000..2bae5b334 --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/HoloLensCaptureApp.csproj @@ -0,0 +1,177 @@ + + + + + Debug + ARM + {D318834B-5A27-4EAC-B17D-A9BD7A2DCA0D} + AppContainerExe + Properties + HoloLensCaptureApp + HoloLensCaptureApp + en-US + UAP + 10.0.19041.0 + 10.0.17763.0 + 16 + 512 + {A5A43C5B-DE2A-4C0C-9213-0A381AF9435A};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + true + false + Language=en + ..\..\..\..\Build\Microsoft.Psi.ruleset + true + + + ARM + true + bin\ARM\Debug\ + DEBUG;TRACE;NETFX_CORE;WINDOWS_UWP + ;2008 + full + false + prompt + true + bin\ARM\Debug\HoloLensCaptureApp.XML + ..\..\..\..\Build\Microsoft.Psi.ruleset + false + true + + + ARM + bin\ARM\Release\ + TRACE;NETFX_CORE;WINDOWS_UWP + true + ;2008 + pdbonly + false + prompt + true + bin\ARM\Release\HoloLensCaptureApp.XML + ..\..\..\..\Build\Microsoft.Psi.ruleset + false + true + + + PackageReference + + + + + + + + Designer + + + + + + + + + + + + + + + PreserveNewest + Assets\%(RecursiveDir)%(Filename)%(Extension) + + + + + 0.6.0 + + + + 1.1.118 + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + {ac5745da-570c-4e57-9ae4-d1974f629428} + Microsoft.Psi.Audio + + + {84ce1fe5-8141-4c2a-ac30-21bdc87f5d0a} + Microsoft.Psi.Calibration + + + {9bf2e5ef-186a-4179-b753-ae11ee90e026} + Microsoft.Psi.Imaging + + + {f50194c0-9561-40c7-b9cb-b977e3b3d76d} + HoloLens2ResearchMode + + + {ecd9e150-8104-4da3-b807-a6a4392a67c6} + Microsoft.Psi.MixedReality.UniversalWindows + + + {d558130a-6086-4bb0-9cdf-084e48c3de2b} + Microsoft.Psi.MixedReality + + + {d6be6801-7a6e-4c33-a681-e6e7306106a3} + Microsoft.Psi.Interop + + + {04147400-0ab0-4f07-9975-d4b7e58150db} + Microsoft.Psi + + + {05f10501-fc07-4f5b-a73d-98290326870d} + Microsoft.Psi.Spatial.Euclidean + + + {a3c3ccb5-ae71-4522-ad34-bc3e7b5ce72e} + HoloLensCaptureInterop + + + + + + + + 16.0 + + + true + bin\ARM\Debug\ + DEBUG;TRACE;NETFX_CORE;WINDOWS_UWP;CODE_ANALYSIS + ;2008 + true + full + ARM + false + latest + prompt + true + + + bin\ARM\Release\ + TRACE;NETFX_CORE;WINDOWS_UWP;CODE_ANALYSIS + true + ;2008 + true + pdbonly + ARM + false + latest + prompt + true + + + + \ No newline at end of file diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Package.appxmanifest b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Package.appxmanifest new file mode 100644 index 000000000..3779d06d2 --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Package.appxmanifest @@ -0,0 +1,67 @@ + + + + + + + + + + HoloLensCaptureApp + Microsoft Corporation + Assets\Logo\StoreLogo.png + + + + + + + + + + + + + + + + + + + + HoloLensCaptureApp Configuration + + .txt + + + + + + + + + + + + + + + + + + diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Properties/AssemblyInfo.cs b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..196994143 --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Properties/AssemblyInfo.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("HoloLensCaptureApp")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("HoloLensCaptureApp")] +[assembly: AssemblyCopyright("© Microsoft Corporation. All rights reserved.")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("0.17.52.1")] +[assembly: AssemblyFileVersion("0.17.52.1")] +[assembly: ComVisible(false)] diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Properties/Default.rd.xml b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Properties/Default.rd.xml new file mode 100644 index 000000000..16232c238 --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Properties/Default.rd.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Readme.md b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Readme.md new file mode 100644 index 000000000..240290ace --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/Readme.md @@ -0,0 +1,28 @@ +# HoloLens Capture App + +This app runs on the HoloLens, capturing data streams and remoting to the companion [HoloLensCaptureServer](../HoloLensCaptureServer) running on another machine. + +Note: The app uses the [Rendezvous System](https://github.com/microsoft/psi/wiki/Rendezvous-System) to connect to the capture server via TCP sockets, and all communication happens in the clear. These communication channels are not secure, and the user must ensure the security of the network as appropriate. + +## Configuration + +The IP address of the capture server must be known. A default (`captureServerAddress`) is given in code. This is overridden when a `CaptureServerIP.txt` file is present in `User Folders\Documents`. A new such file is also created upon first launch. This may be edited by hand and uploaded via the Device Portal. + +## Packaging + +The app may be deployed via Visual Studio or a previously built package may be loaded. The following instructions describe how to create an app package for sideloading onto the HoloLens via the _Device Portal_. + +1) Create a self-signed certificate: + + ```powershell + New-SelfSignedCertificate -Type Custom -Subject "CN=Microsoft Corporation, O=Microsoft Corporation, C=US" -KeyUsage DigitalSignature -FriendlyName "HoloLensCaptureApp Certificate" -CertStoreLocation "Cert:\CurrentUser\My" -TextExtension @("2.5.29.37={text}1.3.6.1.5.5.7.3.3", "2.5.29.19={text}") + ``` + +2) Add this certificate to the `Package.appxmanifest` by opening the manifest in Visual Studio, selecting the _Packaging_ tab and _Choose Certificate_, _Select From Store_. If the _SimgleCaptureApp Certificate_ is not listed, click _More Choices..._. Select the _HoloLensCaptureApp Certificate_ created above. + +3) Create the package by right clicking the _HoloLensCaptureApp_ project, choose _Publish_ > _Create App Package..._ Select _Sideloading_ (the default) and uncheck _Enable Automatic Updates_. _Next_, _Next_, _Create_. + +This will create the package in `\Internal\Applications\HoloLensCapture\HoloLensCaptureApp\AppPackages`. + +In the _Device Portal_, navigate to _Apps Manager_ > _Deploy Apps_ section, select _Local Storage_ > +_Select the application package_, _Choose File_ and browse to the app package and select _Install_. diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/stylecop.json b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/stylecop.json new file mode 100644 index 000000000..6f09427eb --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureApp/stylecop.json @@ -0,0 +1,16 @@ +{ + // ACTION REQUIRED: This file was automatically added to your project, but it + // will not take effect until additional steps are taken to enable it. See the + // following page for additional information: + // + // https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/EnableConfiguration.md + + "$schema": "https://raw.githubusercontent.com/DotNetAnalyzers/StyleCopAnalyzers/master/StyleCop.Analyzers/StyleCop.Analyzers/Settings/stylecop.schema.json", + "settings": { + "documentationRules": { + "companyName": "Microsoft Corporation", + "copyrightText": "Copyright (c) Microsoft Corporation. All rights reserved.\nLicensed under the MIT license.", + "xmlHeader": false + } + } +} \ No newline at end of file diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/DataExporter.cs b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/DataExporter.cs new file mode 100644 index 000000000..6892a9654 --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/DataExporter.cs @@ -0,0 +1,198 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace HoloLensCaptureExporter +{ + using System; + using System.Collections.Generic; + using System.IO; + using MathNet.Spatial.Euclidean; + using Microsoft.Psi; + using Microsoft.Psi.Audio; + using Microsoft.Psi.Imaging; + using Microsoft.Psi.MixedReality; + using Microsoft.Psi.Spatial.Euclidean; + + ///

+ /// Implements the data exporter. + /// + internal class DataExporter + { + /// + /// Exports data based on the specified export command. + /// + /// The export command. + /// An error code or 0 if success. + public static int Run(Verbs.ExportCommand exportCommand) + { + // Create a pipeline + using var p = Pipeline.Create(deliveryPolicy: DeliveryPolicy.SynchronousOrThrottle); + + // Open the psi store for reading + var store = PsiStore.Open(p, exportCommand.StoreName, exportCommand.StorePath); + + // Get references to the various streams. If a stream is not present in the store, + // the reference will be null. + var accelerometer = store.OpenStreamOrDefault<(Vector3D, DateTime)[]>("Accelerometer"); + var gyroscope = store.OpenStreamOrDefault<(Vector3D, DateTime)[]>("Gyroscope"); + var magnetometer = store.OpenStreamOrDefault<(Vector3D, DateTime)[]>("Magnetometer"); + var head = store.OpenStreamOrDefault("Head"); + var eyes = store.OpenStreamOrDefault("Eyes"); + var hands = store.OpenStreamOrDefault<(Hand Left, Hand Right)>("Hands"); + var audio = store.OpenStreamOrDefault("Audio"); + var videoEncodedImageCameraView = store.OpenStreamOrDefault("VideoEncodedImageCameraView"); + var videoImageCameraView = store.OpenStreamOrDefault("VideoImageCameraView"); + var previewEncodedImageCameraView = store.OpenStreamOrDefault("PreviewEncodedImageCameraView"); + var previewImageCameraView = store.OpenStreamOrDefault("PreviewImageCameraView"); + var depthImageCameraView = store.OpenStreamOrDefault("DepthImageCameraView"); + var depthCalibrationMap = store.OpenStreamOrDefault("DepthCalibrationMap"); + var infraredEncodedImageCameraView = store.OpenStreamOrDefault("InfraredEncodedImageCameraView"); + var infraredImageCameraView = store.OpenStreamOrDefault("InfraredImageCameraView"); + var ahatDepthImageCameraView = store.OpenStreamOrDefault("AhatDepthImageCameraView"); + var ahatDepthCalibrationMap = store.OpenStreamOrDefault("AhatDepthCalibrationMap"); + var leftFrontEncodedImageCameraView = store.OpenStreamOrDefault("LeftFrontEncodedImageCameraView"); + var leftFrontGzipImageCameraView = store.OpenStreamOrDefault("LeftFrontGzipImageCameraView"); + var leftFrontImageCameraView = store.OpenStreamOrDefault("LeftFrontImageCameraView"); + var leftFrontCalibrationMap = store.OpenStreamOrDefault("LeftFrontCalibrationMap"); + var rightFrontEncodedImageCameraView = store.OpenStreamOrDefault("RightFrontEncodedImageCameraView"); + var rightFrontGzipImageCameraView = store.OpenStreamOrDefault("RightFrontGzipImageCameraView"); + var rightFrontImageCameraView = store.OpenStreamOrDefault("RightFrontImageCameraView"); + var rightFrontCalibrationMap = store.OpenStreamOrDefault("RightFrontCalibrationMap"); + var leftLeftEncodedImageCameraView = store.OpenStreamOrDefault("LeftLeftEncodedImageCameraView"); + var leftLeftGzipImageCameraView = store.OpenStreamOrDefault("LeftLeftGzipImageCameraView"); + var leftLeftImageCameraView = store.OpenStreamOrDefault("LeftLeftImageCameraView"); + var leftLeftCalibrationMap = store.OpenStreamOrDefault("LeftLeftCalibrationMap"); + var rightRightEncodedImageCameraView = store.OpenStreamOrDefault("RightRightEncodedImageCameraView"); + var rightRightGzipImageCameraView = store.OpenStreamOrDefault("RightRightGzipImageCameraView"); + var rightRightImageCameraView = store.OpenStreamOrDefault("RightRightImageCameraView"); + var rightRightCalibrationMap = store.OpenStreamOrDefault("RightRightCalibrationMap"); + var sceneUnderstanding = store.OpenStreamOrDefault("SceneUnderstanding"); + + // Verify expected stream combinations + void VerifyMutualExclusivity(dynamic a, dynamic b, string name) + { + if (a != null && b != null) + { + throw new Exception($"Found both encoded and unencoded {name} streams (expected one or the other)."); + } + } + + VerifyMutualExclusivity(videoEncodedImageCameraView, videoImageCameraView, "video"); + VerifyMutualExclusivity(previewEncodedImageCameraView, previewImageCameraView, "preview"); + VerifyMutualExclusivity(infraredEncodedImageCameraView, infraredImageCameraView, "infrared"); + VerifyMutualExclusivity(leftFrontEncodedImageCameraView, leftFrontImageCameraView, "left-front"); + VerifyMutualExclusivity(rightFrontEncodedImageCameraView, rightFrontImageCameraView, "right-front"); + VerifyMutualExclusivity(leftLeftEncodedImageCameraView, leftLeftImageCameraView, "left-left"); + VerifyMutualExclusivity(rightRightEncodedImageCameraView, rightRightImageCameraView, "right-right"); + + // Construct a list of stream writers to export data with (these will be closed once + // the export pipeline is completed) + var streamWritersToClose = new List(); + + // Export various encoded image camera views + var pngEncoder = new ImageToPngStreamEncoder(); + var gzipDecoder = new ImageFromStreamDecoder(); + + void Export( + string name, + IProducer imageCameraView, + IProducer encodedImageCameraView, + IProducer gzipImageCameraView = null) + { + void VerifyMutualExclusivity(dynamic s0, dynamic s1) + { + if (s0 != null || s1 != null) + { + throw new Exception($"Expected single stream for each camera (found multiple for {name})."); + } + } + + if (imageCameraView != null) + { + // export raw camera view as lossless PNG + VerifyMutualExclusivity(encodedImageCameraView, gzipImageCameraView); + imageCameraView?.Encode(pngEncoder).Export(name, exportCommand.OutputPath, streamWritersToClose); + } + + if (encodedImageCameraView != null) + { + // export encoded camera view as is + VerifyMutualExclusivity(imageCameraView, gzipImageCameraView); + encodedImageCameraView.Export(name, exportCommand.OutputPath, streamWritersToClose); + } + + if (gzipImageCameraView != null) + { + // export GZIP'd camera view as lossless PNG + VerifyMutualExclusivity(imageCameraView, encodedImageCameraView); + gzipImageCameraView?.Decode(gzipDecoder)?.Encode(pngEncoder).Export(name, exportCommand.OutputPath, streamWritersToClose); + } + } + + Export("Video", videoImageCameraView, videoEncodedImageCameraView); + Export("Preview", previewImageCameraView, previewEncodedImageCameraView); + Export("Infrared", infraredImageCameraView, infraredEncodedImageCameraView); + Export("LeftFront", leftFrontImageCameraView, leftFrontEncodedImageCameraView, leftFrontGzipImageCameraView); + Export("RightFront", rightFrontImageCameraView, rightFrontEncodedImageCameraView, rightFrontGzipImageCameraView); + Export("LeftLeft", leftLeftImageCameraView, leftLeftEncodedImageCameraView, leftLeftGzipImageCameraView); + Export("RightRight", rightRightImageCameraView, rightRightEncodedImageCameraView, rightRightGzipImageCameraView); + + // Export various depth image camera views + depthImageCameraView?.Export("Depth", exportCommand.OutputPath, streamWritersToClose); + ahatDepthImageCameraView?.Export("AhatDepth", exportCommand.OutputPath, streamWritersToClose); + + // Export various camera calibration maps + depthCalibrationMap?.Export("Depth", "CalibrationMap", exportCommand.OutputPath, streamWritersToClose); + ahatDepthCalibrationMap?.Export("AhatDepth", "CalibrationMap", exportCommand.OutputPath, streamWritersToClose); + leftFrontCalibrationMap?.Export("LeftFront", "CalibrationMap", exportCommand.OutputPath, streamWritersToClose); + rightFrontCalibrationMap?.Export("RightFront", "CalibrationMap", exportCommand.OutputPath, streamWritersToClose); + leftLeftCalibrationMap?.Export("LeftLeft", "CalibrationMap", exportCommand.OutputPath, streamWritersToClose); + rightRightCalibrationMap?.Export("RightRight", "CalibrationMap", exportCommand.OutputPath, streamWritersToClose); + + // Export IMU streams + accelerometer?.SelectManyImuSamples(DeliveryPolicy.SynchronousOrThrottle).Export("IMU", "Accelerometer", exportCommand.OutputPath, streamWritersToClose); + gyroscope?.SelectManyImuSamples(DeliveryPolicy.SynchronousOrThrottle).Export("IMU", "Gyroscope", exportCommand.OutputPath, streamWritersToClose); + magnetometer?.SelectManyImuSamples(DeliveryPolicy.SynchronousOrThrottle).Export("IMU", "Magnetometer", exportCommand.OutputPath, streamWritersToClose); + + // Export head, eyes and hands streams + head?.Export("Head", exportCommand.OutputPath, streamWritersToClose); + eyes?.Export("Eyes", exportCommand.OutputPath, streamWritersToClose); + hands?.Select(x => x.Left).Export("Hands", "Left", exportCommand.OutputPath, streamWritersToClose); + hands?.Select(x => x.Right).Export("Hands", "Right", exportCommand.OutputPath, streamWritersToClose); + + // Export audio + audio?.Export("Audio", exportCommand.OutputPath, streamWritersToClose); + + // Export scene understanding + sceneUnderstanding?.Export("SceneUnderstanding", exportCommand.OutputPath, streamWritersToClose); + + p.PipelineCompleted += (_, _) => Console.WriteLine("DONE."); + p.RunAsync(ReplayDescriptor.ReplayAllRealTime, progress: new Progress(p => Console.Write($"Progress: {p:P}\r"))); + p.WaitAll(); + + foreach (var sw in streamWritersToClose) + { + sw.Close(); + } + + Console.WriteLine("Done."); + return 0; + } + + /// + /// Ensures that a specified path exists. + /// + /// The path to ensure the existence of. + /// The path. + internal static string EnsurePathExists(string path) + { + var directory = Path.GetDirectoryName(path); + if (!Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + + return path; + } + } +} diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/HoloLensCaptureExporter.cs b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/HoloLensCaptureExporter.cs new file mode 100644 index 000000000..45e48f3cb --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/HoloLensCaptureExporter.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace HoloLensCaptureExporter +{ + using System; + using System.Collections.Generic; + using CommandLine; + + /// + /// Tool to export \psi store data persisted by HoloLensCaptureServer to other formats. + /// + internal class HoloLensCaptureExporter + { + /// + /// Main entry point. + /// + /// Command-line arguments. + /// Command-line status. + public static int Main(string[] args) + { + Console.WriteLine($"HoloLensCaptureExporter Tool"); + try + { + return Parser.Default.ParseArguments(args) + .MapResult( + (Verbs.ExportCommand command) => DataExporter.Run(command), + DisplayParseErrors); + } + catch (Exception ex) + { + Console.WriteLine($"Error: {ex.Message}"); + return -1; + } + } + + /// + /// Display command-line parser errors. + /// + /// Errors reported. + /// Success flag. + private static int DisplayParseErrors(IEnumerable errors) + { + Console.WriteLine("Errors:"); + var ret = 0; + foreach (var error in errors) + { + Console.WriteLine($"{error}"); + if (error.StopsProcessing) + { + ret = 1; + } + } + + return ret; + } + } +} diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/HoloLensCaptureExporter.csproj b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/HoloLensCaptureExporter.csproj new file mode 100644 index 000000000..81ea0e682 --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/HoloLensCaptureExporter.csproj @@ -0,0 +1,44 @@ + + + + Exe + net472 + ..\..\..\..\Build\Microsoft.Psi.ruleset + true + + + + True + + + + True + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers + + + all + runtime; build; native; contentfiles; analyzers + + + + + + + + + + + + + diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/Operators.cs b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/Operators.cs new file mode 100644 index 000000000..6a08b3383 --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/Operators.cs @@ -0,0 +1,515 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace HoloLensCaptureExporter +{ + using System; + using System.Collections.Generic; + using System.Globalization; + using System.IO; + using System.Linq; + using System.Text; + using MathNet.Numerics.LinearAlgebra; + using MathNet.Spatial.Euclidean; + using Microsoft.Psi; + using Microsoft.Psi.Audio; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Data; + using Microsoft.Psi.Imaging; + using Microsoft.Psi.MixedReality; + using Microsoft.Psi.Spatial.Euclidean; + + /// + /// Stream operators and extension methods for exporting data. + /// + internal static class Operators + { + /// + /// Opens the specified stream for reading and (or returns null if nonexistent). + /// + /// The expected type of the stream to open. + /// Store containing stream. + /// The name of the stream to open. + /// Stream instance that can be used to consume the messages (or null if nonexistent). + internal static IProducer OpenStreamOrDefault(this PsiImporter store, string name) + => store.Contains(name) ? store.OpenStream(name) : null; + + /// + /// Convert to string. + /// + /// to be converted. + /// Text representation. + internal static string ToText(this double d) + { + var text = d.ToString("G17", CultureInfo.InvariantCulture); // https://docs.microsoft.com/en-us/dotnet/standard/base-types/standard-numeric-format-strings#RFormatString + if (!double.IsNaN(d) && double.Parse(text) != d) + { + throw new Exception("Text representation of double did not survive round-trip parsing."); + } + + return text; + } + + /// + /// Convert to string. + /// + /// to be converted. + /// Text representation. + internal static string ToText(this int i) + { + return i.ToString(CultureInfo.InvariantCulture); + } + + /// + /// Convert to string. + /// + /// to be converted. + /// Text representation. + internal static string ToText(this DateTime dt) + { + return dt.Ticks.ToString(CultureInfo.InvariantCulture); + } + + /// + /// Convert to string. + /// + /// to be converted. + /// Text representation. + internal static string ToText(this bool b) + { + return b ? "1" : "0"; + } + + /// + /// Convert to tab-delimited text representation. + /// + /// to be converted. + /// Tab-delimited text representation. + internal static string ToText(this CoordinateSystem c) + => $"{c[0, 0].ToText()}\t{c[0, 1].ToText()}\t{c[0, 2].ToText()}\t{c[0, 3].ToText()}\t" + + $"{c[1, 0].ToText()}\t{c[1, 1].ToText()}\t{c[1, 2].ToText()}\t{c[1, 3].ToText()}\t" + + $"{c[2, 0].ToText()}\t{c[2, 1].ToText()}\t{c[2, 2].ToText()}\t{c[2, 3].ToText()}\t" + + $"{c[3, 0].ToText()}\t{c[3, 1].ToText()}\t{c[3, 2].ToText()}\t{c[3, 3].ToText()}"; + + /// + /// Converts a camera intrinsics to a tab-delimited text representation. + /// + /// The camera intrinsics. + /// Tab-delimited text representation. + internal static string ToText(this ICameraIntrinsics cameraIntrinsics) + => $"{cameraIntrinsics.Transform.ToText()}\t" + + $"{cameraIntrinsics.RadialDistortion.ToText()}\t" + + $"{cameraIntrinsics.TangentialDistortion.ToText()}\t" + + $"{cameraIntrinsics.FocalLength.ToText()}\t" + + $"{cameraIntrinsics.FocalLengthXY.ToText()}\t" + + $"{cameraIntrinsics.PrincipalPoint.ToText()}\t" + + $"{cameraIntrinsics.ClosedFormDistorts.ToText()}\t" + + $"{cameraIntrinsics.ImageWidth.ToText()}\t" + + $"{cameraIntrinsics.ImageHeight.ToText()}"; + + /// + /// Converts a matrix to a tab-delimited text representation. + /// + /// The matrix. + /// Tab-delimited text representation. + internal static string ToText(this Matrix matrix) + { + var result = new StringBuilder(); + for (int i = 0; i < matrix.RowCount; i++) + { + for (int j = 0; j < matrix.ColumnCount; j++) + { + result.Append($"{matrix[i, j].ToText()}\t"); + } + } + + return result.ToString().TrimEnd('\t'); + } + + /// + /// Converts a vector to a tab-delimited text representation. + /// + /// The vector . + /// Tab-delimited text representation. + internal static string ToText(this Vector vector) + { + var result = new StringBuilder(); + for (int i = 0; i < vector.Count; i++) + { + result.Append($"{vector[i].ToText()}\t"); + } + + return result.ToString().TrimEnd('\t'); + } + + /// + /// Converts a 2D point to a tab-delimited text representation. + /// + /// The 2D point. + /// Tab-delimited text representation. + internal static string ToText(this Point2D point2D) => + $"{point2D.X.ToText()}\t{point2D.Y.ToText()}"; + + /// + /// Converts a 3D point to a tab-delimited text representation. + /// + /// The 3D point. + /// Tab-delimited text representation. + internal static string ToText(this Point3D point3D) => + $"{point3D.X.ToText()}\t{point3D.Y.ToText()}\t{point3D.Z.ToText()}"; + + /// + /// Converts a 3D vector to a tab-delimited text representation. + /// + /// The 3D vector. + /// Tab-delimited text representation. + internal static string ToText(this Vector3D vector3D) => + $"{vector3D.X.ToText()}\t{vector3D.Y.ToText()}\t{vector3D.Z.ToText()}"; + + /// + /// Converts a 3D ray to a tab-delimited text representation. + /// + /// The 3D ray. + /// Tab-delimited text representation. + internal static string ToText(this Ray3D ray3D) => + $"{ray3D.ThroughPoint.ToText()}\t{ray3D.Direction.ToVector3D().ToText()}"; + + /// + /// Exports a stream of encoded image camera views. + /// + /// The source stream of encoded image camera views. + /// The name for the source stream. + /// The output path. + /// The collection of stream writers to be closed. + internal static void Export(this IProducer source, string name, string outputPath, List streamWritersToClose) + { + var timingFilePath = DataExporter.EnsurePathExists(Path.Combine(outputPath, name, $"Timing.txt")); + var timingFile = File.CreateText(timingFilePath); + streamWritersToClose.Add(timingFile); + + var poseFilePath = DataExporter.EnsurePathExists(Path.Combine(outputPath, name, $"Pose.txt")); + var poseFile = File.CreateText(poseFilePath); + streamWritersToClose.Add(poseFile); + + var intrinsicsFilePath = DataExporter.EnsurePathExists(Path.Combine(outputPath, name, $"Intrinsics.txt")); + var intrinsicsFile = File.CreateText(intrinsicsFilePath); + streamWritersToClose.Add(intrinsicsFile); + + var imageCounter = 0; + source.Do( + (eicv, envelope) => + { + var buffer = eicv.ViewedObject.Resource.GetBuffer(); + var isPng = buffer.Length >= 8 && + buffer[0] == 0x89 && // look for PNG header + buffer[1] == 0x50 && // see https://en.wikipedia.org/wiki/Portable_Network_Graphics#File_header + buffer[2] == 0x4e && // P + buffer[3] == 0x47 && // N + buffer[4] == 0x0d && // G + buffer[5] == 0x0a && + buffer[6] == 0x1a && + buffer[7] == 0x0a; + var extension = isPng ? "png" : "jpg"; + var videoImagesPath = DataExporter.EnsurePathExists(Path.Combine(outputPath, name, $"{imageCounter:000000}.{extension}")); + using var videoImageFile = File.Create(videoImagesPath); + videoImageFile.Write(buffer, 0, eicv.ViewedObject.Resource.Size); + timingFile.WriteLine($"{imageCounter}\t{envelope.OriginatingTime.ToText()}"); + poseFile.WriteLine($"{envelope.OriginatingTime.ToText()}\t{eicv.CameraPose.ToText()}"); + + if (imageCounter == 0) + { + intrinsicsFile.WriteLine(eicv.CameraIntrinsics.ToText()); + } + + imageCounter++; + }, + DeliveryPolicy.SynchronousOrThrottle); + } + + /// + /// Exports a stream of depth image camera views. + /// + /// The source stream of depth image camera views. + /// The name for the source stream. + /// The output path. + /// The collection of stream writers to be closed. + internal static void Export(this IProducer source, string name, string outputPath, List streamWritersToClose) + { + var timingFilePath = DataExporter.EnsurePathExists(Path.Combine(outputPath, name, $"Timing.txt")); + var timingFile = File.CreateText(timingFilePath); + streamWritersToClose.Add(timingFile); + + var poseFilePath = DataExporter.EnsurePathExists(Path.Combine(outputPath, name, $"Pose.txt")); + var poseFile = File.CreateText(poseFilePath); + streamWritersToClose.Add(poseFile); + + var intrinsicsFilePath = DataExporter.EnsurePathExists(Path.Combine(outputPath, name, $"Intrinsics.txt")); + var intrinsicsFile = File.CreateText(intrinsicsFilePath); + streamWritersToClose.Add(intrinsicsFile); + + var depthImageCounter = 0; + source + .Encode(new DepthImageToPngStreamEncoder(), DeliveryPolicy.SynchronousOrThrottle) + .Do( + (edicv, envelope) => + { + var buffer = edicv.ViewedObject.Resource.GetBuffer(); + var depthImagesPath = DataExporter.EnsurePathExists(Path.Combine(outputPath, name, $"{depthImageCounter:000000}.png")); + using var depthImageFile = File.Create(depthImagesPath); + depthImageFile.Write(buffer, 0, buffer.Length); + timingFile.WriteLine($"{depthImageCounter}\t{envelope.OriginatingTime.ToText()}"); + poseFile.WriteLine($"{envelope.OriginatingTime.ToText()}\t{edicv.CameraPose.ToText()}"); + + if (depthImageCounter == 0) + { + intrinsicsFile.WriteLine(edicv.CameraIntrinsics.ToText()); + } + + depthImageCounter++; + }, + DeliveryPolicy.SynchronousOrThrottle); + } + + /// + /// Exports a stream of IMU readings. + /// + /// The source stream of IMU readings. + /// The directory in which to persist. + /// The name for the source stream. + /// The output path. + /// The collection of stream writers to be closed. + internal static void Export(this IProducer source, string directory, string name, string outputPath, List streamWritersToClose) + { + var filePath = DataExporter.EnsurePathExists(Path.Combine(outputPath, directory, $"{name}.txt")); + var file = File.CreateText(filePath); + streamWritersToClose.Add(file); + source + .Do( + (vector, envelope) => + { + file.WriteLine($"{envelope.OriginatingTime.ToText()}\t{vector.ToText()}"); + }, + DeliveryPolicy.SynchronousOrThrottle); + } + + /// + /// Exports a stream of poses. + /// + /// The source stream of poses. + /// The name for the source stream. + /// The output path. + /// The collection of stream writers to be closed. + internal static void Export(this IProducer source, string name, string outputPath, List streamWritersToClose) + { + var filePath = DataExporter.EnsurePathExists(Path.Combine(outputPath, name, $"{name}.txt")); + var file = File.CreateText(filePath); + streamWritersToClose.Add(file); + source + .Do( + (coordinateSystem, envelope) => + { + file.WriteLine($"{envelope.OriginatingTime.ToText()}\t{coordinateSystem.ToText()}"); + }, + DeliveryPolicy.SynchronousOrThrottle); + } + + /// + /// Exports a stream of 3D rays. + /// + /// The source stream of 3D rays. + /// The name for the source stream. + /// The output path. + /// The collection of stream writers to be closed. + internal static void Export(this IProducer source, string name, string outputPath, List streamWritersToClose) + { + var filePath = DataExporter.EnsurePathExists(Path.Combine(outputPath, name, $"{name}.txt")); + var file = File.CreateText(filePath); + streamWritersToClose.Add(file); + source + .Do( + (ray3D, envelope) => + { + file.WriteLine($"{envelope.OriginatingTime.ToText()}\t{ray3D.ToText()}"); + }, + DeliveryPolicy.SynchronousOrThrottle); + } + + /// + /// Exports a stream of hand infomation. + /// + /// The source stream of hand information. + /// The directory in which to persist. + /// The name for the source stream. + /// The output path. + /// The collection of stream writers to be closed. + internal static void Export(this IProducer source, string directory, string name, string outputPath, List streamWritersToClose) + { + var filePath = DataExporter.EnsurePathExists(Path.Combine(outputPath, directory, $"{name}.txt")); + var file = File.CreateText(filePath); + streamWritersToClose.Add(file); + source + .Do( + (hand, envelope) => + { + var result = new StringBuilder(); + result.Append($"{envelope.OriginatingTime.ToText()}\t"); + result.Append($"{hand.IsGripped.ToText()}\t"); + result.Append($"{hand.IsPinched.ToText()}\t"); + result.Append($"{hand.IsTracked.ToText()}\t"); + if (hand.IsTracked) + { + foreach (var joint in hand.Joints) + { + result.Append($"{joint.ToText()}\t"); + } + } + + file.WriteLine(result.ToString().TrimEnd('\t')); + }, + DeliveryPolicy.SynchronousOrThrottle); + } + + /// + /// Exports a stream of audio. + /// + /// The source stream of audio. + /// The name for the source stream. + /// The output path. + /// The collection of stream writers to be closed. + internal static void Export(this IProducer source, string name, string outputPath, List streamWritersToClose) + { + source.PipeTo( + new WaveFileWriter( + source.Out.Pipeline, + DataExporter.EnsurePathExists(Path.Combine(outputPath, name, $"{name}.wav"))), + DeliveryPolicy.SynchronousOrThrottle); + } + + /// + /// Exports a stream of calibration maps. + /// + /// The source stream of calibration maps. + /// The directory in which to persist. + /// The name for the source stream. + /// The output path. + /// The collection of stream writers to be closed. + internal static void Export(this IProducer source, string directory, string name, string outputPath, List streamWritersToClose) + { + var filePath = DataExporter.EnsurePathExists(Path.Combine(outputPath, directory, $"{name}.txt")); + var file = File.CreateText(filePath); + streamWritersToClose.Add(file); + source + .Do( + (map, envelope) => + { + var result = new StringBuilder(); + result.Append($"{envelope.OriginatingTime.ToText()}\t"); + result.Append($"{map.Width.ToText()}\t"); + result.Append($"{map.Height.ToText()}\t"); + foreach (var point in map.CameraUnitPlanePoints) + { + result.Append($"{point.ToText()}\t"); + } + + file.WriteLine(result.ToString().TrimEnd('\t')); + }, + DeliveryPolicy.SynchronousOrThrottle); + } + + /// + /// Exports a stream of scene objects. + /// + /// The source stream of scene objects. + /// The name for the source stream. + /// The output path. + /// The collection of stream writers to be closed. + internal static void Export(this IProducer source, string name, string outputPath, List streamWritersToClose) + { + void ExportScene(IProducer sceneObject, string sceneName) + { + void BuildRectangle(Rectangle3D? rect, StringBuilder sb) + { + void BuildPoint(Point3D point, StringBuilder sb) + { + sb.Append($"{point.ToText()}\t"); + } + + if (rect.HasValue) + { + var r = rect.Value; + BuildPoint(r.TopLeft, sb); + BuildPoint(r.TopRight, sb); + BuildPoint(r.BottomLeft, sb); + BuildPoint(r.BottomRight, sb); + } + else + { + var nan = double.NaN.ToText(); + for (var i = 0; i < 8; i++) + { + sb.Append($"{nan}\t"); + } + } + } + + var path = Path.Combine(outputPath, name, sceneName); + + void ExportMeshes(List meshes, string directory, string name) + { + for (var i = 0; i < meshes.Count; i++) + { + // .obj file format: https://en.wikipedia.org/wiki/Wavefront_.obj_file + var mesh = meshes[i]; + var meshFile = File.CreateText(DataExporter.EnsurePathExists(Path.Combine(path, directory, name, $"Mesh{i}.obj"))); + meshFile.Write($"# {directory} {name}"); + foreach (var v in mesh.Vertices) + { + meshFile.Write($"\nv {v.X.ToText()} {v.Y.ToText()} {v.Z.ToText()}"); + } + + var indices = mesh.TriangleIndices; + for (var j = 0; j < indices.Length; j += 3) + { + meshFile.WriteLine($"\nf {indices[j] + 1} {indices[j + 1] + 1} {indices[j + 2] + 1}"); + } + + meshFile.Close(); + } + } + + var rectanglesFile = File.CreateText(DataExporter.EnsurePathExists(Path.Combine(path, $"Rectangles.txt"))); + var meshesFile = File.CreateText(DataExporter.EnsurePathExists(Path.Combine(path, $"Meshes.txt"))); + streamWritersToClose.Add(rectanglesFile); + streamWritersToClose.Add(meshesFile); + sceneObject + .Do( + (s, envelope) => + { + var originatingTime = $"{envelope.OriginatingTime.ToText()}"; + var result = new StringBuilder(); + result.Append(originatingTime).Append('\t'); + for (var i = 0; i < s.Rectangles.Count; i++) + { + BuildRectangle(s.Rectangles[i], result); + BuildRectangle(s.PlacementRectangles.Count > 0 ? s.PlacementRectangles[i] : null, result); + } + + rectanglesFile.WriteLine(result.ToString().TrimEnd('\t')); + meshesFile.WriteLine($"{originatingTime}\t{s.Meshes.Count}\t{s.ColliderMeshes.Count}"); + ExportMeshes(s.Meshes, nameof(SceneObjectCollection.SceneObject.Meshes), originatingTime); + ExportMeshes(s.ColliderMeshes, nameof(SceneObjectCollection.SceneObject.ColliderMeshes), originatingTime); + }, + DeliveryPolicy.SynchronousOrThrottle); + } + + ExportScene(source.Select(s => s.Background), nameof(SceneObjectCollection.Background)); + ExportScene(source.Select(s => s.Ceiling), nameof(SceneObjectCollection.Ceiling)); + ExportScene(source.Select(s => s.Floor), nameof(SceneObjectCollection.Floor)); + ExportScene(source.Select(s => s.Inferred), nameof(SceneObjectCollection.Inferred)); + ExportScene(source.Select(s => s.Platform), nameof(SceneObjectCollection.Platform)); + ExportScene(source.Select(s => s.Unknown), nameof(SceneObjectCollection.Unknown)); + ExportScene(source.Select(s => s.Wall), nameof(SceneObjectCollection.Wall)); + ExportScene(source.Select(s => s.World), nameof(SceneObjectCollection.World)); + } + } +} diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/Readme.md b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/Readme.md new file mode 100644 index 000000000..33158f154 --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/Readme.md @@ -0,0 +1,288 @@ +# HoloLens Capture Exporter + +The HoloLensCaptureExporter is a tool to convert data within \psi stores that have been collected by the [HoloLensCaptureServer](..\HoloLensCaptureServer) to other formats. Example usage: + +```bash +> HoloLensCaptureExporter -p C:\data\Temp\HoloLensCapture.0009 -o C:\data\Temp\HoloLensCapture.0009\Export -v +``` + +This will open the store specified by `-p`, find and convert streams within, and export the results to the output directory given by `-o`. In this example, the `-v` flag has been given to indicate that video streams should be exported as individual image frames. + +## Options + +The following options are available: + +| Option | Abbr | Description | +| ------ | -------- | --------------------------------------------------------------------- | +| `p` | `path` | Path to the input Psi data store. | +| `o` | `output` | Output path to export data to. | +| `n` | `name` | Optional name of the input Psi data store (default: HoloLensCapture). | +| `v` | `video` | Optional flag indicating whether to export video. | + +## Output Formats + +The following describes the output structure and formats that are exported by the tool. + +### Primitives + +#### Timestamps + +Timestamps represent the originating time of sensor readings. They are in Coordinated Universal Time (UTC), represented by a single 64-bit integer counting 100-nanosecond "ticks" since 00:00:00 (midnight), January 1, 0001 (C.E.) in the Gregorian calendar. + +For example, 603202464000000000 would be midnight on Alan Turing's birthday (23 JUN 1912). + +#### Poses + +Poses for cameras, head, and hand joints are represented by a coordinate system. These are persisted as tab-separated values from the underlying 4×4 matrix in row-major order. For example: + +```text +M₀₀ M₀₁ M₀₂ M₀₃ M₁₀ M₁₁ M₁₂ M₁₃ M₂₀ M₂₁ M₂₂ M₂₃ M₃₀ M₃₁ M₃₂ M₃₃ +------------------------------------------------------------------------------------------------------------------------------------------- +0.947... -0.010... 0.318... 0.081... 0.053... 0.990... -0.125... -0.003... -0.314... 0.135... 0.939... 0.042... 0 0 0 1 +``` + +#### Gaze + +Eye gaze is represented by a 3D ray, and is persisted as the tab-separated x, y, and z values of the origin point (P) and then the direction vector (V). For example: + +```text +P[x] P[y] P[z] V[x] V[y] V[z] +----------------------------------------------------------- +0.547... -0.710... 0.418... 0.267... 0.159... 0.990... +``` + +### Sensor Streams + +The following are the HoloLens 2 sensor streams that are exported from the original \psi store. + +#### IMU + +##### Accelerometer + +Accelerometer data is persisted to a text file (`Accelerometer/Accelerometer.txt`) containing newline delimited records, each containing the originating timestamp (see above), and the X-, Y-, and Z-axis inertial force in m/s² as tab-separated fields. For example: + +```text +Originating Time X-axis Y-axis Z-axis +---------------------------------------------------------------------------------- +637835894408235676 -0.76258039474487305 -9.7197809219360352 -1.3236149549484253 +637835894408244692 -0.7307015061378479 -9.7032041549682617 -1.2934621572494507 +637835894408253708 -0.77219980955123901 -9.7326211929321289 -1.3198481798171997 +... +``` + +##### Gyroscope + +Gyroscope data is persisted to a text file (`Gyroscope/Gyroscope.txt`) containing newline delimited records, each containing the originating timestamp (see above), and the X-, Y-, and Z-axis angular momentum in rad/s as tab-separated fields. For example: + +```text +Originating Time X-axis Y-axis Z-axis +----------------------------------------------------------------------------------- +637835894409333078 -0.015870876610279083 0.16464650630950928 -0.0266859270632267 +637835894409334594 -0.068954452872276306 0.15596729516983032 0.0281371828168636 +637835894409336109 -0.023053843528032303 0.21979841589927673 -0.0425699315965173 +... +``` + +##### Magnetometer + +Magnetometer data is persisted to a text file (`Magnetometer/Magnetometer.txt`) containing newline delimited records, each containing the originating timestamp (see above), and the X-, Y-, and Z-axis magnetic flux density in μT (microteslas) as tab-separated fields. For example: + +```text +Originating Time X-axis Y-axis Z-axis +------------------------------------------------------------------------------- +637835894403011447 261.45001220703125 -364.6500244140625 459.75003051757813 +637835894403211379 261.75000012000007 -364.95001220703125 458.70001220703125 +637835894403411400 262.20001220703125 -365.70001220703125 460.80001831054688 +... +``` + +#### Head + +Head pose data is persisted to a text file (`Head/Head.txt`) containing newline delimited records, each containing the originating timestamp (see above), and the pose (see above) values as tab-separated fields. For example: + +```text +Originating Time M₀₀ M₀₁ M₀₂ M₀₃ M₁₀ M₁₁ M₁₂ M₁₃ M₂₀ M₂₁ M₂₂ M₂₃ M₃₀ M₃₁ M₃₂ M₃₃ +--------------------------------------------------------------------------------------------------------------------------------------------------------------- +637835897125443909 0.947... -0.010... 0.318... 0.081... 0.053... 0.990... -0.125... -0.003... -0.314... 0.135... 0.939... 0.042... 0 0 0 1 +637835897132944315 0.949... -0.013... 0.314... 0.079... 0.057... 0.989... -0.124... -0.003... -0.303... 0.140... 0.940... 0.042... 0 0 0 1 +637835897135110505 0.945... -0.023... 0.323... 0.080... 0.067... 0.989... -0.127... -0.005... -0.317... 0.140... 0.937... 0.041... 0 0 0 1 +... +``` + +#### Eyes + +Eyes pose data is persisted to a text file (`Eyes/Eyes.txt`) containing newline delimited records, each containing the originating timestamp (see above), and the 3D ray (see above) values as tab-separated fields. For example: + +```text +Originating Time P[x] P[y] P[z] V[x] V[y] V[z] +------------------------------------------------------------------------------- +637835897125443909 0.547... -0.710... 0.418... 0.267... 0.159... 0.990... +637835897132944315 0.547... -0.712... 0.418... 0.267... 0.159... 0.960... +637835897135110505 0.548... -0.712... 0.419... 0.297... 0.160... 0.960... +... +``` + +#### Hands + +Hand pose data is persisted to two text files (`Hands/Left.txt`, `Hands/Right.txt`) containing newline delimited records, each containing the originating timestamp (see above), the `IsGripped`, `IsPinched`, and `IsTracked` flags as booleans (`0` = false, `1` = true), and the pose of each joint (if tracked). For example: + +```text +Originating Time Gripped Pinched Tracked M₀₀ M₀₁ M₀₂ M₀₃ M₁₀ M₁₁ M₁₂ M₁₃ M₂₀ M₂₁ M₂₂ M₂₃ M₃₀ M₃₁ M₃₂ M₃₃ M₀₀ M₀₁ M₀₂ ... +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +637835897125443909 0 1 1 -0.214... -0.336... 0.916... 0.201... -0.531... -0.747... -0.399... 0.110... 0.819... -0.573... -0.018... -0.460... 0 0 0 1 0.916... -0.336... 0.214... ... +637835897125447654 0 0 0 +``` + +The first three values are the flags indicating gripped/pinched/tracked state. If the hand is tracked, then the remaining are sets of 16 doubles for each of the following 26 joints (416 values). + +- Palm +- Wrist +- ThumbMetacarpal +- ThumbProximal +- ThumbDistal +- ThumbTip +- IndexMetacarpal +- IndexProximal +- IndexIntermediate +- IndexDistal +- IndexTip +- MiddleMetacarpal +- MiddleProximal +- MiddleIntermediate +- MiddleDistal +- MiddleTip +- RingMetacarpal +- RingProximal +- RingIntermediate +- RingDistal +- RingTip +- PinkyMetacarpal +- PinkyProximal +- PinkyIntermediate +- PinkyDistal +- PinkyTip + +#### Audio + +Audio buffers are persisted to a WAVE file (`Audio/Audio.wav`) containing IEEE float encoded, 48KHz, single channel data. + +#### Video + +The following camera streams are potentially available (depending on the configuration of the HoloLensCaptureApp): + +- Video - color front-facing camera. +- Preview - mixed reality preview combining video with holograms. +- Infrared - infrared view from the depth camera. +- Depth - far-depth camera. +- AhatDepth - near-depth camera. +- LeftFront - gray-scale camera. +- RightFront - gray-scale camera. +- LeftLeft - gray-scale camera. +- RightRight - gray-scale camera. + +Data from each camera is persisted in separate folder (e.g. `Video/`, `Depth/`, `LeftFront/`, ...). + +##### Frame Images + +A set of frame-by-frame image files in the form (`000001.jpg`, `000071.png`) are persisted. If the original stream is encoded (depending on HoloLensCaptureApp configuration), then frames are persisted as JPEG files. If the original stream is GZIPed or unencoded, then frames are persisted as lossless PNG files. + +##### Timings + +Per-frame originating timestamps are persisted to a `Timing.txt` file as a tab-separated pair of frame number and timestamp. For example: + +```text +Frame Originating Time +------------------------- +0 637835897126152279 +1 637835897132152359 +2 637835897142152493 +``` + +##### Camera Pose + +The pose of the camera over time is persisted to a `Pose.txt` file containing newline delimited records, each containing the originating timestamp (see above), and the pose (see above) values as tab-separated fields. For example: + +```text +Originating Time M₀₀ M₀₁ M₀₂ M₀₃ M₁₀ M₁₁ M₁₂ M₁₃ M₂₀ M₂₁ M₂₂ M₂₃ M₃₀ M₃₁ M₃₂ M₃₃ +--------------------------------------------------------------------------------------------------------------------------------------------------------------- +637835897126152279 0.768... -0.013... 0.639... 0.147... 0.097... 0.990... -0.097... -0.011... -0.631... 0.136... 0.762... 0.041... 0 0 0 1 +``` + +##### Camera Intrinsics + +The intrinsics of the camera are persisted to an `Intrinsics.txt` file containing tab-separated fields containing the intrinsics matrix, distortion parameters, focal length information, the principal point, etc. as described in detail below. + +```text +M₀₀ M₀₁ M₀₂ M₁₀ M₁₁ M₁₂ M₂₀ M₂₁ M₂₂ R₀ R₁ R₂ R₃ R₄ R₅ T₀ T₁ FL FLₓ FLᵧ PPₓ PPᵧ D W H +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- +251.437... 0 160.242... 0 251.647... 168.953... 0 0 1 -0.282... 0.064... 0 0 0 0 0 0 251.542... 251.437... 251.647... 160.242... 168.953... 1 320 288 +``` + +The first 9 values represent the 3×3 intrinsics matrix (M₀₀ - M₂₂). This transform converts camera coordinates (in the camera's local space) into normalized device coordinates (NDC) ranging from -1 .. +1. + +The next 6 values represent the radial distortion parameters (R₀ - R₅). + +The next 2 values represent the tangential distortion parameters (T₀, T₁). + +The next value represents the focal length in pixels (FL). + +The next 2 values represent the focal length separated in X and Y in pixels (FLₓ, FLᵧ). + +The next 2 values represent the principal point in pixels (PPₓ, PPᵧ). + +The next value is a boolean indicating whether the closed form equation of the Brown-Conrady Distortion model distorts or undistorts (D). + +The final two values represent the image width (W) and height (H) in pixels. + +##### Calibration Maps + +When configured in the HoloLensCaptureApp (`includeDepthCalibrationMap = true`), pixel-by-pixel calibration maps are persisted to a `CalibrationMap.txt` file containing tab-separated fields describing the width (W), height (H) and per-pixel values of points on the camera unit plane (P₀ - Pₙ). + +```text +W H P₀ P₁ ... Pₙ +---------------------------------- +640 480 0.123 0.456 ... 0.789 +``` + +##### Scene Understanding + +The [scene understanding](https://docs.microsoft.com/en-us/windows/mixed-reality/design/scene-understanding) stream contains information about observed surfaces in the environment. This includes integrated 3D meshes and rectangular "flat" areas. Sets of scene information is classified into the following categories: + +- __World__ - The World objects are _all_ of the observed surfaces. +- __Inferred__ - Inferred objects are generally bits of mesh to fill in unobserved, but assumed, portions of surfaces. +- __Background__ - Known to be not one of the other recognized kinds of scene object. This class shouldn't be confused with Unknown where Background is known not to be wall/floor/ceiling etc. while Unknown isn't yet categorized. +- __Wall__ - A physical wall. Walls are assumed to be immovable environmental structures. +- __Floor__ - Floors are any surfaces on which one can walk. Note: stairs aren't floors. Also note, that floors assume any walkable surface and therefore there's no explicit assumption of a singular floor. Multi-level structures, ramps etc. should all classify as floor. +- __Ceiling__ - The upper surface of a room. +- __Platform__ - A large flat surface on which you could place holograms. These tend to represent tables, countertops, and other large horizontal surfaces. +- __Unknown__ - This scene object has yet to be classified and assigned a kind. This shouldn't be confused with Background, as this object could be anything, the system has just not come up with a strong enough classification for it yet. + +Each of these classes of scene objects are given a directory within `SceneUnderstanding` (e.g. `SceneUnderstanding/World`, `SceneUnderstanding/Floor`, ...). Within each directory there are two text files and two subdirectories containing meshes. + +`Rectangles.txt` contains flat surface rectangles defined by the corner points. Newline delimited records, each contain the originating timestamp (see above), and sets of four pairs (8) of tab-separated fields defining a rectangle (in top-left, top-right, bottom-left, bottom-right order). Zero or more rectangles may be present for a given timestamp. Each rectangle is followed by a "placement rectangle" which represents a smaller rectangle within the first that is deemed the best area to place something (e.g. the best area on a table surface, roughly centered but without intersecting objects). If no suitable placement area is found, then 8 `NaN` values will be given. For example: + +```text +Originating Time R₀ₓ R₀ᵧ R₁ₓ R₁ᵧ R₂ₓ R₂ᵧ R₃ₓ R₃ᵧ P₀ₓ P₀ᵧ P₁ₓ P₁ᵧ P₂ₓ P₂ᵧ P₃ₓ P₃ᵧ ... +------------------------------------------------------------------------------------------------------------------------------------------------------ +637835897112518938 -4.574... -1.062... -4.144... 2.392... 0.701... -1.718... 1.736... -1.745... NaN NaN NaN NaN NaN NaN NaN NaN ... +637835897712518938 1.143... 1.832... 0.701... -1.725... -4.133... 2.486... -4.574... -1.071... NaN NaN NaN NaN NaN NaN NaN NaN ... +``` + +A `Meshes.txt` file contains counts of the number of meshes exported at each timestamp. Meshes come in two flavors: plain `Meshes` which may contain a high degree of detail and `ColliderMeshes` which may be simplified and are intended for collision and occlusion rather than high fidelity rendering. Newline delimited records, each contain the originating timestamp (see above), and the count of meshes and collider meshes. For example: + +```text +Originating Time Meshes Collider +------------------------------------ +637835897112518938 9 9 +637835897712518938 9 9 +``` + +Within the subfolders `/Meshes` and `/ColliderMeshes`, [mesh `.obj` files](https://en.wikipedia.org/wiki/Wavefront_.obj_file) are exported. Directories are created at each timestamp (e.g. `Meshes/637835897112518938`) and files named `Mesh0.obj`, `Mesh1.obj`, are exported. + +### Debugging + +The following streams are for debugging purposes and are not persisted, but may be viewed in the original \psi store using PsiStudio. + +- HoloLensDiagnostics +- DepthDebugOutOfOrderFrames +- AhatDebugOutOfOrderFrames diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/Verbs.cs b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/Verbs.cs new file mode 100644 index 000000000..bcf95d807 --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/Verbs.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace HoloLensCaptureExporter +{ + using CommandLine; + + /// + /// Command-line verbs. + /// + internal class Verbs + { + /// + /// Base command-line options. + /// + internal class ExportCommand + { + /// + /// Gets or sets the file path of the input Psi data store. + /// + [Option('n', "name", Required = false, HelpText = "Name of the input Psi data store (default: HoloLensCapture).")] + public string StoreName { get; set; } = "HoloLensCapture"; + + /// + /// Gets or sets the file path of the input Psi data store. + /// + [Option('p', "path", Required = true, HelpText = "Path to the input Psi data store.")] + public string StorePath { get; set; } + + /// + /// Gets or sets the output path to export data to. + /// + [Option('o', "output", Required = true, HelpText = "Output path to export data to.")] + public string OutputPath { get; set; } + } + } +} \ No newline at end of file diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/stylecop.json b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/stylecop.json new file mode 100644 index 000000000..6f09427eb --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureExporter/stylecop.json @@ -0,0 +1,16 @@ +{ + // ACTION REQUIRED: This file was automatically added to your project, but it + // will not take effect until additional steps are taken to enable it. See the + // following page for additional information: + // + // https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/EnableConfiguration.md + + "$schema": "https://raw.githubusercontent.com/DotNetAnalyzers/StyleCopAnalyzers/master/StyleCop.Analyzers/StyleCop.Analyzers/Settings/stylecop.schema.json", + "settings": { + "documentationRules": { + "companyName": "Microsoft Corporation", + "copyrightText": "Copyright (c) Microsoft Corporation. All rights reserved.\nLicensed under the MIT license.", + "xmlHeader": false + } + } +} \ No newline at end of file diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureInterop/HoloLensCaptureInterop.csproj b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureInterop/HoloLensCaptureInterop.csproj new file mode 100644 index 000000000..6f35a4879 --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureInterop/HoloLensCaptureInterop.csproj @@ -0,0 +1,46 @@ + + + + Library + netstandard2.0 + ..\..\..\..\Build\Microsoft.Psi.ruleset + true + + + + true + + true + + + + + true + + true + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureInterop/Serializers.cs b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureInterop/Serializers.cs new file mode 100644 index 000000000..4d9620eb7 --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureInterop/Serializers.cs @@ -0,0 +1,1672 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace HoloLensCaptureInterop +{ + using System; + using System.Collections.Generic; + using System.Drawing; + using System.IO; + using System.Linq; + using MathNet.Numerics.LinearAlgebra; + using MathNet.Spatial.Euclidean; + using Microsoft.Psi; + using Microsoft.Psi.Audio; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Diagnostics; + using Microsoft.Psi.Imaging; + using Microsoft.Psi.Interop.Serialization; + using Microsoft.Psi.MixedReality; + using Microsoft.Psi.Spatial.Euclidean; + using static Microsoft.Psi.Diagnostics.PipelineDiagnostics; + using Image = Microsoft.Psi.Imaging.Image; + + /// + /// Provides serializers and deserializers for the various mixed reality streams. + /// + public static class Serializers + { + private static readonly WaveFormat AssumedWaveFormat = WaveFormat.CreateIeeeFloat(48000, 1); + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format SceneObjectCollectionFormat() + => new (WriteSceneObjectCollection, ReadSceneObjectCollection); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteSceneObjectCollection(SceneObjectCollection sceneObjectCollection, BinaryWriter writer) + { + void WriteSceneObject(SceneObjectCollection.SceneObject obj) + { + void WriteMeshes(List meshes) + { + WriteCollection(meshes, writer, m => + { + WriteCollection(m.Vertices, writer, WritePoint3D); + WriteCollection(m.TriangleIndices, writer, i => writer.Write(i)); + }); + } + + if (obj != null) + { + WriteMeshes(obj.Meshes); + WriteMeshes(obj.ColliderMeshes); + WriteCollection(obj.Rectangles, writer, WriteRectangle3D); + WriteCollection(obj.PlacementRectangles, writer, rect => WriteNullable(rect, writer, r => WriteRectangle3D(r, writer))); + } + else + { + // treat null as empty collections + writer.Write(0); // empty meshes + writer.Write(0); // empty collider meshes + writer.Write(0); // empty rectangles + writer.Write(0); // empty placement rectangles + } + } + + WriteBool(sceneObjectCollection != null, writer); + if (sceneObjectCollection == null) + { + return; + } + + WriteSceneObject(sceneObjectCollection.Background); + WriteSceneObject(sceneObjectCollection.Ceiling); + WriteSceneObject(sceneObjectCollection.Floor); + WriteSceneObject(sceneObjectCollection.Inferred); + WriteSceneObject(sceneObjectCollection.Platform); + WriteSceneObject(sceneObjectCollection.Unknown); + WriteSceneObject(sceneObjectCollection.Wall); + WriteSceneObject(sceneObjectCollection.World); + } + + /// + /// Read from . + /// + /// from which to read. + /// . + public static SceneObjectCollection ReadSceneObjectCollection(BinaryReader reader) + { + SceneObjectCollection.SceneObject ReadSceneObject() + { + List ReadMeshes() + { + return ReadCollection(reader, () => + { + var vertices = ReadCollection(reader, ReadPoint3D).ToArray(); + var indices = ReadCollection(reader, reader.ReadUInt32).ToArray(); + return new Mesh3D(vertices, indices); + }).ToList(); + } + + var meshes = ReadMeshes(); + var colliderMeshes = ReadMeshes(); + var rectangles = ReadCollection(reader, ReadRectangle3D).ToList(); + var placementRectangles = ReadCollection(reader, () => ReadNullable(reader, () => ReadRectangle3D(reader))).ToList(); + + return new SceneObjectCollection.SceneObject(meshes, colliderMeshes, rectangles, placementRectangles); + } + + if (!ReadBool(reader)) + { + return null; + } + + var background = ReadSceneObject(); + var ceiling = ReadSceneObject(); + var floor = ReadSceneObject(); + var inferred = ReadSceneObject(); + var platform = ReadSceneObject(); + var unknown = ReadSceneObject(); + var wall = ReadSceneObject(); + var world = ReadSceneObject(); + var scene = new SceneObjectCollection( + background, + ceiling, + floor, + inferred, + platform, + unknown, + wall, + world); + return scene; + } + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format CoordinateSystemFormat() + => new (WriteCoordinateSystem, ReadCoordinateSystem); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteCoordinateSystem(CoordinateSystem coordinateSystem, BinaryWriter writer) + { + WriteBool(coordinateSystem != null, writer); + if (coordinateSystem == null) + { + return; + } + + var m = coordinateSystem.AsColumnMajorArray(); + for (var i = 0; i < 16; i++) + { + if ((i + 1) % 4 != 0 /* not bottom row? */) + { + writer.Write(m[i]); + } + } + } + + /// + /// Read from . + /// + /// from which to read. + /// . + public static CoordinateSystem ReadCoordinateSystem(BinaryReader reader) + { + if (!ReadBool(reader)) + { + return null; + } + + var m = new double[12]; + for (var i = 0; i < 12; i++) + { + m[i] = reader.ReadDouble(); + } + + return new CoordinateSystem( + Matrix.Build.DenseOfArray(new double[,] + { + { m[0], m[3], m[6], m[9] }, + { m[1], m[4], m[7], m[10] }, + { m[2], m[5], m[8], m[11] }, + { 0, 0, 0, 1 }, + })); + } + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format Ray3DFormat() + => new (WriteRay3D, ReadRay3D); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteRay3D(Ray3D ray3d, BinaryWriter writer) + { + WritePoint3D(ray3d.ThroughPoint, writer); + WriteVector3D(ray3d.Direction.ToVector3D(), writer); + } + + /// + /// Read from . + /// + /// from which to read. + /// . + public static Ray3D ReadRay3D(BinaryReader reader) + { + var throughPoint = ReadPoint3D(reader); + var direction = ReadVector3D(reader); + return new Ray3D(throughPoint, direction); + } + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format CalibrationPointsMapFormat() + => new (WriteCalibrationPointsMap, ReadCalibrationPointsMap); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteCalibrationPointsMap(CalibrationPointsMap calibrationPointsMap, BinaryWriter writer) + { + writer.Write(calibrationPointsMap.Width); + writer.Write(calibrationPointsMap.Height); + WriteCollection(calibrationPointsMap.CameraUnitPlanePoints, writer, f => writer.Write(f)); + } + + /// + /// Read from . + /// + /// from which to read. + /// . + public static CalibrationPointsMap ReadCalibrationPointsMap(BinaryReader reader) + => new () + { + Width = reader.ReadInt32(), + Height = reader.ReadInt32(), + CameraUnitPlanePoints = ReadCollection(reader, () => reader.ReadDouble()).ToArray(), + }; + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format HandFormat() + => new (WriteHand, ReadHand); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteHand(Hand hand, BinaryWriter writer) + { + WriteBool(hand != null, writer); + if (hand == null) + { + return; + } + + WriteBool(hand.IsTracked, writer); + WriteBool(hand.IsPinched, writer); + WriteBool(hand.IsGripped, writer); + + foreach (var j in hand.Joints) + { + WriteCoordinateSystem(j, writer); + } + } + + /// + /// Read from . + /// + /// from which to read. + /// . + public static Hand ReadHand(BinaryReader reader) + { + if (!ReadBool(reader)) + { + return null; + } + + var isTracked = ReadBool(reader); + var isPinched = ReadBool(reader); + var isGripped = ReadBool(reader); + var numJoints = (int)HandJointIndex.MaxIndex; + var joints = new CoordinateSystem[numJoints]; + for (var i = 0; i < numJoints; i++) + { + joints[i] = ReadCoordinateSystem(reader); + } + + return new Hand(isTracked, isPinched, isGripped, joints); + } + + /// + /// Format for . + /// + /// of serializer/deserializer. + public static Format<(Hand Left, Hand Right)> HandsFormat() + => new (WriteHands, ReadHands); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteHands((Hand Left, Hand Right) hands, BinaryWriter writer) + { + WriteHand(hands.Left, writer); + WriteHand(hands.Right, writer); + } + + /// + /// Read from . + /// + /// from which to read. + /// . + public static (Hand Left, Hand Right) ReadHands(BinaryReader reader) + { + return (ReadHand(reader), ReadHand(reader)); + } + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format AudioBufferFormat() + => new (WriteAudioBuffer, ReadAudioBuffer); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteAudioBuffer(AudioBuffer audioBuffer, BinaryWriter writer) + { + if (audioBuffer.Format.Channels != AssumedWaveFormat.Channels || + audioBuffer.Format.FormatTag != AssumedWaveFormat.FormatTag || + audioBuffer.Format.BitsPerSample != AssumedWaveFormat.BitsPerSample || + audioBuffer.Format.SamplesPerSec != AssumedWaveFormat.SamplesPerSec) + { + throw new ArgumentException("Unexpected audio format."); + } + + writer.Write(audioBuffer.Length); + writer.Write(audioBuffer.Data); + } + + /// + /// Read from . + /// + /// from which to read. + /// . + public static AudioBuffer ReadAudioBuffer(BinaryReader reader) + => new (reader.ReadBytes(reader.ReadInt32()), AssumedWaveFormat); + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format CameraIntrinsicsFormat() + => new (WriteCameraIntrinsics, ReadCameraIntrinsics); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteCameraIntrinsics(ICameraIntrinsics cameraIntrinsics, BinaryWriter writer) + { + static void WriteVector(Vector vector, int size, BinaryWriter writer) + { + if (vector.Count != size) + { + throw new ArgumentException($"Expected vector of size {size} (actual: {vector.Count}."); + } + + foreach (var v in vector) + { + writer.Write(v); + } + } + + WriteBool(cameraIntrinsics != null, writer); + if (cameraIntrinsics == null) + { + return; + } + + writer.Write(cameraIntrinsics.ImageWidth); + writer.Write(cameraIntrinsics.ImageHeight); + WriteTransformMatrix(cameraIntrinsics.Transform, writer); + WriteVector(cameraIntrinsics.RadialDistortion, 6, writer); + WriteVector(cameraIntrinsics.TangentialDistortion, 2, writer); + WriteBool(cameraIntrinsics.ClosedFormDistorts, writer); + } + + /// + /// Read from . + /// + /// from which to read. + /// . + public static CameraIntrinsics ReadCameraIntrinsics(BinaryReader reader) + { + static Vector ReadVector(int size, BinaryReader reader) + { + var v = new double[size]; + for (var i = 0; i < size; i++) + { + v[i] = reader.ReadDouble(); + } + + return Vector.Build.DenseOfArray(v); + } + + if (!ReadBool(reader)) + { + return null; + } + + var width = reader.ReadInt32(); + var height = reader.ReadInt32(); + var transform = ReadTransformMatrix(reader); + var radialDistortion = ReadVector(6, reader); + var tangentialDistortion = ReadVector(2, reader); + var closedFormDistorts = ReadBool(reader); + var intrinsics = new CameraIntrinsics( + width, + height, + transform, + radialDistortion, + tangentialDistortion, + closedFormDistorts); + return intrinsics; + } + + /// + /// Format for encoded image camera views. + /// + /// of encoded image camera view serializer/deserializer. + public static Format EncodedImageCameraViewFormat() + => new (WriteEncodedImageCameraView, (reader, payload, _, _) => ReadEncodedImageCameraView(reader, payload)); + + /// + /// Write a encoded image camera view to . + /// + /// The encoded camera image view. + /// to which to write. + public static void WriteEncodedImageCameraView(EncodedImageCameraView encodedImageCameraView, BinaryWriter writer) + { + WriteEncodedImage(encodedImageCameraView.ViewedObject, writer); + WriteCameraIntrinsics(encodedImageCameraView.CameraIntrinsics, writer); + WriteCoordinateSystem(encodedImageCameraView.CameraPose, writer); + } + + /// + /// Read encoded image camera view from . + /// + /// from which to read. + /// The payload of bytes. + /// The encoded image camera view. + public static EncodedImageCameraView ReadEncodedImageCameraView(BinaryReader reader, byte[] payload) + { + using var sharedEncodedImage = ReadEncodedImage(reader, payload); + var cameraIntrinsics = ReadCameraIntrinsics(reader); + var cameraPose = ReadCoordinateSystem(reader); + return new EncodedImageCameraView(sharedEncodedImage, cameraIntrinsics, cameraPose); + } + + /// + /// Format for . + /// + /// of serializer/deserializer. + public static Format> SharedEncodedImageFormat() + => new (WriteEncodedImage, (reader, payload, _, _) => ReadEncodedImage(reader, payload)); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteEncodedImage(Shared sharedEncodedImage, BinaryWriter writer) + { + WriteBool(sharedEncodedImage != null, writer); + if (sharedEncodedImage == null) + { + return; + } + + var image = sharedEncodedImage.Resource; + var data = image.GetBuffer(); + writer.Write(image.Width); + writer.Write(image.Height); + writer.Write((int)image.PixelFormat); + writer.Write(image.Size); + writer.Write(data, 0, image.Size); + } + + /// + /// Read from . + /// + /// from which to read. + /// The payload of bytes. + /// . + public static Shared ReadEncodedImage(BinaryReader reader, byte[] payload) + { + if (!ReadBool(reader)) + { + return null; + } + + var width = reader.ReadInt32(); + var height = reader.ReadInt32(); + var pixelFormat = (PixelFormat)reader.ReadInt32(); + var size = reader.ReadInt32(); + var image = EncodedImagePool.GetOrCreate(width, height, pixelFormat); + int position = (int)reader.BaseStream.Position; + image.Resource.CopyFrom(payload, position, size); + reader.BaseStream.Position = position + size; + return image; + } + + /// + /// Format for depth image camera views. + /// + /// of depth image camera view serializer/deserializer. + public static Format DepthImageCameraViewFormat() + => new (WriteDepthImageCameraView, (reader, payload, _, _) => ReadDepthImageCameraView(reader, payload)); + + /// + /// Write a shared depth image camera view to . + /// + /// The depth image camera view. + /// to which to write. + public static void WriteDepthImageCameraView(DepthImageCameraView depthImageCameraView, BinaryWriter writer) + { + WriteDepthImage(depthImageCameraView.ViewedObject, writer); + WriteCameraIntrinsics(depthImageCameraView.CameraIntrinsics, writer); + WriteCoordinateSystem(depthImageCameraView.CameraPose, writer); + } + + /// + /// Read shared depth image camera view from . + /// + /// from which to read. + /// The payload of bytes. + /// The shared depth image camera view. + public static DepthImageCameraView ReadDepthImageCameraView(BinaryReader reader, byte[] payload) + { + using var sharedDepthImage = ReadDepthImage(reader, payload); + var cameraIntrinsics = ReadCameraIntrinsics(reader); + var cameraPose = ReadCoordinateSystem(reader); + return new DepthImageCameraView(sharedDepthImage, cameraIntrinsics, cameraPose); + } + + /// + /// Format for . + /// + /// of serializer/deserializer. + public static Format> SharedDepthImageFormat() + => new (WriteDepthImage, (reader, payload, _, _) => ReadDepthImage(reader, payload)); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteDepthImage(Shared sharedDepthImage, BinaryWriter writer) + { + WriteBool(sharedDepthImage != null, writer); + if (sharedDepthImage == null) + { + return; + } + + var image = sharedDepthImage.Resource; + writer.Write(image.Width); + writer.Write(image.Height); + writer.Write(image.Size); + writer.Write((byte)image.DepthValueSemantics); + writer.Write(image.DepthValueToMetersScaleFactor); + writer.Write(image.ReadBytes(image.Size)); + } + + /// + /// Read from . + /// + /// from which to read. + /// The payload of bytes. + /// . + public static Shared ReadDepthImage(BinaryReader reader, byte[] payload) + { + if (!ReadBool(reader)) + { + return null; + } + + var width = reader.ReadInt32(); + var height = reader.ReadInt32(); + var size = reader.ReadInt32(); + var depthValueSemantics = (DepthValueSemantics)reader.ReadByte(); + var depthValueToMetersScaleFactor = reader.ReadDouble(); + var image = DepthImagePool.GetOrCreate(width, height, depthValueSemantics, depthValueToMetersScaleFactor); + int position = (int)reader.BaseStream.Position; + image.Resource.CopyFrom(payload, position, size); + reader.BaseStream.Position = position + size; + return image; + } + + /// + /// Format for image camera views. + /// + /// of image camera view serializer/deserializer. + public static Format ImageCameraViewFormat() + => new (WriteImageCameraView, (reader, payload, _, _) => ReadImageCameraView(reader, payload)); + + /// + /// Write an image camera view to . + /// + /// The image camera view. + /// to which to write. + public static void WriteImageCameraView(ImageCameraView imageCameraView, BinaryWriter writer) + { + WriteSharedImage(imageCameraView.ViewedObject, writer); + WriteCameraIntrinsics(imageCameraView.CameraIntrinsics, writer); + WriteCoordinateSystem(imageCameraView.CameraPose, writer); + } + + /// + /// Read an image camera view from . + /// + /// from which to read. + /// The payload of bytes. + /// The shared camera image view. + public static ImageCameraView ReadImageCameraView(BinaryReader reader, byte[] payload) + { + var sharedImage = ReadSharedImage(reader, payload); + var cameraIntrinsics = ReadCameraIntrinsics(reader); + var cameraPose = ReadCoordinateSystem(reader); + return new ImageCameraView(sharedImage, cameraIntrinsics, cameraPose); + } + + /// + /// Format for . + /// + /// of serializer/deserializer. + public static Format> SharedImageFormat() + => new (WriteSharedImage, (reader, payload, _, _) => ReadSharedImage(reader, payload)); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteSharedImage(Shared sharedImage, BinaryWriter writer) + { + WriteBool(sharedImage != null, writer); + if (sharedImage == null) + { + return; + } + + var image = sharedImage.Resource; + writer.Write(image.Width); + writer.Write(image.Height); + writer.Write((int)image.PixelFormat); + writer.Write(image.Size); + writer.Write(image.ReadBytes(image.Size)); + } + + /// + /// Read from . + /// + /// from which to read. + /// The payload of bytes. + /// . + public static Shared ReadSharedImage(BinaryReader reader, byte[] payload) + { + if (!ReadBool(reader)) + { + return null; + } + + var width = reader.ReadInt32(); + var height = reader.ReadInt32(); + var pixelFormat = (PixelFormat)reader.ReadInt32(); + var size = reader.ReadInt32(); + var image = ImagePool.GetOrCreate(width, height, pixelFormat); + int position = (int)reader.BaseStream.Position; + image.Resource.CopyFrom(payload, position, size); + reader.BaseStream.Position = position + size; + return image; + } + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format Int32Format() + { + return new ( + (value, writer) => writer.Write(value), + (reader) => reader.ReadInt32()); + } + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format PipelineDiagnosticsFormat() + { + /* Pipeline diagnostics structure is a graph with many cycles. The strategy to serialize/deserialize + * this is to use mutually recursive write/read functions and hashsets/dictionaries of known entities. + * We write out whole entities (emitters/receivers/elements/...) the first time they're seen while + * walking the graph, but then to write out only the ID when seen again in the future. Then, while + * deserializing, we remember entities that have already been deserialized and upon seeing an ID for + * which we have a cached entity, we stop and return the cached instance. One catch is that instanced + * may be in the process of being deserialized when referenced to them are seen. In this case, a + * partially complete instance is used that is later completed by the time deserialization has finished. + */ + return new ( + (pipelineDiagnostics, writer) => + { + WriteBool(pipelineDiagnostics != null, writer); + if (pipelineDiagnostics == null) + { + return; + } + + var knownEmitterIds = new HashSet(); + var knownReceiverIds = new HashSet(); + var knownPipelineElementIds = new HashSet(); + var knownPipelineIds = new HashSet(); + + void WriteEmitterDiagnostics(EmitterDiagnostics emitterDiagnostics) + { + WriteBool(emitterDiagnostics != null, writer); + if (emitterDiagnostics == null) + { + return; + } + + writer.Write(emitterDiagnostics.Id); + + // serialize only if an instance with this ID has not already been serialized + if (!knownEmitterIds.Contains(emitterDiagnostics.Id)) + { + knownEmitterIds.Add(emitterDiagnostics.Id); + WriteString(emitterDiagnostics.Name, writer); + WriteString(emitterDiagnostics.Type, writer); + WritePipelineElementDiagnostics(emitterDiagnostics.PipelineElement); + WriteCollection(emitterDiagnostics.Targets, writer, WriteReceiverDiagnostics); + } + } + + void WriteReceiverDiagnostics(ReceiverDiagnostics receiverDiagnostics) + { + WriteBool(receiverDiagnostics != null, writer); + if (receiverDiagnostics == null) + { + return; + } + + writer.Write(receiverDiagnostics.Id); + + // serialize only if an instance with this ID has not already been serialized + if (!knownReceiverIds.Contains(receiverDiagnostics.Id)) + { + knownReceiverIds.Add(receiverDiagnostics.Id); + WriteString(receiverDiagnostics.ReceiverName, writer); + WriteString(receiverDiagnostics.DeliveryPolicyName, writer); + WriteString(receiverDiagnostics.TypeName, writer); + WriteBool(receiverDiagnostics.ReceiverIsThrottled, writer); + writer.Write(receiverDiagnostics.LastDeliveryQueueSize); + writer.Write(receiverDiagnostics.AvgDeliveryQueueSize); + writer.Write(receiverDiagnostics.TotalMessageEmittedCount); + writer.Write(receiverDiagnostics.WindowMessageEmittedCount); + writer.Write(receiverDiagnostics.TotalMessageProcessedCount); + writer.Write(receiverDiagnostics.WindowMessageProcessedCount); + writer.Write(receiverDiagnostics.TotalMessageDroppedCount); + writer.Write(receiverDiagnostics.WindowMessageDroppedCount); + writer.Write(receiverDiagnostics.LastMessageCreatedLatency); + writer.Write(receiverDiagnostics.AvgMessageCreatedLatency); + writer.Write(receiverDiagnostics.LastMessageEmittedLatency); + writer.Write(receiverDiagnostics.AvgMessageEmittedLatency); + writer.Write(receiverDiagnostics.LastMessageReceivedLatency); + writer.Write(receiverDiagnostics.AvgMessageReceivedLatency); + writer.Write(receiverDiagnostics.LastMessageProcessTime); + writer.Write(receiverDiagnostics.AvgMessageProcessTime); + writer.Write(receiverDiagnostics.LastMessageSize); + writer.Write(receiverDiagnostics.AvgMessageSize); + WritePipelineElementDiagnostics(receiverDiagnostics.PipelineElement); + WriteEmitterDiagnostics(receiverDiagnostics.Source); + } + } + + void WritePipelineElementDiagnostics(PipelineElementDiagnostics pipelineElementDiagnostics) + { + WriteBool(pipelineElementDiagnostics != null, writer); + if (pipelineElementDiagnostics == null) + { + return; + } + + writer.Write(pipelineElementDiagnostics.Id); + + // serialize only if an instance with this ID has not already been serialized + if (!knownPipelineElementIds.Contains(pipelineElementDiagnostics.Id)) + { + knownPipelineElementIds.Add(pipelineElementDiagnostics.Id); + WriteString(pipelineElementDiagnostics.Name, writer); + WriteString(pipelineElementDiagnostics.TypeName, writer); + writer.Write((int)pipelineElementDiagnostics.Kind); + WriteBool(pipelineElementDiagnostics.IsRunning, writer); + WriteBool(pipelineElementDiagnostics.Finalized, writer); + WriteString(pipelineElementDiagnostics.DiagnosticState, writer); + writer.Write(pipelineElementDiagnostics.PipelineId); + WriteCollection(pipelineElementDiagnostics.Emitters, writer, WriteEmitterDiagnostics); + WriteCollection(pipelineElementDiagnostics.Receivers, writer, WriteReceiverDiagnostics); + WritePipelineDiagnostics(pipelineElementDiagnostics.RepresentsSubpipeline); + WritePipelineElementDiagnostics(pipelineElementDiagnostics.ConnectorBridgeToPipelineElement); + } + } + + void WritePipelineDiagnostics(PipelineDiagnostics diagnostics) + { + WriteBool(diagnostics != null, writer); + if (diagnostics == null) + { + return; + } + + writer.Write(diagnostics.Id); + + // serialize only if an instance with this ID has not already been serialized + if (!knownPipelineIds.Contains(diagnostics.Id)) + { + knownPipelineIds.Add(diagnostics.Id); + WriteString(diagnostics.Name, writer); + WriteBool(diagnostics.IsPipelineRunning, writer); + WritePipelineDiagnostics(diagnostics.ParentPipelineDiagnostics); + WriteCollection(diagnostics.SubpipelineDiagnostics, writer, WritePipelineDiagnostics); + WriteCollection(diagnostics.PipelineElements, writer, WritePipelineElementDiagnostics); + } + } + + WritePipelineDiagnostics(pipelineDiagnostics); + }, + (reader) => + { + if (!ReadBool(reader)) + { + return null; + } + + var knownEmitters = new Dictionary(); + var knownReceivers = new Dictionary(); + var knownPipelineElements = new Dictionary(); + var knownPipelines = new Dictionary(); + + EmitterDiagnostics ReadEmitterDiagnostics() + { + if (!ReadBool(reader)) + { + return null; + } + + var id = reader.ReadInt32(); + if (knownEmitters.TryGetValue(id, out var emitter)) + { + // if this ID is already known, merely return the known instance (potentially partially complete) + return emitter; + } + + // construct and add a partially complete instance before recursing + emitter = new EmitterDiagnostics( + id, + ReadString(reader), // name + ReadString(reader), // type + null, // pipelineElement + null); // targets + knownEmitters.Add(id, emitter); + + // recurse and complete assignment of instance fields + emitter.PipelineElement = ReadPipelineElementDiagnostics(); + emitter.Targets = ReadCollection(reader, ReadReceiverDiagnostics).ToArray(); + + return emitter; + } + + ReceiverDiagnostics ReadReceiverDiagnostics() + { + if (!ReadBool(reader)) + { + return null; + } + + var id = reader.ReadInt32(); + if (knownReceivers.TryGetValue(id, out var receiver)) + { + // if this ID is already known, merely return the known instance (potentially partially complete) + return receiver; + } + + // construct and add a partially complete instance before recursing + receiver = new ReceiverDiagnostics( + id, + ReadString(reader), // receiverName + ReadString(reader), // deliverPolicyName + ReadString(reader), // typeName + ReadBool(reader), // receiverIsThrottled + reader.ReadDouble(), // lastDeliveryQueueSize + reader.ReadDouble(), // avgDeliveryQueueSize + reader.ReadInt32(), // totalMessageEmittedCount + reader.ReadInt32(), // windowMessageEmittedCount + reader.ReadInt32(), // totalMessageProcessedCount + reader.ReadInt32(), // windowMessageProcessedCount + reader.ReadInt32(), // totalMessageDroppedCount + reader.ReadInt32(), // windowMessageDroppedCount + reader.ReadDouble(), // lastMessageCreatedLatency + reader.ReadDouble(), // avgMessageCreatedLatency + reader.ReadDouble(), // lastMessageEmittedLatency + reader.ReadDouble(), // avgMessageEmittedLatency + reader.ReadDouble(), // lastMessageReceivedLatency + reader.ReadDouble(), // avgMessageReceivedLatency + reader.ReadDouble(), // lastMessageProcessTime + reader.ReadDouble(), // avgMessageProcessTime + reader.ReadDouble(), // lastMessageSize + reader.ReadDouble(), // avgMessageSize + null, // pipelineElement + null); // source + knownReceivers.Add(id, receiver); + + // recurse and complete assignment of instance fields + receiver.PipelineElement = ReadPipelineElementDiagnostics(); + receiver.Source = ReadEmitterDiagnostics(); + + return receiver; + } + + PipelineElementDiagnostics ReadPipelineElementDiagnostics() + { + if (!ReadBool(reader)) + { + return null; + } + + var id = reader.ReadInt32(); + if (knownPipelineElements.TryGetValue(id, out var pipelineElement)) + { + // if this ID is already known, merely return the known instance (potentially partially complete) + return pipelineElement; + } + + // construct and add a partially complete instance before recursing + pipelineElement = new PipelineElementDiagnostics( + id, + ReadString(reader), // name + ReadString(reader), // typeName + (PipelineElementKind)reader.ReadInt32(), // kind + ReadBool(reader), // isRunning + ReadBool(reader), // finalized + ReadString(reader), // diagnosticState + reader.ReadInt32(), // pipelineId + null, // emitters + null, // receivers + null, // representsSubpipeline + null); // connectorBridgeToPipelineElement + knownPipelineElements.Add(id, pipelineElement); + + // recurse and complete assignment of instance fields + pipelineElement.Emitters = ReadCollection(reader, ReadEmitterDiagnostics).ToArray(); + pipelineElement.Receivers = ReadCollection(reader, ReadReceiverDiagnostics).ToArray(); + pipelineElement.RepresentsSubpipeline = ReadPipelineDiagnostics(); + pipelineElement.ConnectorBridgeToPipelineElement = ReadPipelineElementDiagnostics(); + + return pipelineElement; + } + + PipelineDiagnostics ReadPipelineDiagnostics() + { + if (!ReadBool(reader)) + { + return null; + } + + var id = reader.ReadInt32(); + if (knownPipelines.TryGetValue(id, out var pipeline)) + { + // if this ID is already known, merely return the known instance (potentially partially complete) + return pipeline; + } + + // construct and add a partially complete instance before recursing + pipeline = new PipelineDiagnostics( + id, + ReadString(reader), // name + ReadBool(reader), // isPipelineRunning + null, // parentPipelineDiagnostics + null, // subpipelineDiagnostics + null); // pipelineElements + knownPipelines.Add(id, pipeline); + + // recurse and complete assignment of instance fields + pipeline.ParentPipelineDiagnostics = ReadPipelineDiagnostics(); + pipeline.SubpipelineDiagnostics = ReadCollection(reader, ReadPipelineDiagnostics).ToArray(); + pipeline.PipelineElements = ReadCollection(reader, ReadPipelineElementDiagnostics).ToArray(); + + return pipeline; + } + + return ReadPipelineDiagnostics(); + }); + } + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format DateTimeFormat() + => new (WriteDateTime, ReadDateTime); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteDateTime(DateTime dateTime, BinaryWriter writer) + => writer.Write(dateTime.ToFileTimeUtc()); + + /// + /// Read from . + /// + /// from which to read. + /// . + public static DateTime ReadDateTime(BinaryReader reader) + => DateTime.FromFileTimeUtc(reader.ReadInt64()); + + /// + /// Format for heartbeat of . + /// + /// of serializer/deserializer. + public static Format<(float VideoFps, float DepthFps)> HeartbeatFormat() + => new (WriteHeartbeat, ReadHeartbeat); + + /// + /// Write heartbeat of to . + /// + /// to write. + /// to which to write. + public static void WriteHeartbeat((float VideoFps, float DepthFps) heartbeat, BinaryWriter writer) + { + writer.Write(heartbeat.VideoFps); + writer.Write(heartbeat.DepthFps); + } + + /// + /// Read heartbeat of from . + /// + /// from which to read. + /// . + public static (float VideoFps, float DepthFps) ReadHeartbeat(BinaryReader reader) + { + var videoFps = reader.ReadSingle(); + var depthFps = reader.ReadSingle(); + return (videoFps, depthFps); + } + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format ColorFormat() + => new (WriteColor, ReadColor); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteColor(Color color, BinaryWriter writer) + => writer.Write(color.ToArgb()); + + /// + /// Read from . + /// + /// from which to read. + /// . + public static Color ReadColor(BinaryReader reader) + => Color.FromArgb(reader.ReadInt32()); + + /// + /// Format for IMU data of . + /// + /// of serializer/deserializer. + public static Format<(Vector3D Sample, DateTime OriginatingTIme)[]> ImuFormat() + => new (WriteImu, ReadImu); + + /// + /// Write IMU data of to . + /// + /// to write. + /// to which to write. + public static void WriteImu((Vector3D Value, DateTime OriginatingTime)[] imu, BinaryWriter writer) + { + WriteCollection(imu, writer, sample => + { + WriteVector3D(sample.Value, writer); + WriteDateTime(sample.OriginatingTime, writer); + }); + } + + /// + /// Read IMU data of from . + /// + /// from which to read. + /// . + public static (Vector3D Value, DateTime OriginatingTime)[] ReadImu(BinaryReader reader) + { + return ReadCollection(reader, () => + { + var vector3D = ReadVector3D(reader); + var originatingTime = ReadDateTime(reader); + return (vector3D, originatingTime); + }).ToArray(); + } + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteVector3D(Vector3D vector3D, BinaryWriter writer) + { + writer.Write(vector3D.X); + writer.Write(vector3D.Y); + writer.Write(vector3D.Z); + } + + /// + /// Read from . + /// + /// from which to read. + /// . + public static Vector3D ReadVector3D(BinaryReader reader) + { + var x = reader.ReadDouble(); + var y = reader.ReadDouble(); + var z = reader.ReadDouble(); + return new Vector3D(x, y, z); + } + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format BoolFormat() + => new (WriteBool, ReadBool); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteBool(bool boolean, BinaryWriter writer) + => writer.Write((byte)(boolean ? 0xff : 0)); + + /// + /// Read from . + /// + /// from which to read. + /// . + public static bool ReadBool(BinaryReader reader) + => reader.ReadByte() == 0xff; + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format GuidFormat() + => new (WriteGuid, ReadGuid); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteGuid(Guid guid, BinaryWriter writer) + => WriteString(guid.ToString(), writer); + + /// + /// Read from . + /// + /// from which to read. + /// . + public static Guid ReadGuid(BinaryReader reader) + => Guid.Parse(ReadString(reader)); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteTransformMatrix(Matrix matrix, BinaryWriter writer) + { + WriteBool(matrix != null, writer); + if (matrix == null) + { + return; + } + + var m = matrix.AsColumnMajorArray(); + for (var i = 0; i < 9; i++) + { + if ((i + 1) % 3 != 0 /* not bottom row? */) + { + writer.Write(m[i]); + } + } + } + + /// + /// Read from . + /// + /// from which to read. + /// . + public static Matrix ReadTransformMatrix(BinaryReader reader) + { + if (!ReadBool(reader)) + { + return null; + } + + var m = new double[6]; + for (var i = 0; i < 6; i++) + { + m[i] = reader.ReadDouble(); + } + + var matrix = Matrix.Build.Dense(3, 3, 0); + matrix[0, 0] = m[0]; + matrix[1, 0] = m[1]; + matrix[0, 1] = m[2]; + matrix[1, 1] = m[3]; + matrix[0, 2] = m[4]; + matrix[1, 2] = m[5]; + matrix[2, 2] = 1; + + return matrix; + } + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format Box3DFormat() + => new (WriteBox3D, ReadBox3D); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteBox3D(Box3D box3D, BinaryWriter writer) + { + WriteBounds3D(box3D.Bounds, writer); + WriteCoordinateSystem(box3D.Pose, writer); + } + + /// + /// Read from . + /// + /// from which to read. + /// . + public static Box3D ReadBox3D(BinaryReader reader) + => new (ReadBounds3D(reader), ReadCoordinateSystem(reader)); + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format Bounds3DFormat() + => new (WriteBounds3D, ReadBounds3D); + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteBounds3D(Bounds3D bounds3D, BinaryWriter writer) + { + writer.Write(bounds3D.Min.X); + writer.Write(bounds3D.Min.Y); + writer.Write(bounds3D.Min.Z); + writer.Write(bounds3D.Max.X); + writer.Write(bounds3D.Max.Y); + writer.Write(bounds3D.Max.Z); + } + + /// + /// Read from . + /// + /// from which to read. + /// . + public static Bounds3D ReadBounds3D(BinaryReader reader) + { + var minX = reader.ReadDouble(); + var minY = reader.ReadDouble(); + var minZ = reader.ReadDouble(); + var maxX = reader.ReadDouble(); + var maxY = reader.ReadDouble(); + var maxZ = reader.ReadDouble(); + return new Bounds3D(minX, maxX, minY, maxY, minZ, maxZ); + } + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WritePoint3D(Point3D point3D, BinaryWriter writer) + { + writer.Write(point3D.X); + writer.Write(point3D.Y); + writer.Write(point3D.Z); + } + + /// + /// Read from . + /// + /// from which to read. + /// . + public static Point3D ReadPoint3D(BinaryReader reader) + { + var x = reader.ReadDouble(); + var y = reader.ReadDouble(); + var z = reader.ReadDouble(); + return new Point3D(x, y, z); + } + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WritePoint2D(Point2D point2D, BinaryWriter writer) + { + writer.Write(point2D.X); + writer.Write(point2D.Y); + } + + /// + /// Read from . + /// + /// from which to read. + /// . + public static Point2D ReadPoint2D(BinaryReader reader) + { + var x = reader.ReadDouble(); + var y = reader.ReadDouble(); + return new Point2D(x, y); + } + + /// + /// Write to . + /// + /// to write. + /// to which to write. + public static void WriteRectangle3D(Rectangle3D rectangle3D, BinaryWriter writer) + { + // note: persisting 3 points. Remaining properties inferred (TopRight, Width, Height, IsDegenerate) + WritePoint3D(rectangle3D.TopLeft, writer); + WritePoint3D(rectangle3D.BottomLeft, writer); + WritePoint3D(rectangle3D.BottomRight, writer); + } + + /// + /// Read from . + /// + /// from which to read. + /// . + public static Rectangle3D ReadRectangle3D(BinaryReader reader) + { + var topLeft = ReadPoint3D(reader); + var bottomLeft = ReadPoint3D(reader); + var bottomRight = ReadPoint3D(reader); + return new Rectangle3D( + bottomLeft, // assumed origin + (bottomRight - bottomLeft).Normalize(), // width axis + (topLeft - bottomLeft).Normalize(), // height axis + 0, // left + 0, // bottom + bottomLeft.DistanceTo(bottomRight), // width + bottomLeft.DistanceTo(topLeft)); // height + } + + /// + /// Format for . + /// + /// serializer/deserializer. + public static Format StringFormat() + => new (WriteString, ReadString); + + /// + /// Write optional string (may be null) to . + /// + /// String value to write. + /// to which to write. + public static void WriteString(string str, BinaryWriter writer) + { + WriteBool(str != null, writer); + if (str == null) + { + return; + } + + writer.Write(str); + } + + /// + /// Read optional string (may be null) from . + /// + /// from which to read. + /// Optional value (may be null). + public static string ReadString(BinaryReader reader) + { + if (!ReadBool(reader)) + { + return null; + } + + return reader.ReadString(); + } + + /// + /// Write to using an to write each element. + /// + /// Type of nullable value. + /// Value to write. + /// to which to write. + /// used to write each element. + public static void WriteNullable(T? value, BinaryWriter writer, Action writeAction) + where T : struct + { + WriteBool(value.HasValue, writer); + if (value.HasValue) + { + writeAction(value.Value); + } + } + + /// + /// Write to using an to write each element. + /// + /// Type of nullable value. + /// Value to write. + /// to which to write. + /// used to write each element. + public static void WriteNullable(T? value, BinaryWriter writer, Action writeAction) + where T : struct + => WriteNullable(value, writer, v => writeAction(v, writer)); + + /// + /// Read from using a to read each element. + /// + /// Type of value. + /// from which to read. + /// used to read each element. + /// of elements. + public static T? ReadNullable(BinaryReader reader, Func readFunc) + where T : struct + { + return ReadBool(reader) ? readFunc() : null; + } + + /// + /// Read from using a to read each element. + /// + /// Type of value. + /// from which to read. + /// used to read each element. + /// of elements. + public static T? ReadNullable(BinaryReader reader, Func readFunc) + where T : struct + => ReadNullable(reader, () => readFunc(reader)); + + /// + /// Write to using an to write each element. + /// + /// Type of collection elements. + /// to write. + /// to which to write. + /// used to write each element. + public static void WriteCollection(IEnumerable collection, BinaryWriter writer, Action writeAction) + { + if (collection == null) + { + WriteBool(false, writer); + } + else + { + WriteBool(true, writer); + var len = collection.Count(); + writer.Write(len); + foreach (var value in collection) + { + writeAction(value); + } + } + } + + /// + /// Write to using an to write each element. + /// + /// Type of collection elements. + /// to write. + /// to which to write. + /// used to write each element. + public static void WriteCollection(IEnumerable collection, BinaryWriter writer, Action writeAction) + => WriteCollection(collection, writer, c => writeAction(c, writer)); + + /// + /// Read from using a to read each element. + /// + /// Type of collection elements. + /// from which to read. + /// used to read each element. + /// of elements. + public static IEnumerable ReadCollection(BinaryReader reader, Func readFunc) + { + if (!ReadBool(reader)) + { + return null; + } + else + { + var len = reader.ReadInt32(); + var collection = new T[len]; + for (var i = 0; i < len; i++) + { + collection[i] = readFunc(); + } + + return collection; + } + } + + /// + /// Read from using a to read each element. + /// + /// Type of collection elements. + /// from which to read. + /// used to read each element. + /// of elements. + public static IEnumerable ReadCollection(BinaryReader reader, Func readFunc) + => ReadCollection(reader, () => readFunc(reader)); + + /// + /// Write to . + /// + /// Type of dictionary key elements. + /// Type of dictionary value elements. + /// to write. + /// to which to write. + /// used to write each key element. + /// used to write each value element. + public static void WriteDictionary( + Dictionary dictionary, + BinaryWriter writer, + Action writeKeyAction, + Action writeValueAction) + { + if (dictionary == null) + { + WriteBool(false, writer); + } + else + { + WriteBool(true, writer); + writer.Write(dictionary.Count); + foreach (var kvp in dictionary) + { + writeKeyAction(kvp.Key); + writeValueAction(kvp.Value); + } + } + } + + /// + /// Write to . + /// + /// Type of dictionary key elements. + /// Type of dictionary value elements. + /// to write. + /// to which to write. + /// used to write each key element. + /// used to write each value element. + public static void WriteDictionary( + Dictionary dictionary, + BinaryWriter writer, + Action writeKeyAction, + Action writeValueAction) + => WriteDictionary(dictionary, writer, d => writeKeyAction(d, writer), d => writeValueAction(d, writer)); + + /// + /// Read from . + /// + /// Type of dictionary key elements. + /// Type of dictionary value elements. + /// from which to read. + /// used to read each key element. + /// used to read each value element. + /// . + public static Dictionary ReadDictionary( + BinaryReader reader, + Func readKeyFunc, + Func readValueFunc) + { + if (!ReadBool(reader)) + { + return null; + } + else + { + var result = new Dictionary(); + var count = reader.ReadInt32(); + for (int i = 0; i < count; i++) + { + result.Add(readKeyFunc(), readValueFunc()); + } + + return result; + } + } + + /// + /// Read from . + /// + /// Type of dictionary key elements. + /// Type of dictionary value elements. + /// from which to read. + /// used to read each key element. + /// used to read each value element. + /// . + public static Dictionary ReadDictionary( + BinaryReader reader, + Func readKeyFunc, + Func readValueFunc) + => ReadDictionary(reader, () => readKeyFunc(reader), () => readValueFunc(reader)); + } +} diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureInterop/stylecop.json b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureInterop/stylecop.json new file mode 100644 index 000000000..6f09427eb --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureInterop/stylecop.json @@ -0,0 +1,16 @@ +{ + // ACTION REQUIRED: This file was automatically added to your project, but it + // will not take effect until additional steps are taken to enable it. See the + // following page for additional information: + // + // https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/EnableConfiguration.md + + "$schema": "https://raw.githubusercontent.com/DotNetAnalyzers/StyleCopAnalyzers/master/StyleCop.Analyzers/StyleCop.Analyzers/Settings/stylecop.schema.json", + "settings": { + "documentationRules": { + "companyName": "Microsoft Corporation", + "copyrightText": "Copyright (c) Microsoft Corporation. All rights reserved.\nLicensed under the MIT license.", + "xmlHeader": false + } + } +} \ No newline at end of file diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/App.config b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/App.config new file mode 100644 index 000000000..a865d1f14 --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/App.config @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/HoloLensCaptureServer.cs b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/HoloLensCaptureServer.cs new file mode 100644 index 000000000..efb90e222 --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/HoloLensCaptureServer.cs @@ -0,0 +1,542 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace HoloLensCaptureServer +{ + using System; + using System.Collections.Generic; + using System.Configuration; + using System.IO; + using System.Text; + using HoloLensCaptureInterop; + using MathNet.Spatial.Euclidean; + using Microsoft.Psi; + using Microsoft.Psi.Audio; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Data; + using Microsoft.Psi.Diagnostics; + using Microsoft.Psi.Imaging; + using Microsoft.Psi.Interop.Rendezvous; + using Microsoft.Psi.Interop.Serialization; + using Microsoft.Psi.Interop.Transport; + using Microsoft.Psi.MixedReality; + using Microsoft.Psi.Spatial.Euclidean; + + /// + /// Capture server to persist streams from the accompanying HoloLencCaptureApp. + /// + public class HoloLensCaptureServer + { + // version number shared by capture app and server to ensure compatiblity. + private const string Version = "v1"; + + // capture actions to execute for expected stream types + private static readonly Dictionary> CaptureStreamAction = new () + { + { + // CoordinateSystem + SimplifyTypeName(typeof(CoordinateSystem).FullName), + (s, t) => CaptureTcpStream(s, t, Serializers.CoordinateSystemFormat()) + }, + { + // Ray3D + SimplifyTypeName(typeof(Ray3D).FullName), + (s, t) => CaptureTcpStream(s, t, Serializers.Ray3DFormat()) + }, + { + // Hand + SimplifyTypeName(typeof(Hand).FullName), + (s, t) => CaptureTcpStream(s, t, Serializers.HandFormat()) + }, + { + // AudioBuffer + SimplifyTypeName(typeof(AudioBuffer).FullName), + (s, t) => CaptureTcpStream(s, t, Serializers.AudioBufferFormat()) + }, + { + // Shared + SimplifyTypeName(typeof(Shared).FullName), + (s, t) => CaptureTcpStream>(s, t, Serializers.SharedImageFormat(), largeMessage: true) + }, + { + // Shared + SimplifyTypeName(typeof(Shared).FullName), + (s, t) => CaptureTcpStream>(s, t, Serializers.SharedEncodedImageFormat(), largeMessage: true, persistFrameRate: true) + }, + { + // Shared + SimplifyTypeName(typeof(Shared).FullName), + (s, t) => CaptureTcpStream>(s, t, Serializers.SharedDepthImageFormat(), largeMessage: true, persistFrameRate: true) + }, + { + // CameraIntrinsics + SimplifyTypeName(typeof(CameraIntrinsics).FullName), + (s, t) => CaptureTcpStream(s, t, Serializers.CameraIntrinsicsFormat()) + }, + { + // CalibrationPointsMap + SimplifyTypeName(typeof(CalibrationPointsMap).FullName), + (s, t) => CaptureTcpStream(s, t, Serializers.CalibrationPointsMapFormat(), largeMessage: true) + }, + { + // SceneObjectCollection + SimplifyTypeName(typeof(SceneObjectCollection).FullName), + (s, t) => CaptureTcpStream(s, t, Serializers.SceneObjectCollectionFormat(), largeMessage: true) + }, + { + // PipelineDiagnostics + SimplifyTypeName(typeof(PipelineDiagnostics).FullName), + (s, t) => CaptureTcpStream(s, t, Serializers.PipelineDiagnosticsFormat()) + }, + { + // int + SimplifyTypeName(typeof(int).FullName), + (s, t) => CaptureTcpStream(s, t, Serializers.Int32Format()) + }, + { + // (Vector3D, DateTime)[] + SimplifyTypeName(typeof((Vector3D, DateTime)[]).FullName), + (s, t) => + { + // relay *frames* of IMU samples + CaptureTcpStream<(Vector3D, DateTime)[]>(s, t, Serializers.ImuFormat()); + + // relay *individual* IMU samples + // GetTcpStream<(Vector3D, DateTime)[]>(Serializers.ImuFormat()).SelectManyImuSamples().Write(stream.StreamName, store); + } + }, + { + // (Hand Left, Hand Right) + SimplifyTypeName(typeof((Hand Left, Hand Right)).FullName), + (s, t) => CaptureTcpStream<(Hand Left, Hand Right)>(s, t, Serializers.HandsFormat(), persistFrameRate: true) + }, + { + // EncodedImageCameraView + SimplifyTypeName(typeof(EncodedImageCameraView).FullName), + (s, t) => CaptureTcpStream(s, t, Serializers.EncodedImageCameraViewFormat(), true, t => t.Dispose(), true) + }, + { + // ImageCameraView + SimplifyTypeName(typeof(ImageCameraView).FullName), + (s, t) => CaptureTcpStream(s, t, Serializers.ImageCameraViewFormat(), true, t => t.Dispose(), true) + }, + { + // DepthImageCameraView + SimplifyTypeName(typeof(DepthImageCameraView).FullName), + (s, t) => CaptureTcpStream(s, t, Serializers.DepthImageCameraViewFormat(), true, t => t.Dispose(), true) + }, + }; + + private static readonly RendezvousServer RendezvousServer = new (); + private static Pipeline captureServerPipeline = null; + private static PsiExporter captureServerStore = null; + private static string logFile = null; + private static Dictionary statistics = null; + + private static void Main(string[] args) + { + // Listen for the client app (HoloLensCaptureApp) to connect and create the pipeline + RendezvousServer.Rendezvous.ProcessAdded += (_, process) => + { + ReportProcessAdded(process); + + if (process.Name == "HoloLensCaptureApp") + { + if (process.Version != Version) + { + throw new Exception($"Connection received from unexpected version of HoloLensCaptureApp (expected {Version}, actual {process.Version})."); + } + + Console.WriteLine($" Starting Capture Server Pipeline"); + CreateAndRunComputeServerPipeline(process); + captureServerPipeline.PipelineExceptionNotHandled += (_, args) => + { + StopComputeServerPipeline($"SERVER PIPELINE RUNTIME EXCEPTION: {args.Exception.Message}"); + }; + } + else if (process.Name != nameof(HoloLensCaptureServer)) + { + throw new Exception($"Connection received from unexpected process named {process.Name}."); + } + }; + + // When the client app is stopped (removed), disposed of the capture server pipeline + RendezvousServer.Rendezvous.ProcessRemoved += (_, process) => + { + ReportProcessRemoved(process); + if (process.Name == "HoloLensCaptureApp") + { + StopComputeServerPipeline("Client stopped recording"); + } + }; + + RendezvousServer.Error += (_, ex) => + { + Console.WriteLine(); + StopComputeServerPipeline($"RENDEZVOUS ERROR: {ex.Message}"); + }; + + AppDomain.CurrentDomain.ProcessExit += (_, _) => + { + StopComputeServerPipeline($"SERVER EXITED"); + }; + + RendezvousServer.Start(); + Console.WriteLine($"Listening on TCP port {RendezvousServer.DefaultPort} for client HoloLensCaptureApp."); + Console.WriteLine("Be sure to check firewall settings (may need to enable Public)."); + + // Wait to press a key to exit + Console.WriteLine("Press any key to exit."); + Console.ReadKey(); + + RendezvousServer.Stop(); + StopComputeServerPipeline("Server manually stopped"); + } + + private static void CreateAndRunComputeServerPipeline(Rendezvous.Process inputRendezvousProcess) + { + var config = ConfigurationManager.AppSettings; + var storeName = config["storeName"]; + var storePath = config["storePath"]; + var diagnosticsInterval = double.Parse(config["diagnosticsIntervalSeconds"]); + + // Create the pipeline, store and output diagnostics + if (captureServerPipeline != null) + { + StopComputeServerPipeline("CLIENT STARTED NEW RECORDING WHILE PREVIOUS STILL RUNNING"); + } + + captureServerPipeline = Pipeline.Create( + enableDiagnostics: diagnosticsInterval > 0, + diagnosticsConfiguration: new DiagnosticsConfiguration() + { + SamplingInterval = TimeSpan.FromSeconds(diagnosticsInterval), + }); + captureServerStore = PsiStore.Create(captureServerPipeline, storeName, storePath); + + captureServerPipeline.Diagnostics.Write("ServerDiagnostics", captureServerStore); + + // Detect app disconnect + var lastAppHeartBeat = DateTime.MaxValue; + var appHeartBeatTimeout = TimeSpan.FromSeconds(20); + Timers.Timer(captureServerPipeline, TimeSpan.FromSeconds(1)).Do(_ => + { + if (DateTime.UtcNow - lastAppHeartBeat > appHeartBeatTimeout) + { + StopComputeServerPipeline("APPLICATION HEARTBEAT LOST"); + } + }); + + // Connect to remote clock on the client app to synchronize clocks + foreach (var endpoint in inputRendezvousProcess.Endpoints) + { + if (endpoint is Rendezvous.RemoteClockExporterEndpoint remoteClockExporterEndpoint) + { + var remoteClock = remoteClockExporterEndpoint.ToRemoteClockImporter(captureServerPipeline); + Console.Write(" Connecting to clock sync ..."); + if (!remoteClock.Connected.WaitOne(10000)) + { + Console.WriteLine("FAILED."); + throw new Exception("Failed to connect to remote clock exporter."); + } + + Console.WriteLine("DONE."); + } + } + + statistics = new (); + foreach (var endpoint in inputRendezvousProcess.Endpoints) + { + if (endpoint is Rendezvous.TcpSourceEndpoint tcpEndpoint) + { + foreach (var stream in tcpEndpoint.Streams) + { + // Determine the correct action to execute for capturing the rendezvous stream, + // based on a simplified version of the stream's type name. + var simpleTypeName = SimplifyTypeName(stream.TypeName); + + if (!CaptureStreamAction.ContainsKey(simpleTypeName)) + { + throw new Exception($"Unknown stream type: {stream.StreamName} ({stream.TypeName})"); + } + + CaptureStreamAction[simpleTypeName](stream, tcpEndpoint); + } + } + else if (endpoint is not Rendezvous.RemoteClockExporterEndpoint) + { + throw new Exception("Unexpected endpoint type."); + } + } + + // Send a server heartbeat + var serverHeartbeat = Generators.Sequence( + captureServerPipeline, + (0f, 0f), + _ => + { + if (statistics.TryGetValue("VideoEncodedImageCameraView", out var videoStats)) + { + if (statistics.TryGetValue("DepthImageCameraView", out var depthStats)) + { + return ((float)videoStats.MessagesPerSecond, + (float)depthStats.MessagesPerSecond); + } + else + { + return ((float)videoStats.MessagesPerSecond, 0.0f); + } + } + else + { + if (statistics.TryGetValue("DepthImageCameraView", out var depthStats)) + { + return (0.0f, (float)depthStats.MessagesPerSecond); + } + else + { + return (0.0f, 0.0f); + } + } + }, + TimeSpan.FromSeconds(0.2) /* 5Hz */); + serverHeartbeat.Write("ServerHeartbeat", captureServerStore); + var heartbeatTcpSource = new TcpWriter<(float, float)>(captureServerPipeline, 16000, Serializers.HeartbeatFormat()); + serverHeartbeat.PipeTo(heartbeatTcpSource); + RendezvousServer.Rendezvous.TryAddProcess( + new Rendezvous.Process( + nameof(HoloLensCaptureServer), + new[] { heartbeatTcpSource.ToRendezvousEndpoint("0.0.0.0", "ServerHeartbeat") }, // dummy host name, ignored by app + Version)); + + // Report statistics to console + logFile = Path.Combine(captureServerStore.Path, "CaptureLog.txt"); + Generators.Sequence( + captureServerPipeline, + string.Empty, + _ => + { + var sb = new StringBuilder(); + foreach (var kv in statistics) + { + sb.Append($"{kv.Key}: {kv.Value}\n"); + } + + return sb.ToString(); + }, + TimeSpan.FromSeconds(10)) + .Do(log => + { + Console.WriteLine(); + Console.WriteLine(log); + File.WriteAllText(logFile, $"Capture Statistics\nVersion: {Version}\n\n{log}\n\nIn progress... "); + }); + + // Run the pipeline + captureServerPipeline.RunAsync(); + Console.WriteLine(" Running..."); + } + + private static void CaptureTcpStream( + Rendezvous.Stream stream, + Rendezvous.TcpSourceEndpoint tcpEndpoint, + IFormatDeserializer deserializer, + bool largeMessage = false, + Action deallocator = null, + bool persistFrameRate = false) + { + var tcpSource = tcpEndpoint.ToTcpSource(captureServerPipeline, deserializer, deallocator: deallocator); + var stats = new StreamStatistics(); + statistics.Add(stream.StreamName, stats); + tcpSource + .Do((_, e) => stats.ReportMessage(e.OriginatingTime)) + .Write(stream.StreamName, captureServerStore, largeMessage); + + if (persistFrameRate) + { + tcpSource + .Window(TimeSpan.FromSeconds(-3), TimeSpan.Zero, DeliveryPolicy.SynchronousOrThrottle) + .Select(b => b.Length / 3.0, DeliveryPolicy.SynchronousOrThrottle) + .Write($"{stream.StreamName}.AvgFrameRate", captureServerStore); + } + } + + private static void StopComputeServerPipeline(string message) + { + if (captureServerPipeline != null) + { + RendezvousServer.Rendezvous.TryRemoveProcess(nameof(HoloLensCaptureServer)); + RendezvousServer.Rendezvous.TryRemoveProcess("HoloLensCaptureApp"); + captureServerPipeline?.Dispose(); + if (captureServerPipeline != null) + { + Console.WriteLine($" Stopped Capture Server Pipeline."); + } + + captureServerPipeline = null; + } + + if (logFile != null) + { + using var writer = File.AppendText(logFile); + writer.Write($"Complete!\n\n({message})"); + logFile = null; + } + } + + private static void ReportProcessAdded(Rendezvous.Process process) + { + Console.WriteLine(); + Console.WriteLine($"PROCESS ADDED: {process.Name}"); + foreach (var endpoint in process.Endpoints) + { + if (endpoint is Rendezvous.TcpSourceEndpoint tcpEndpoint) + { + Console.WriteLine($" ENDPOINT: TCP {tcpEndpoint.Host} {tcpEndpoint.Port}"); + } + else if (endpoint is Rendezvous.NetMQSourceEndpoint netMQEndpoint) + { + Console.WriteLine($" ENDPOINT: NetMQ {netMQEndpoint.Address}"); + } + else if (endpoint is Rendezvous.RemoteExporterEndpoint remoteExporterEndpoint) + { + Console.WriteLine($" ENDPOINT: Remote {remoteExporterEndpoint.Host} {remoteExporterEndpoint.Port} {remoteExporterEndpoint.Transport}"); + } + else if (endpoint is Rendezvous.RemoteClockExporterEndpoint remoteClockExporterEndpoint) + { + Console.WriteLine($" ENDPOINT: Remote Clock {remoteClockExporterEndpoint.Host} {remoteClockExporterEndpoint.Port}"); + } + else + { + throw new ArgumentException($"Unknown type of Endpoint ({endpoint.GetType().Name})."); + } + + foreach (var stream in endpoint.Streams) + { + Console.WriteLine($" STREAM: {stream.StreamName} ({stream.TypeName.Split(',')[0]})"); + } + } + } + + private static void ReportProcessRemoved(Rendezvous.Process process) + { + Console.WriteLine(); + Console.WriteLine($"PROCESS REMOVED: {process.Name}"); + } + + /// + /// Simplify the full type name into just the basic underlying type names, + /// stripping away details like assembly, version, culture, token, etc. + /// For example, for the type (Vector3D, DateTime)[] + /// Input: "System.ValueTuple`2 + /// [[MathNet.Spatial.Euclidean.Vector3D, MathNet.Spatial, Version=0.6.0.0, Culture=neutral, PublicKeyToken=000000000000], + /// [System.DateTime, System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=000000000000]] + /// [], System.Private.CoreLib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=000000000000" + /// Output: "System.ValueTuple`2[[MathNet.Spatial.Euclidean.Vector3D],[System.DateTime]][]". + /// + private static string SimplifyTypeName(string typeName) + { + static string SubstringToComma(string s) + { + var commaIndex = s.IndexOf(','); + if (commaIndex >= 0) + { + return s.Substring(0, commaIndex); + } + else + { + return s; + } + } + + // Split first on open bracket, then on closed bracket + var allSplits = new List(); + foreach (var openSplit in typeName.Split('[')) + { + allSplits.Add(openSplit.Split(']')); + } + + // Re-assemble into a simplified string (without assembly, version, culture, token, etc). + var assembledString = string.Empty; + for (int i = 0; i < allSplits.Count; i++) + { + // Add back an open bracket (except the first time) + if (i != 0) + { + assembledString += "["; + } + + for (int j = 0; j < allSplits[i].Length; j++) + { + // Remove everything after the comma (assembly, version, culture, token, etc). + assembledString += SubstringToComma(allSplits[i][j]); + + // Add back a closed bracket (except the last time) + if (j != allSplits[i].Length - 1) + { + assembledString += "]"; + } + } + } + + return assembledString; + } + + private class StreamStatistics + { + private readonly Queue pastOriginatingTimes = new (); + + /// + /// Gets First message time. + /// + public DateTime FirstMessage { get; private set; } = DateTime.MinValue; + + /// + /// Gets last message time. + /// + public DateTime LastMessage { get; private set; } = DateTime.MaxValue; + + /// + /// Gets message count. + /// + public long MessageCount { get; private set; } = 0; + + /// + /// Gets messages per second. + /// + public double MessagesPerSecond { get; private set; } = 0; + + public void ReportMessage(DateTime originatingTime) + { + this.MessageCount++; + this.LastMessage = originatingTime; + if (this.FirstMessage == DateTime.MinValue) + { + this.FirstMessage = originatingTime; + } + + // compute messages per second over 5 sec sliding window + var windowSeconds = TimeSpan.FromSeconds(5); + this.pastOriginatingTimes.Enqueue(originatingTime); + while (this.pastOriginatingTimes.Peek() <= originatingTime - windowSeconds) + { + this.pastOriginatingTimes.Dequeue(); + } + + this.MessagesPerSecond = (double)this.pastOriginatingTimes.Count / windowSeconds.TotalSeconds; + } + + public override string ToString() + { + if (this.FirstMessage == DateTime.MinValue) + { + return "No messages"; + } + + var elapsed = this.LastMessage - this.FirstMessage; + var mps = this.MessageCount / elapsed.TotalSeconds; + return $"{mps:0.#}/s (frames={this.MessageCount} time={elapsed})"; + } + } + } +} diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/HoloLensCaptureServer.csproj b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/HoloLensCaptureServer.csproj new file mode 100644 index 000000000..435eecb4f --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/HoloLensCaptureServer.csproj @@ -0,0 +1,48 @@ + + + + Exe + net472 + ..\..\..\..\Build\Microsoft.Psi.ruleset + true + + + + AnyCPU + True + + + + AnyCPU + True + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + + diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/Readme.md b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/Readme.md new file mode 100644 index 000000000..f64d02318 --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/Readme.md @@ -0,0 +1,35 @@ +# HoloLens Capture Server + +This app is the companion to the [HoloLensCaptureApp](..\HoloLensCaptureApp) which runs on the HoloLens, capturing data streams and remoting to this app running on another machine. + +Streams are written to a \psi store. Upon stopping in the client (or upon errors) the store is closed. Additionally, pressing a key at the console will exit and properly close any open store. + +Note: The capture server uses the [Rendezvous System](https://github.com/microsoft/psi/wiki/Rendezvous-System) to connect to the HoloLens app via TCP sockets, and all communication happens in the clear. These communication channels are not secure, and the user must ensure the security of the network as appropriate. + +## Configuration + +The store name and path may be configured in `HoloLensCaptureServer.config`: + +```xml + + +``` + +## Statistics + +A text file (`CaptureLog.txt`) is saved alongside the \psi store, containing statistics about the video and depth streams, including the frame count, the time extents and average frames per second. This is updated every 10 seconds while capturing. The file ends with the line "In progress..." while capturing and says "Complete!" followed by information about how/why it completed (e.g. stopped at the client/server or errors). Example: + +```text +Capture Statistics + +Video: FPS 8.3439778778468 (frames=390 time=00:00:46.7402965) +Depth: FPS 1.02304885288577 (frames=43 time=00:00:42.0312284) + +In progress... Complete! + +(Client stopped recording) +``` + +## Deployment + +To deploy a build of the HoloLensCaptureServer, simply build in Visual Studio and copy the entire `Internal\Applications\HoloLensCapture\HoloLensCaptureServer\bin\\net472` folder. This contains the executable `HoloLensCaptureServer.exe` itself and all of its dependencies. diff --git a/Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/stylecop.json b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/stylecop.json new file mode 100644 index 000000000..6f09427eb --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/HoloLensCaptureServer/stylecop.json @@ -0,0 +1,16 @@ +{ + // ACTION REQUIRED: This file was automatically added to your project, but it + // will not take effect until additional steps are taken to enable it. See the + // following page for additional information: + // + // https://github.com/DotNetAnalyzers/StyleCopAnalyzers/blob/master/documentation/EnableConfiguration.md + + "$schema": "https://raw.githubusercontent.com/DotNetAnalyzers/StyleCopAnalyzers/master/StyleCop.Analyzers/StyleCop.Analyzers/Settings/stylecop.schema.json", + "settings": { + "documentationRules": { + "companyName": "Microsoft Corporation", + "copyrightText": "Copyright (c) Microsoft Corporation. All rights reserved.\nLicensed under the MIT license.", + "xmlHeader": false + } + } +} \ No newline at end of file diff --git a/Sources/MixedReality/HoloLensCapture/Readme.md b/Sources/MixedReality/HoloLensCapture/Readme.md new file mode 100644 index 000000000..dd2bb6c5a --- /dev/null +++ b/Sources/MixedReality/HoloLensCapture/Readme.md @@ -0,0 +1,9 @@ +# Capturing HoloLens Sensor Streams + +Several tools are provided for capturing sensor data from the HoloLens to \psi stores and for exporting these stores to other formats. + +* [HoloLensCaptureApp](.\HoloLensCaptureApp) runs on the HoloLens, capturing and remoting data streams. +* [HoloLensCaptureServer](.\HoloLensCaptureServer) runs on a separate machine, receiving remoted data streams and writing to a \psi store. +* [HoloLensCaptureExporter](.\HoloLensCaptureExporter) is a tool to convert data within \psi stores to other formats. + +Data can be collected by running the capture app on the HoloLens and remoting the streams of sensor data to the server app which is running on a different machine. Communication may be over WiFi or via USB tethering. Data stores written by the server may then be examined and analyzed in PsiStudio or may be processed by other \psi applications. While \psi stores are optimized for performance and work well with PsiStudio and other \psi applications, you can also use the exporter tool to export to other standard formats. diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Accelerometer.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Accelerometer.cs index 9a8219472..d9c42b2e9 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Accelerometer.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Accelerometer.cs @@ -3,7 +3,6 @@ namespace Microsoft.Psi.MixedReality { - using System.Numerics; using HoloLens2ResearchMode; using Microsoft.Psi; @@ -16,8 +15,9 @@ public class Accelerometer : ResearchModeImu /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - public Accelerometer(Pipeline pipeline) - : base(pipeline, ResearchModeSensorType.ImuAccel) + /// An optional name for the component. + public Accelerometer(Pipeline pipeline, string name = nameof(Accelerometer)) + : base(pipeline, ResearchModeSensorType.ImuAccel, name) { } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/DepthCamera.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/DepthCamera.cs index 859e8ab18..0e01d9139 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/DepthCamera.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/DepthCamera.cs @@ -8,17 +8,16 @@ namespace Microsoft.Psi.MixedReality using HoloLens2ResearchMode; using Microsoft.Psi; using Microsoft.Psi.Imaging; - using Windows.Perception; + using Microsoft.Psi.Spatial.Euclidean; -/// -/// Depth camera source component. -/// + /// + /// Depth camera source component. + /// public class DepthCamera : ResearchModeCamera { private const byte InvalidMask = 0x80; private const ushort InvalidAhatValue = 4090; - private readonly DepthCameraConfiguration configuration; private readonly bool isLongThrow; /// @@ -26,25 +25,15 @@ public class DepthCamera : ResearchModeCamera /// /// The pipeline to add the component to. /// The configuration for this component. - public DepthCamera(Pipeline pipeline, DepthCameraConfiguration configuration = null) - : base( - pipeline, - (configuration ?? new DepthCameraConfiguration()).Mode, - (configuration ?? new DepthCameraConfiguration()).OutputCalibrationMap, - (configuration ?? new DepthCameraConfiguration()).OutputCalibration) + /// An optional name for the component. + public DepthCamera(Pipeline pipeline, DepthCameraConfiguration configuration = null, string name = nameof(DepthCamera)) + : base(pipeline, configuration ?? new DepthCameraConfiguration(), name) { - this.configuration = configuration ?? new DepthCameraConfiguration(); - - if (this.configuration.Mode != ResearchModeSensorType.DepthLongThrow && - this.configuration.Mode != ResearchModeSensorType.DepthAhat) - { - throw new ArgumentException($"Initializing the depth camera in {this.configuration.Mode} mode is not supported."); - } - - this.isLongThrow = this.configuration.Mode == ResearchModeSensorType.DepthLongThrow; - + this.isLongThrow = this.Configuration.SensorType == ResearchModeSensorType.DepthLongThrow; this.DepthImage = pipeline.CreateEmitter>(this, nameof(this.DepthImage)); + this.DepthImageCameraView = pipeline.CreateEmitter(this, nameof(this.DepthImageCameraView)); this.InfraredImage = pipeline.CreateEmitter>(this, nameof(this.InfraredImage)); + this.InfraredImageCameraView = pipeline.CreateEmitter(this, nameof(this.InfraredImageCameraView)); } /// @@ -52,92 +41,120 @@ public DepthCamera(Pipeline pipeline, DepthCameraConfiguration configuration = n /// public Emitter> DepthImage { get; } + /// + /// Gets the depth image camera view stream. + /// + public Emitter DepthImageCameraView { get; } + /// /// Gets the infrared image stream. /// public Emitter> InfraredImage { get; } + /// + /// Gets the infrared image camera view stream. + /// + public Emitter InfraredImageCameraView { get; } + + /// + /// Gets the depth camera configuration. + /// + protected new DepthCameraConfiguration Configuration => base.Configuration as DepthCameraConfiguration; + /// protected override void ProcessSensorFrame(IResearchModeSensorFrame sensorFrame, ResearchModeSensorResolution resolution, ulong frameTicks, DateTime originatingTime) { - if (this.configuration.OutputCalibrationMap && - (originatingTime - this.CalibrationPointsMap.LastEnvelope.OriginatingTime) > this.configuration.OutputCalibrationMapInterval) - { - // Post the calibration map created at the start - this.CalibrationPointsMap.Post(this.GetCalibrationPointsMap(), originatingTime); - } + var shouldOutputDepthImage = this.Configuration.OutputDepthImage && + (originatingTime - this.DepthImage.LastEnvelope.OriginatingTime) > this.Configuration.OutputMinInterval; - if (this.configuration.OutputCalibration) - { - // Post the intrinsics computed at the start - this.CameraIntrinsics.Post(this.GetCameraIntrinsics(), originatingTime); - } + var shouldOutputDepthImageCameraView = this.Configuration.OutputDepthImageCameraView && + (originatingTime - this.DepthImageCameraView.LastEnvelope.OriginatingTime) > this.Configuration.OutputMinInterval; - if (this.configuration.OutputPose) - { - var timestamp = PerceptionTimestampHelper.FromSystemRelativeTargetTime(TimeSpan.FromTicks((long)frameTicks)); - var rigNodeLocation = this.RigNodeLocator.TryLocateAtTimestamp(timestamp, MixedReality.WorldSpatialCoordinateSystem); + var shouldOutputInfraredImage = this.Configuration.OutputInfraredImage && + (originatingTime - this.InfraredImage.LastEnvelope.OriginatingTime) > this.Configuration.OutputMinInterval; - // The rig node may not always be locatable, so we need a null check - if (rigNodeLocation != null) - { - // Compute the camera pose from the rig node location - var cameraWorldPose = this.ToCameraPose(rigNodeLocation); - this.Pose.Post(cameraWorldPose, originatingTime); - } - } + var shouldOutputInfraredImageCameraView = this.Configuration.OutputInfraredImageCameraView && + (originatingTime - this.InfraredImageCameraView.LastEnvelope.OriginatingTime) > this.Configuration.OutputMinInterval; - var depthFrame = sensorFrame as ResearchModeSensorDepthFrame; - int depthImageWidth = (int)resolution.Width; - int depthImageHeight = (int)resolution.Height; - - // Process and post the depth image if requested - if (this.configuration.OutputDepth) + if (shouldOutputDepthImage || + shouldOutputDepthImageCameraView || + shouldOutputInfraredImage || + shouldOutputInfraredImageCameraView) { - byte[] sigmaBuffer = null; - var depthBuffer = depthFrame.GetBuffer(); + var depthFrame = sensorFrame as ResearchModeSensorDepthFrame; + int depthImageWidth = (int)resolution.Width; + int depthImageHeight = (int)resolution.Height; - if (this.isLongThrow) + // Process and post the depth image if need be + if (shouldOutputDepthImage || shouldOutputDepthImageCameraView) { - sigmaBuffer = depthFrame.GetSigmaBuffer(); // Long-throw only - Debug.Assert(depthBuffer.Length == sigmaBuffer.Length, "Depth and sigma buffers should be of equal size!"); - } + byte[] sigmaBuffer = null; + var depthBuffer = depthFrame.GetBuffer(); + + if (this.isLongThrow) + { + sigmaBuffer = depthFrame.GetSigmaBuffer(); // Long-throw only + Debug.Assert(depthBuffer.Length == sigmaBuffer.Length, "Depth and sigma buffers should be of equal size!"); + } - using var depthImage = DepthImagePool.GetOrCreate(depthImageWidth, depthImageHeight); - Debug.Assert(depthImage.Resource.Size == depthBuffer.Length * sizeof(ushort), "DepthImage size does not match raw depth buffer size!"); + using var depthImage = DepthImagePool.GetOrCreate( + depthImageWidth, + depthImageHeight, + DepthValueSemantics.DistanceToPoint, + 0.001); + Debug.Assert(depthImage.Resource.Size == depthBuffer.Length * sizeof(ushort), "DepthImage size does not match raw depth buffer size!"); - unsafe - { - ushort* depthData = (ushort*)depthImage.Resource.ImageData.ToPointer(); - for (int i = 0; i < depthBuffer.Length; ++i) + unsafe { - bool invalid = this.isLongThrow ? - ((sigmaBuffer[i] & InvalidMask) > 0) : - (depthBuffer[i] >= InvalidAhatValue); + ushort* depthData = (ushort*)depthImage.Resource.ImageData.ToPointer(); + for (int i = 0; i < depthBuffer.Length; ++i) + { + bool invalid = this.isLongThrow ? + ((sigmaBuffer[i] & InvalidMask) > 0) : + (depthBuffer[i] >= InvalidAhatValue); + + *depthData++ = invalid ? (ushort)0 : depthBuffer[i]; + } + } - *depthData++ = invalid ? (ushort)0 : depthBuffer[i]; + if (shouldOutputDepthImage) + { + this.DepthImage.Post(depthImage, originatingTime); + } + + if (shouldOutputDepthImageCameraView) + { + using var depthImageCameraView = new DepthImageCameraView(depthImage, this.GetCameraIntrinsics(), this.GetCameraPose()); + this.DepthImageCameraView.Post(depthImageCameraView, originatingTime); } } - this.DepthImage.Post(depthImage, originatingTime); - } + // Process and post the infrared image if need be + if (shouldOutputInfraredImage || shouldOutputInfraredImageCameraView) + { + var infraredBuffer = depthFrame.GetAbDepthBuffer(); + using var infraredImage = ImagePool.GetOrCreate(depthImageWidth, depthImageHeight, PixelFormat.Gray_16bpp); + Debug.Assert(infraredImage.Resource.Size == infraredBuffer.Length * sizeof(ushort), "InfraredImage size does not match raw infrared buffer size!"); - // Process and post the infrared image if requested - if (this.configuration.OutputInfrared) - { - var infraredBuffer = depthFrame.GetAbDepthBuffer(); - using var infraredImage = ImagePool.GetOrCreate(depthImageWidth, depthImageHeight, PixelFormat.Gray_16bpp); - Debug.Assert(infraredImage.Resource.Size == infraredBuffer.Length * sizeof(ushort), "InfraredImage size does not match raw infrared buffer size!"); + unsafe + { + fixed (ushort* p = infraredBuffer) + { + infraredImage.Resource.CopyFrom((IntPtr)p); + } + } - unsafe - { - fixed (ushort* p = infraredBuffer) + if (shouldOutputInfraredImage) { - infraredImage.Resource.CopyFrom((IntPtr)p); + this.InfraredImage.Post(infraredImage, originatingTime); } - } - this.InfraredImage.Post(infraredImage, originatingTime); + if (shouldOutputInfraredImageCameraView) + { + using var infraredImageCameraView = new ImageCameraView(infraredImage, this.GetCameraIntrinsics(), this.GetCameraPose()); + this.InfraredImageCameraView.Post(infraredImageCameraView, originatingTime); + } + } } } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/DepthCameraConfiguration.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/DepthCameraConfiguration.cs index 7c7431fcc..144ac0a91 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/DepthCameraConfiguration.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/DepthCameraConfiguration.cs @@ -9,42 +9,63 @@ namespace Microsoft.Psi.MixedReality /// /// Configuration for the component. /// - public class DepthCameraConfiguration + public class DepthCameraConfiguration : ResearchModeCameraConfiguration { /// - /// Gets or sets a value indicating whether the calibration settings are emitted. + /// Initializes a new instance of the class. /// - public bool OutputCalibration { get; set; } = true; + public DepthCameraConfiguration() + { + this.DepthSensorType = ResearchModeSensorType.DepthLongThrow; + } /// - /// Gets or sets a value indicating whether the original map of points for calibration are emitted. + /// Gets or sets a value indicating whether the component emits depth images. /// - public bool OutputCalibrationMap { get; set; } = true; + public bool OutputDepthImage { get; set; } = true; /// - /// Gets or sets the minimum interval between posting calibration map messages. + /// Gets or sets a value indicating whether the component emits depth image camera views. /// - public TimeSpan OutputCalibrationMapInterval { get; set; } = TimeSpan.FromSeconds(20); + public bool OutputDepthImageCameraView { get; set; } = true; /// - /// Gets or sets a value indicating whether the camera pose stream is emitted. + /// Gets or sets a value indicating whether the component emits infrared images. /// - public bool OutputPose { get; set; } = true; + public bool OutputInfraredImage { get; set; } = true; /// - /// Gets or sets a value indicating whether the depth stream is emitted. + /// Gets or sets a value indicating whether the component emits infrared image camera views. /// - public bool OutputDepth { get; set; } = true; + public bool OutputInfraredImageCameraView { get; set; } = true; /// - /// Gets or sets a value indicating whether the infrared stream is emitted. + /// Gets or sets the depth sensor type. /// - public bool OutputInfrared { get; set; } = true; + public ResearchModeSensorType DepthSensorType + { + get => this.SensorType; + set + { + if (value != ResearchModeSensorType.DepthLongThrow && value != ResearchModeSensorType.DepthAhat) + { + throw new ArgumentException($"{value} mode is not valid for {nameof(DepthCameraConfiguration)}.{nameof(this.DepthSensorType)}."); + } - /// - /// Gets or sets the sensor mode. - /// - /// Valid values are: DepthLongThrow or DepthAhat. - public ResearchModeSensorType Mode { get; set; } = ResearchModeSensorType.DepthLongThrow; + this.SensorType = value; + } + } + + /// + internal override bool RequiresCalibrationPointsMap() + => this.OutputCalibrationPointsMap || this.OutputCameraIntrinsics || this.OutputDepthImageCameraView || this.OutputInfraredImageCameraView; + + /// + internal override bool RequiresCameraIntrinsics() + => this.OutputCameraIntrinsics || this.OutputDepthImageCameraView || this.OutputInfraredImageCameraView; + + /// + internal override bool RequiresPose() + => this.OutputPose || this.OutputDepthImageCameraView || this.OutputInfraredImageCameraView; } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Gyroscope.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Gyroscope.cs index 3dc25c997..c57ab20f1 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Gyroscope.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Gyroscope.cs @@ -15,8 +15,9 @@ public class Gyroscope : ResearchModeImu /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - public Gyroscope(Pipeline pipeline) - : base(pipeline, ResearchModeSensorType.ImuGyro) + /// An optional name for the component. + public Gyroscope(Pipeline pipeline, string name = nameof(Gyroscope)) + : base(pipeline, ResearchModeSensorType.ImuGyro, name) { } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ImageToGzipStreamEncoder.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ImageToGzipStreamEncoder.cs index c18546c8c..ca19a9f9f 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ImageToGzipStreamEncoder.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ImageToGzipStreamEncoder.cs @@ -11,6 +11,9 @@ namespace Microsoft.Psi.Imaging /// public class ImageToGZipStreamEncoder : IImageToStreamEncoder { + /// + public string Description => "GZip"; + /// public void EncodeToStream(Image image, Stream stream) { diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ImageToJpegStreamEncoder.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ImageToJpegStreamEncoder.cs index 7963b27ab..9b49ac091 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ImageToJpegStreamEncoder.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ImageToJpegStreamEncoder.cs @@ -15,6 +15,7 @@ namespace Microsoft.Psi.MixedReality public class ImageToJpegStreamEncoder : IImageToStreamEncoder { private readonly BitmapPropertySet propertySet; + private readonly double imageQuality; /// /// Initializes a new instance of the class. @@ -22,10 +23,14 @@ public class ImageToJpegStreamEncoder : IImageToStreamEncoder /// Optional image quality (0.0 - 1.0, default 1.0). public ImageToJpegStreamEncoder(double imageQuality = 1.0) { - this.propertySet = new BitmapPropertySet(); + this.imageQuality = imageQuality; + this.propertySet = new (); this.propertySet.Add("ImageQuality", new BitmapTypedValue(imageQuality, Windows.Foundation.PropertyType.Single)); } + /// + public string Description => $"Jpeg({this.imageQuality:0.00})"; + /// public void EncodeToStream(Image image, Stream stream) { diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Magnetometer.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Magnetometer.cs index 6bc908c9c..c955b9407 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Magnetometer.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Magnetometer.cs @@ -15,8 +15,9 @@ public class Magnetometer : ResearchModeImu /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - public Magnetometer(Pipeline pipeline) - : base(pipeline, ResearchModeSensorType.ImuMag) + /// An optional name for the component. + public Magnetometer(Pipeline pipeline, string name = nameof(Magnetometer)) + : base(pipeline, ResearchModeSensorType.ImuMag, name) { } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Microsoft.Psi.MixedReality.UniversalWindows.csproj b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Microsoft.Psi.MixedReality.UniversalWindows.csproj index b6118e2dc..3dceced4f 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Microsoft.Psi.MixedReality.UniversalWindows.csproj +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Microsoft.Psi.MixedReality.UniversalWindows.csproj @@ -62,7 +62,9 @@ + + @@ -148,7 +150,6 @@ - diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/MixedReality.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/MixedReality.cs index 7b4c3e83b..7ed95089b 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/MixedReality.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/MixedReality.cs @@ -4,9 +4,7 @@ namespace Microsoft.Psi.MixedReality { using System; - using System.Numerics; using System.Threading.Tasks; - using MathNet.Spatial.Euclidean; using StereoKit; using Windows.Perception.Spatial; @@ -15,7 +13,7 @@ namespace Microsoft.Psi.MixedReality /// public static class MixedReality { - private const string WorldSpatialAnchorId = "_world"; + private const string DefaultWorldSpatialAnchorId = "_world"; /// /// Gets the world coordinate system. @@ -28,16 +26,18 @@ public static class MixedReality public static SpatialAnchorHelper SpatialAnchorHelper { get; private set; } /// - /// Initializes static members of the class. Attempts to initialize - /// the world coordinate system from a persisted spatial anchor. If one is not found, a stationary - /// frame of reference is created at the current location and its position is used as the world - /// coordinate system. + /// Initializes static members of the class. + /// Attempts to initialize the world coordinate system from a given spatial anchor. + /// If no spatial anchor is given, it attempts to load a default spatial anchor. + /// If the default spatial anchor is not found (e.g., if the app is being run for the first time), + /// a stationary frame of reference for the world is created at the current location. /// + /// A spatial anchor to use for the world (optional). /// A representing the asynchronous operation. /// /// This method should be called after SK.Initialize. /// - public static async Task InitializeAsync() + public static async Task InitializeAsync(SpatialAnchor worldSpatialAnchor = null) { if (!SK.IsInitialized) { @@ -47,7 +47,11 @@ public static async Task InitializeAsync() // Create the spatial anchor helper SpatialAnchorHelper = new SpatialAnchorHelper(await SpatialAnchorManager.RequestStoreAsync()); - InitializeWorldCoordinateSystem(); + InitializeWorldCoordinateSystem(worldSpatialAnchor); + + // By default, don't render the hands or register with StereoKit physics system. + Input.HandVisible(Handed.Max, false); + Input.HandSolid(Handed.Max, false); } /// @@ -55,11 +59,12 @@ public static async Task InitializeAsync() /// or creates it at a stationary frame of reference if it does not exist. Once initialized, the /// world coordinate system will be consistent across application sessions, unless the associated /// spatial anchor is modified or deleted. + /// A spatial anchor to use for the world (may be null). /// - private static void InitializeWorldCoordinateSystem() + private static void InitializeWorldCoordinateSystem(SpatialAnchor worldSpatialAnchor) { - // Try to get a previously saved world spatial anchor - var worldSpatialAnchor = SpatialAnchorHelper.TryGetSpatialAnchor(WorldSpatialAnchorId); + // If no world anchor was given, try to load the default world spatial anchor if it was previously persisted + worldSpatialAnchor ??= SpatialAnchorHelper.TryGetSpatialAnchor(DefaultWorldSpatialAnchorId); if (worldSpatialAnchor != null) { @@ -68,6 +73,7 @@ private static void InitializeWorldCoordinateSystem() } else { + // Generate and persist the default world spatial anchor var locator = SpatialLocator.GetDefault(); if (locator != null) @@ -80,7 +86,7 @@ private static void InitializeWorldCoordinateSystem() // Create a spatial anchor to represent the world origin and persist it to the spatial // anchor store to ensure that the origin remains coherent between sessions. - worldSpatialAnchor = SpatialAnchorHelper.TryCreateSpatialAnchor(WorldSpatialAnchorId, WorldSpatialCoordinateSystem); + worldSpatialAnchor = SpatialAnchorHelper.TryCreateSpatialAnchor(DefaultWorldSpatialAnchorId, WorldSpatialCoordinateSystem); if (worldSpatialAnchor == null) { @@ -99,14 +105,15 @@ private static void InitializeWorldCoordinateSystem() // These transforms will allow us to convert world coordinates to/from StereoKit coordinates where needed: // on input from StereoKit -> \psi, and on output (rendering) \psi -> StereoKit - // The pose of world anchor is essentially the inverse of the startup pose of StereoKit with respect to the world. - Matrix4x4 worldStereoKitMatrix = World.FromPerceptionAnchor(worldSpatialAnchor).ToMatrix(); - StereoKitTransforms.StereoKitStartingPoseInverse = new CoordinateSystem(worldStereoKitMatrix.ToMathNetMatrix().ChangeBasisHoloLensToPsi()); + // Query the pose of the world anchor. We use this pose for rendering correctly in the world, + // and for transforming from world coordinates to StereoKit coordinates. + StereoKitTransforms.WorldHierarchy = World.FromPerceptionAnchor(worldSpatialAnchor).ToMatrix(); + StereoKitTransforms.WorldToStereoKit = StereoKitTransforms.WorldHierarchy.ToCoordinateSystem(); - // Inverting then gives us the starting pose of StereoKit in the "world" (relative to the world anchor). - StereoKitTransforms.StereoKitStartingPose = StereoKitTransforms.StereoKitStartingPoseInverse.Invert(); + // Inverting gives us a coordinate system that can be used for transforming from StereoKit to world coordinates. + StereoKitTransforms.StereoKitToWorld = StereoKitTransforms.WorldToStereoKit.Invert(); - System.Diagnostics.Trace.WriteLine($"StereoKit origin: {StereoKitTransforms.StereoKitStartingPose.Origin.X},{StereoKitTransforms.StereoKitStartingPose.Origin.Y},{StereoKitTransforms.StereoKitStartingPose.Origin.Z}"); + System.Diagnostics.Trace.WriteLine($"StereoKit origin: {StereoKitTransforms.StereoKitToWorld.Origin.X},{StereoKitTransforms.StereoKitToWorld.Origin.Y},{StereoKitTransforms.StereoKitToWorld.Origin.Z}"); // TODO: It would be nice if we could actually just shift the origin coordinate system in StereoKit // to the pose currently defined in StereoKitTransforms.WorldPose. diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Operators.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Operators.cs index dbbcfaf85..a002a7af8 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Operators.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Operators.cs @@ -3,10 +3,19 @@ namespace Microsoft.Psi.MixedReality { + using System; + using System.Collections.Generic; + using System.IO; using System.Numerics; + using System.Threading.Tasks; + using System.Xml.Serialization; + using MathNet.Numerics.LinearAlgebra; using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; using Windows.Perception.Spatial; + using Windows.Storage; using Quaternion = System.Numerics.Quaternion; + using VectorDouble = MathNet.Numerics.LinearAlgebra.Vector; /// /// Implements operators. @@ -21,7 +30,7 @@ public static partial class Operators public static CoordinateSystem TryConvertSpatialCoordinateSystemToPsiCoordinateSystem(this SpatialCoordinateSystem spatialCoordinateSystem) { var worldPose = spatialCoordinateSystem.TryGetTransformTo(MixedReality.WorldSpatialCoordinateSystem); - return worldPose.HasValue ? new CoordinateSystem(worldPose.Value.ToMathNetMatrix().ChangeBasisHoloLensToPsi()) : null; + return worldPose.HasValue ? new CoordinateSystem(worldPose.Value.ToMathNetMatrix()) : null; } /// @@ -31,12 +40,168 @@ public static CoordinateSystem TryConvertSpatialCoordinateSystemToPsiCoordinateS /// The . public static SpatialCoordinateSystem TryConvertPsiCoordinateSystemToSpatialCoordinateSystem(this CoordinateSystem coordinateSystem) { - var holoLensMatrix = coordinateSystem.ChangeBasisPsiToHoloLens().ToSystemNumericsMatrix(); + var holoLensMatrix = coordinateSystem.ToHoloLensSystemMatrix(); var translation = holoLensMatrix.Translation; holoLensMatrix.Translation = Vector3.Zero; var rotation = Quaternion.CreateFromRotationMatrix(holoLensMatrix); var spatialAnchor = SpatialAnchor.TryCreateRelativeTo(MixedReality.WorldSpatialCoordinateSystem, translation, rotation); return spatialAnchor?.CoordinateSystem; } + + /// + /// Computes camera intrinsics from a map of calibration points. + /// + /// The map of calibration points to compute camera intrinsics from. + /// The computed . + internal static CameraIntrinsics ComputeCameraIntrinsics(this CalibrationPointsMap calibrationPointsMap) + { + var width = calibrationPointsMap.Width; + var height = calibrationPointsMap.Height; + + // Convert unit plane points in the calibration map to a lookup table mapping image points to 3D points in camera space. + List cameraPoints = new (); + List imagePoints = new (); + int ci = 0; + for (int j = 0; j < height; j++) + { + for (int i = 0; i < width; i++) + { + var x = calibrationPointsMap.CameraUnitPlanePoints[ci++]; + var y = calibrationPointsMap.CameraUnitPlanePoints[ci++]; + + if (!double.IsNaN(x) && !double.IsNaN(y)) + { + var norm = Math.Sqrt((x * x) + (y * y) + 1.0); + imagePoints.Add(new Point2D(i + 0.5, j + 0.5)); + cameraPoints.Add(new Point3D(x / norm, y / norm, 1.0 / norm)); + } + } + } + + // Initialize a starting camera matrix + var initialCameraMatrix = Matrix.Build.Dense(3, 3); + var initialDistortion = VectorDouble.Build.Dense(2); + initialCameraMatrix[0, 0] = 250; // fx + initialCameraMatrix[1, 1] = 250; // fy + initialCameraMatrix[0, 2] = width / 2.0; // cx + initialCameraMatrix[1, 2] = height / 2.0; // cy + initialCameraMatrix[2, 2] = 1; + CalibrationExtensions.CalibrateCameraIntrinsics( + cameraPoints, + imagePoints, + initialCameraMatrix, + initialDistortion, + out var computedCameraMatrix, + out var computedDistortionCoefficients, + false); + + return new CameraIntrinsics(width, height, computedCameraMatrix, computedDistortionCoefficients); + } + + /// + /// Serialize camera intrinsics to a file on the device. + /// + /// The camera intrinsics to serialize. + /// The device file to serialize to. + /// A representing the result of the asynchronous operation. + internal static async Task SerializeAsync(this CameraIntrinsics cameraIntrinsics, StorageFile writeFile) + { + SerializedCameraIntrinsics serializedCameraIntrinsics = new () + { + ImageWidth = cameraIntrinsics.ImageWidth, + ImageHeight = cameraIntrinsics.ImageHeight, + Transform = cameraIntrinsics.Transform.AsColumnMajorArray(), + RadialDistortion = cameraIntrinsics.RadialDistortion.ToArray(), + TangentialDistortion = cameraIntrinsics.TangentialDistortion.ToArray(), + ClosedFormDistorts = cameraIntrinsics.ClosedFormDistorts, + }; + + using var stream = await writeFile.OpenStreamForWriteAsync(); + var serializer = new XmlSerializer(typeof(SerializedCameraIntrinsics)); + serializer.Serialize(stream, serializedCameraIntrinsics); + + return true; + } + + /// + /// Serialize calibration points map to a file on the device. + /// + /// The calibration points map to serialize. + /// The device file to serialize to. + /// A representing the result of the asynchronous operation. + internal static async Task SerializeAsync(this CalibrationPointsMap calibrationPointsMap, StorageFile writeFile) + { + using var stream = await writeFile.OpenStreamForWriteAsync(); + using var writer = new BinaryWriter(stream); + writer.Write(calibrationPointsMap.Width); + writer.Write(calibrationPointsMap.Height); + writer.Write(calibrationPointsMap.CameraUnitPlanePoints.Length); + foreach (var value in calibrationPointsMap.CameraUnitPlanePoints) + { + writer.Write(value); + } + + return true; + } + + /// + /// Reads camera intrinsics from a file on the device. + /// + /// The file with serialized camera intrinsics. + /// A result with the deserialized camera intrinsics. + internal static async Task DeserializeCameraIntrinsicsAsync(this StorageFile file) + { + var serializer = new XmlSerializer(typeof(SerializedCameraIntrinsics)); + using var stream = await file.OpenStreamForReadAsync(); + var data = serializer.Deserialize(stream) as SerializedCameraIntrinsics; + + // Parse out and return + var transform = Matrix.Build.DenseOfColumnMajor(3, 3, data.Transform); + VectorDouble radialDistortion = null; + if (data.RadialDistortion is not null) + { + radialDistortion = VectorDouble.Build.DenseOfArray(data.RadialDistortion); + } + + VectorDouble tangentialDistortion = null; + if (data.TangentialDistortion is not null) + { + tangentialDistortion = VectorDouble.Build.DenseOfArray(data.TangentialDistortion); + } + + return new ( + data.ImageWidth, + data.ImageHeight, + transform, + radialDistortion, + tangentialDistortion, + data.ClosedFormDistorts); + } + + /// + /// Reads calibration points map from a file on the device. + /// + /// The file with serialized calibration points map. + /// A result with the deserialized calibration points map. + internal static async Task DeserializeCalibrationPointsMapAsync(this StorageFile file) + { + using var stream = await file.OpenStreamForReadAsync(); + using var reader = new BinaryReader(stream); + var width = reader.ReadInt32(); + var height = reader.ReadInt32(); + var len = reader.ReadInt32(); + double[] cameraUnitPlanePoints = new double[len]; + for (var i = 0; i < len; i++) + { + cameraUnitPlanePoints[i] = reader.ReadDouble(); + } + + return new () + { + Width = width, + Height = height, + CameraUnitPlanePoints = cameraUnitPlanePoints, + }; + } } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/PhotoVideoCamera.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/PhotoVideoCamera.cs index ed65f8d27..30cd3b2d3 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/PhotoVideoCamera.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/PhotoVideoCamera.cs @@ -14,6 +14,7 @@ namespace Microsoft.Psi.MixedReality using Microsoft.Psi.Calibration; using Microsoft.Psi.Components; using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; using Windows.Foundation; using Windows.Graphics.Imaging; using Windows.Media.Capture; @@ -26,6 +27,7 @@ public class PhotoVideoCamera : ISourceComponent, IDisposable { private readonly PhotoVideoCameraConfiguration configuration; private readonly Pipeline pipeline; + private readonly string name; private readonly Task initMediaCaptureTask; private MediaCapture mediaCapture; @@ -39,16 +41,21 @@ public class PhotoVideoCamera : ISourceComponent, IDisposable /// /// The pipeline to add the component to. /// The configuration for this component. - public PhotoVideoCamera(Pipeline pipeline, PhotoVideoCameraConfiguration configuration = null) + /// An optional name for the component. + public PhotoVideoCamera(Pipeline pipeline, PhotoVideoCameraConfiguration configuration = null, string name = nameof(PhotoVideoCamera)) { + this.name = name; this.pipeline = pipeline; this.configuration = configuration ?? new PhotoVideoCameraConfiguration(); - this.VideoImage = pipeline.CreateEmitter>(this, nameof(this.VideoImage)); - this.PreviewImage = pipeline.CreateEmitter>(this, nameof(this.PreviewImage)); + + this.VideoEncodedImage = pipeline.CreateEmitter>(this, nameof(this.VideoEncodedImage)); this.VideoIntrinsics = pipeline.CreateEmitter(this, nameof(this.VideoIntrinsics)); this.VideoPose = pipeline.CreateEmitter(this, nameof(this.VideoPose)); + this.VideoEncodedImageCameraView = pipeline.CreateEmitter(this, nameof(this.VideoEncodedImageCameraView)); + this.PreviewEncodedImage = pipeline.CreateEmitter>(this, nameof(this.PreviewEncodedImage)); this.PreviewIntrinsics = pipeline.CreateEmitter(this, nameof(this.PreviewIntrinsics)); this.PreviewPose = pipeline.CreateEmitter(this, nameof(this.PreviewPose)); + this.PreviewEncodedImageCameraView = pipeline.CreateEmitter(this, nameof(this.PreviewEncodedImageCameraView)); // Call this here (rather than in the Start() method, which is executed on the thread pool) to // ensure that MediaCapture.InitializeAsync() is called from an STA thread (this constructor must @@ -59,9 +66,9 @@ public PhotoVideoCamera(Pipeline pipeline, PhotoVideoCameraConfiguration configu } /// - /// Gets the video image stream. + /// Gets the original video NV12-encoded image stream. /// - public Emitter> VideoImage { get; } + public Emitter> VideoEncodedImage { get; } /// /// Gets the video camera pose stream. @@ -74,9 +81,14 @@ public PhotoVideoCamera(Pipeline pipeline, PhotoVideoCameraConfiguration configu public Emitter VideoIntrinsics { get; } /// - /// Gets the preview image stream. + /// Gets the original video NV12-encoded image camera view. + /// + public Emitter VideoEncodedImageCameraView { get; } + + /// + /// Gets the original preview NV12-encoded image stream. /// - public Emitter> PreviewImage { get; } + public Emitter> PreviewEncodedImage { get; } /// /// Gets the preview camera pose stream. @@ -88,6 +100,11 @@ public PhotoVideoCamera(Pipeline pipeline, PhotoVideoCameraConfiguration configu /// public Emitter PreviewIntrinsics { get; } + /// + /// Gets the original preview NV12-encoded image camera view. + /// + public Emitter PreviewEncodedImageCameraView { get; } + /// public void Dispose() { @@ -128,9 +145,11 @@ public async void Start(Action notifyCompletionTime) // whenever a new Video frame is available. The frame image, pose and intrinsics // (if configured) are then posted on the respective output emitters. this.videoFrameHandler = this.CreateMediaFrameHandler( - this.VideoImage, - this.configuration.VideoStreamSettings.OutputIntrinsics ? this.VideoIntrinsics : null, - this.configuration.VideoStreamSettings.OutputPose ? this.VideoPose : null); + this.configuration.VideoStreamSettings, + this.VideoEncodedImage, + this.VideoIntrinsics, + this.VideoPose, + this.VideoEncodedImageCameraView); this.videoFrameReader.FrameArrived += this.videoFrameHandler; } @@ -156,9 +175,11 @@ public async void Start(Action notifyCompletionTime) // whenever a new Preview frame is available. The frame image, pose and intrinsics // (if configured) are then posted on the respective output emitters. this.previewFrameHandler = this.CreateMediaFrameHandler( - this.PreviewImage, - this.configuration.PreviewStreamSettings.OutputIntrinsics ? this.PreviewIntrinsics : null, - this.configuration.PreviewStreamSettings.OutputPose ? this.PreviewPose : null); + this.configuration.PreviewStreamSettings, + this.PreviewEncodedImage, + this.PreviewIntrinsics, + this.PreviewPose, + this.PreviewEncodedImageCameraView); this.previewFrameReader.FrameArrived += this.previewFrameHandler; } @@ -188,6 +209,9 @@ public async void Stop(DateTime finalOriginatingTime, Action notifyCompleted) notifyCompleted(); } + /// + public override string ToString() => this.name; + /// /// Initializes the MediaCapture object and creates the MediaFrameReaders for the configured capture streams. /// @@ -444,14 +468,18 @@ private async Task CreateMediaFrameReaderAsync(MediaFrameSourc /// /// Creates an event handler that handles the FrameArrived event of the MediaFrameReader. /// - /// The stream on which to post the output image. + /// The stream settings. + /// The stream on which to post the output encoded image. /// The stream on which to post the camera intrinsics. /// The stream on which to post the camera pose. + /// The stream on which to post the encoded image camera view. /// The event handler. private TypedEventHandler CreateMediaFrameHandler( - Emitter> imageStream, - Emitter intrinsicsStream = null, - Emitter poseStream = null) + PhotoVideoCameraConfiguration.StreamSettings streamSettings, + Emitter> encodedImageStream, + Emitter intrinsicsStream, + Emitter poseStream, + Emitter encodedImageCameraViewStream) { // Cache the intrinsics ICameraIntrinsics cameraIntrinsics = null; @@ -465,41 +493,70 @@ private async Task CreateMediaFrameReaderAsync(MediaFrameSourc var frameTimestamp = frame.SystemRelativeTime.Value.Ticks; var originatingTime = this.pipeline.GetCurrentTimeFromElapsedTicks(frameTimestamp); - // Post the camera intrinsics if requested - if (intrinsicsStream != null) + // Compute the camera intrinsics if needed + if (cameraIntrinsics == null && streamSettings.OutputCameraIntrinsics) + { + cameraIntrinsics = this.GetCameraIntrinsics(frame); + } + + // Post the intrinsics + if (streamSettings.OutputCameraIntrinsics) { - cameraIntrinsics ??= this.GetCameraIntrinsics(frame); intrinsicsStream.Post(cameraIntrinsics, originatingTime); } - // Post the camera pose if requested - if (poseStream != null) + // Compute the camera pose if needed + var cameraPose = default(CoordinateSystem); + if (streamSettings.OutputPose) { // Convert the frame coordinate system to world pose in psi basis - var worldPose = frame.CoordinateSystem?.TryConvertSpatialCoordinateSystemToPsiCoordinateSystem(); - poseStream.Post(worldPose, originatingTime); + cameraPose = frame.CoordinateSystem?.TryConvertSpatialCoordinateSystemToPsiCoordinateSystem(); } - // Accessing the VideoMediaFrame.SoftwareBitmap property creates a strong reference which needs to be Disposed, per the remarks here: - // https://docs.microsoft.com/en-us/uwp/api/windows.media.capture.frames.mediaframereference?view=winrt-19041#remarks - using var frameBitmap = frame.VideoMediaFrame.SoftwareBitmap; - - // Convert from NV12 to BGRA32 - using var softwareBitmap = SoftwareBitmap.Convert(frameBitmap, BitmapPixelFormat.Bgra8); + // Post the pose + if (streamSettings.OutputPose) + { + poseStream.Post(cameraPose, originatingTime); + } - // Copy bitmap data into a Shared - unsafe + if (streamSettings.OutputEncodedImage || streamSettings.OutputEncodedImageCameraView) { - using var sharedImage = ImagePool.GetOrCreate(softwareBitmap.PixelWidth, softwareBitmap.PixelHeight, PixelFormat.BGRA_32bpp); - using var input = softwareBitmap.LockBuffer(BitmapBufferAccessMode.Read); - using var inputReference = input.CreateReference(); - ((UnsafeNative.IMemoryBufferByteAccess)inputReference).GetBuffer(out byte* imageData, out uint size); + // Accessing the VideoMediaFrame.SoftwareBitmap property creates a strong reference + // which needs to be Disposed, per the remarks here: + // https://docs.microsoft.com/en-us/uwp/api/windows.media.capture.frames.mediaframereference?view=winrt-19041#remarks + using var frameBitmap = frame.VideoMediaFrame.SoftwareBitmap; + using var sharedEncodedImage = EncodedImagePool.GetOrCreate(frameBitmap.PixelWidth, frameBitmap.PixelHeight, PixelFormat.BGRA_32bpp); + + // Copy bitmap data into the shared encoded image + unsafe + { + using var input = frameBitmap.LockBuffer(BitmapBufferAccessMode.Read); + using var inputReference = input.CreateReference(); + ((UnsafeNative.IMemoryBufferByteAccess)inputReference).GetBuffer(out byte* imageData, out uint size); + + // Copy NV12-encoded bytes directly (leaving room for 4-byte header) + sharedEncodedImage.Resource.CopyFrom((IntPtr)imageData, 4, (int)size); + + // Add NV12 header to identify encoding + var buffer = sharedEncodedImage.Resource.GetBuffer(); + buffer[0] = (byte)'N'; + buffer[1] = (byte)'V'; + buffer[2] = (byte)'1'; + buffer[3] = (byte)'2'; + } - // Debug.Assert(size == sharedImage.Resource.Size); - sharedImage.Resource.CopyFrom((IntPtr)imageData); + // Post encoded image stream + if (streamSettings.OutputEncodedImage) + { + encodedImageStream.Post(sharedEncodedImage, originatingTime); + } - // Post image stream - imageStream.Post(sharedImage, originatingTime); + // Post the encoded image camera view stream if requested + if (streamSettings.OutputEncodedImageCameraView) + { + using var encodedImageCameraView = new EncodedImageCameraView(sharedEncodedImage, cameraIntrinsics, cameraPose); + encodedImageCameraViewStream.Post(encodedImageCameraView, originatingTime); + } } } }; diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/PhotoVideoCameraConfiguration.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/PhotoVideoCameraConfiguration.cs index 9997764ad..e1d9fe86f 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/PhotoVideoCameraConfiguration.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/PhotoVideoCameraConfiguration.cs @@ -9,12 +9,12 @@ namespace Microsoft.Psi.MixedReality public class PhotoVideoCameraConfiguration { /// - /// Gets or sets the settings for the stream, or null to omit. + /// Gets or sets the settings for the stream, or null to omit. /// public StreamSettings VideoStreamSettings { get; set; } = new (); // use defaults /// - /// Gets or sets the settings for the stream, or null to omit. + /// Gets or sets the settings for the stream, or null to omit. /// public StreamSettings PreviewStreamSettings { get; set; } = null; @@ -22,25 +22,54 @@ public class PhotoVideoCameraConfiguration /// Defines the capture settings for the Video or Preview streams. /// /// - /// Valid capture profiles for HoloLens2 are as follows. + /// If capturing both the Video and Preview streams, the requested resolutions must both + /// be supported by the same profile. Valid capture profiles for HoloLens2 are as follows. /// + /// Profile 6B52B017-42C7-4A21-BFE3-23F009149887: /// 2272x1278 (15,30fps, Video, Preview) + /// 896x504 (15,30fps, Video, Preview) + /// Profile 6B52B017-42C7-4A21-BFE3-23F009149887: + /// 2284x1284 (24fps, Video only) + /// 2284x1284 (15,30fps, Video, Preview) + /// 1522x856 (24fps, Video only) + /// 1522x856 (15,30fps, Video, Preview) + /// Profile 6B52B017-42C7-4A21-BFE3-23F009149887: /// 1952x1100 (15,30fps, Video, Preview) /// 1920x1080 (15,30fps, Video, Preview) + /// 1504x846 (5fps, Video only) /// 1504x846 (15,30fps, Video, Preview) /// 1280x720 (15,30fps, Video, Preview) /// 1128x636 (15,30fps, Video only) /// 960x540 (15,30fps, Video only) + /// 760x428 (15,30fps, Video only) + /// 640x360 (15,30fps, Video only) + /// 500x282 (15,30fps, Video only) + /// 424x240 (15,30fps, Video only) + /// Profile B4894D81-62B7-4EEC-8740-80658C4A9D3E: + /// 2272x1278 (15,30fps, Video, Preview) /// 896x504 (15,30fps, Video, Preview) + /// Profile C5444A88-E1BF-4597-B2DD-9E1EAD864BB8: + /// 1952x1100 (60fps, Video only) + /// 1952x1100 (15,30fps, Video, Preview) + /// 1920x1080 (15,30fps, Video, Preview) + /// 1504x846 (5,60fps, Video only) + /// 1504x846 (15,30fps, Video, Preview) + /// 1280x720 (15,30fps, Video, Preview) + /// 1128x636 (15,30fps, Video only) + /// 960x540 (15,30fps, Video only) /// 760x428 (15,30fps, Video only) /// 640x360 (15,30fps, Video only) /// 500x282 (15,30fps, Video only) /// 424x240 (15,30fps, Video only) + /// Profile CDF68AD0-7B8D-4DE3-BB64-AC1CE06DA333: + /// 2284x1284 (24fps, Video only) + /// 2284x1284 (15,30fps, Video, Preview) + /// 1522x856 (24fps, Video only) + /// 1522x856 (15,30fps, Video, Preview) /// /// For more info, /// see https://docs.microsoft.com/en-us/windows/mixed-reality/develop/platform-capabilities-and-apis/locatable-camera#hololens-2. /// - /// If capturing both Video and Preview streams, the selected capture settings must be supported in the same camera profile. /// Each stream represents a virtual camera in the camera profile and therefore each has its own Instrinsics and Pose streams. /// For the HoloLens, since the Video and Preview streams both ultimately originate from the PV camera, the data on the Pose /// streams will be identical, representing the PV camera pose. It is therefore only necessary to capture one of the Pose @@ -72,16 +101,26 @@ public StreamSettings() /// public int ImageHeight { get; set; } = 720; + /// + /// Gets or sets a value indicating whether the original NV12-encoded image is emitted. + /// + public bool OutputEncodedImage { get; set; } = true; + /// /// Gets or sets a value indicating whether the camera intrinsics are emitted. /// - public bool OutputIntrinsics { get; set; } = true; + public bool OutputCameraIntrinsics { get; set; } = true; /// /// Gets or sets a value indicating whether the camera pose is emitted. /// public bool OutputPose { get; set; } = true; + /// + /// Gets or sets a value indicating whether the original NV12-encoded camera view is emitted. + /// + public bool OutputEncodedImageCameraView { get; set; } = true; + /// /// Gets or sets the settings for mixed reality capture, or null to omit holograms on this stream. /// diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Properties/AssemblyInfo.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Properties/AssemblyInfo.cs index 3fab76572..43419d2e7 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Properties/AssemblyInfo.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/Properties/AssemblyInfo.cs @@ -13,7 +13,7 @@ [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("Microsoft Corporation")] [assembly: AssemblyProduct("Microsoft.Psi.MixedReality.UniversalWindows")] -[assembly: AssemblyCopyright("© Microsoft Corporation. All rights reserved.")] +[assembly: AssemblyCopyright("© Microsoft Corporation. All rights reserved.")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] @@ -27,7 +27,7 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("0.16.92.1")] -[assembly: AssemblyFileVersion("0.16.92.1")] -[assembly: AssemblyInformationalVersion("0.16.92.1-beta")] +[assembly: AssemblyVersion("0.17.52.1")] +[assembly: AssemblyFileVersion("0.17.52.1")] +[assembly: AssemblyInformationalVersion("0.17.52.1-beta")] [assembly: ComVisible(false)] diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ResearchModeCamera.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ResearchModeCamera.cs index 6e3df855d..52bf87a69 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ResearchModeCamera.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ResearchModeCamera.cs @@ -4,7 +4,7 @@ namespace Microsoft.Psi.MixedReality { using System; - using System.Collections.Generic; + using System.IO; using System.Numerics; using System.Threading; using System.Threading.Tasks; @@ -14,30 +14,39 @@ namespace Microsoft.Psi.MixedReality using Microsoft.Psi.Calibration; using Microsoft.Psi.Components; using Windows.Foundation; + using Windows.Perception; using Windows.Perception.Spatial; using Windows.Perception.Spatial.Preview; + using Windows.Storage; /// /// Represents an abstract base class for a HoloLens 2 research mode camera component. /// public abstract class ResearchModeCamera : ISourceComponent { - // Camera coordinate system (x - right, y - down, z - forward) relative - // to the HoloLens coordinate system (x - right, y - up, z - back) - private static readonly CoordinateSystem CameraCoordinateSystem = - new (default, UnitVector3D.XAxis, UnitVector3D.YAxis.Negate(), UnitVector3D.ZAxis.Negate()); +#pragma warning disable SA1117 // Parameters should be on same line or separate lines + // Camera basis (x - right, y - down, z - forward) relative + // to the HoloLens basis (x - right, y - up, z - back) + private static readonly Matrix4x4 CameraBasis = new ( + 1, 0, 0, 0, + 0, -1, 0, 0, + 0, 0, -1, 0, + 0, 0, 0, 1); +#pragma warning restore SA1117 // Parameters should be on same line or separate lines private readonly Pipeline pipeline; + private readonly ResearchModeCameraConfiguration configuration; + private readonly string name; private readonly ResearchModeSensorDevice sensorDevice; private readonly ResearchModeCameraSensor cameraSensor; private readonly Task requestCameraAccessTask; private readonly SpatialLocator rigNodeLocator; - private readonly bool createCalibrationMap; - private readonly bool computeCameraIntrinsics; - private CalibrationPointsMap calibrationPointsMap; - private ICameraIntrinsics cameraIntrinsics; - private CoordinateSystem cameraExtrinsics; + private bool pipelineIsRunning = false; + private CameraIntrinsics cameraIntrinsics = null; + private CoordinateSystem cameraPose = null; + private CalibrationPointsMap calibrationPointsMap = null; + private Matrix4x4? invertedCameraExtrinsics = null; private Thread captureThread; private bool shutdown; @@ -50,30 +59,22 @@ public abstract class ResearchModeCamera : ISourceComponent /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - /// The research mode sensor type. - /// A value indicating whether to create a map of calibration points (needed to compute intrinsics). - /// A value indicating whether to compute camera intrinsics. - public ResearchModeCamera(Pipeline pipeline, ResearchModeSensorType sensorType, bool createCalibrationMap = true, bool computeCameraIntrinsics = true) + /// The research mode camera configuration. + /// An optional name for the component. + protected ResearchModeCamera(Pipeline pipeline, ResearchModeCameraConfiguration configuration, string name = nameof(ResearchModeCamera)) { this.pipeline = pipeline; - this.createCalibrationMap = createCalibrationMap; - this.computeCameraIntrinsics = computeCameraIntrinsics; + this.configuration = configuration; + this.name = name; + this.pipeline.PipelineRun += (_, _) => this.pipelineIsRunning = true; this.Pose = pipeline.CreateEmitter(this, nameof(this.Pose)); - - if (this.computeCameraIntrinsics) - { - this.CameraIntrinsics = pipeline.CreateEmitter(this, nameof(this.CameraIntrinsics)); - } - - if (this.createCalibrationMap) - { - this.CalibrationPointsMap = pipeline.CreateEmitter(this, nameof(this.CalibrationPointsMap)); - } + this.CameraIntrinsics = pipeline.CreateEmitter(this, nameof(this.CameraIntrinsics)); + this.CalibrationPointsMap = pipeline.CreateEmitter(this, nameof(this.CalibrationPointsMap)); this.sensorDevice = new ResearchModeSensorDevice(); this.requestCameraAccessTask = this.sensorDevice.RequestCameraAccessAsync().AsTask(); - this.cameraSensor = (ResearchModeCameraSensor)this.sensorDevice.GetSensor(sensorType); + this.cameraSensor = (ResearchModeCameraSensor)this.sensorDevice.GetSensor(this.configuration.SensorType); Guid rigNodeGuid = this.sensorDevice.GetRigNodeId(); this.rigNodeLocator = SpatialGraphInteropPreview.CreateLocatorForNode(rigNodeGuid); @@ -92,12 +93,12 @@ public ResearchModeCamera(Pipeline pipeline, ResearchModeSensorType sensorType, /// /// Gets the camera intrinsics stream. /// - public Emitter CameraIntrinsics { get; } + public Emitter CameraIntrinsics { get; } = null; /// /// Gets the stream for calibration map (image points and corresponding 3D camera points). /// - public Emitter CalibrationPointsMap { get; } + public Emitter CalibrationPointsMap { get; } = null; /// /// Gets the stream on which the count of out of order frames are posted. @@ -108,11 +109,92 @@ public ResearchModeCamera(Pipeline pipeline, ResearchModeSensorType sensorType, public Emitter DebugOutOfOrderFrames { get; } = null; // DEBUG builds only #endif + /// + /// Gets the research mode camera configuration. + /// + protected ResearchModeCameraConfiguration Configuration => this.configuration; + /// /// Gets the rig node locator. /// protected SpatialLocator RigNodeLocator => this.rigNodeLocator; + /// + /// Calibrates the camera sensor. + /// + public void Calibrate() + { + if (this.pipelineIsRunning) + { + throw new InvalidOperationException($"The {nameof(this.Calibrate)}() method should only be called before the pipeline has started running."); + } + + this.calibrationPointsMap = this.ComputeCalibrationPointsMap(); + this.cameraIntrinsics = this.calibrationPointsMap.ComputeCameraIntrinsics(); + } + + /// + /// Calibrates the camera sensor, using stored files. + /// + /// The device folder containing calibration files. Defaults to Documents. + /// True when complete. + /// + /// Camera intrinsics are loaded from the file: {calibrationFolder}/{ResearchModeSensorType}Intrinsics.xml, + /// and the map of calibration points are loaded from the file: {calibrationFolder}/{ResearchModeSensorType}Points.map. + /// Follow these steps to give your app read/write access to the Documents folder: + /// https://docs.microsoft.com/en-us/uwp/api/windows.storage.knownfolders.documentslibrary?view=winrt-22000#prerequisites + /// If the files do not already exist, they will be created, and calibration will be computed from scratch and saved. + /// + public async Task CalibrateFromFileAsync(StorageFolder calibrationFolder = null) + { + if (this.pipelineIsRunning) + { + throw new InvalidOperationException($"The {nameof(this.CalibrateFromFileAsync)}() method should only be called before the pipeline has started running."); + } + + // Use the Documents folder by default. + calibrationFolder ??= KnownFolders.DocumentsLibrary; + + // Attempt to load from file + var sensorType = this.cameraSensor.GetSensorType(); + string calibrationPointsMapFileName = $"{sensorType}Points.map"; + string cameraIntrinsicsFileName = $"{sensorType}Intrinsics.xml"; + + try + { + // Get the file (throws FileNotFoundException if it doesn't exist, handled below). + var calibrationPointsMapFile = await calibrationFolder.GetFileAsync(calibrationPointsMapFileName); + this.calibrationPointsMap = await calibrationPointsMapFile.DeserializeCalibrationPointsMapAsync(); + } + catch (FileNotFoundException) + { + // Create the file if it didn't exist. + var calibrationPointsMapFile = await calibrationFolder.CreateFileAsync(calibrationPointsMapFileName, CreationCollisionOption.FailIfExists); + + // Compute and serialize + this.calibrationPointsMap = this.ComputeCalibrationPointsMap(); + await this.calibrationPointsMap.SerializeAsync(calibrationPointsMapFile); + } + + try + { + // Get the file (throws FileNotFoundException if it doesn't exist, handled below). + var cameraIntrinsicsFile = await calibrationFolder.GetFileAsync(cameraIntrinsicsFileName); + this.cameraIntrinsics = await cameraIntrinsicsFile.DeserializeCameraIntrinsicsAsync(); + } + catch (FileNotFoundException) + { + // Create the file if it didn't exist. + var cameraIntrinsicsFile = await calibrationFolder.CreateFileAsync(cameraIntrinsicsFileName, CreationCollisionOption.FailIfExists); + + // Compute and serialize + this.cameraIntrinsics = this.calibrationPointsMap.ComputeCameraIntrinsics(); + await this.cameraIntrinsics.SerializeAsync(cameraIntrinsicsFile); + } + + return true; + } + /// public void Start(Action notifyCompletionTime) { @@ -135,6 +217,9 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) notifyCompleted(); } + /// + public override string ToString() => this.name; + /// /// Processes a sensor frame received from the sensor. /// @@ -148,7 +233,7 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) /// Gets the camera intrinsics. /// /// The camera's intrinsics. - protected ICameraIntrinsics GetCameraIntrinsics() => this.cameraIntrinsics; + protected CameraIntrinsics GetCameraIntrinsics() => this.cameraIntrinsics; /// /// Gets the calibration points map (used for computing intrinsics)). @@ -156,6 +241,12 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) /// The calibration points map. protected CalibrationPointsMap GetCalibrationPointsMap() => this.calibrationPointsMap; + /// + /// Gets the camera pose. + /// + /// The camera's pose. + protected CoordinateSystem GetCameraPose() => this.cameraPose; + /// /// Converts the rig node location to the camera pose. /// @@ -169,13 +260,17 @@ protected CoordinateSystem ToCameraPose(SpatialLocation rigNodeLocation) m.Translation = p; // Extrinsics of the camera relative to the rig node - this.cameraExtrinsics ??= new CoordinateSystem(this.cameraSensor.GetCameraExtrinsicsMatrix().ToMathNetMatrix()); + if (!this.invertedCameraExtrinsics.HasValue) + { + Matrix4x4.Invert(this.cameraSensor.GetCameraExtrinsicsMatrix(), out var invertedMatrix); + this.invertedCameraExtrinsics = invertedMatrix; + } // Transform the rig node location to camera pose in world coordinates - var cameraPose = m.ToMathNetMatrix() * this.cameraExtrinsics.Invert() * CameraCoordinateSystem; + var cameraPose = CameraBasis * this.invertedCameraExtrinsics.Value * m; // Convert to \psi basis - return new CoordinateSystem(cameraPose.ChangeBasisHoloLensToPsi()); + return new CoordinateSystem(cameraPose.ToMathNetMatrix()); } private void CaptureThread() @@ -185,54 +280,19 @@ private void CaptureThread() try { - if (this.createCalibrationMap || this.computeCameraIntrinsics) + // Compute the map of calibration points if we don't have it already, and we need + // it, either b/c we need to compute camera intrinsics, or b/c we need to emit them. + if (this.calibrationPointsMap is null && this.configuration.RequiresCalibrationPointsMap()) { - // Get the resolution from the initial frame. We could also just have used constants - // based on the sensor type, but this approach keeps things more general/flexible. - var sensorFrame = this.cameraSensor.GetNextBuffer(); - var resolution = sensorFrame.GetResolution(); - var width = (int)resolution.Width; - var height = (int)resolution.Height; - - // Compute a lookup table of calibration points - List cameraPoints = new (); - List imagePoints = new (); - float[] cameraUnitPlanePoints = new float[width * height * 2]; - - int ci = 0; - for (int y = 0; y < height; y++) - { - for (int x = 0; x < width; x++) - { - // Check the return value for success (HRESULT == S_OK) - if (this.cameraSensor.MapImagePointToCameraUnitPlane(new Point(x + 0.5, y + 0.5), out var xy) == 0) - { - // Add the camera space mapping for the image pixel - cameraUnitPlanePoints[ci++] = (float)xy.X; - cameraUnitPlanePoints[ci++] = (float)xy.Y; - - var norm = Math.Sqrt((xy.X * xy.X) + (xy.Y * xy.Y) + 1.0); - imagePoints.Add(new Point2D(x + 0.5, y + 0.5)); - cameraPoints.Add(new Point3D(xy.X / norm, xy.Y / norm, 1.0 / norm)); - } - else - { - cameraUnitPlanePoints[ci++] = float.NaN; - cameraUnitPlanePoints[ci++] = float.NaN; - } - } - } - - this.calibrationPointsMap = new CalibrationPointsMap(width, height, cameraUnitPlanePoints); + this.calibrationPointsMap = this.ComputeCalibrationPointsMap(); + } - if (this.computeCameraIntrinsics) - { - // Compute instrinsics before the main loop as it could take a while. This avoids a long - // observed initial delay for the first posted frame while intrinsics are being computed. - this.cameraIntrinsics = this.ComputeCameraIntrinsics(width, height, cameraPoints, imagePoints); - } + if (this.cameraIntrinsics is null && this.configuration.RequiresCameraIntrinsics()) + { + this.cameraIntrinsics = this.calibrationPointsMap.ComputeCameraIntrinsics(); } + // Main capture loop while (!this.shutdown) { var sensorFrame = this.cameraSensor.GetNextBuffer(); @@ -261,6 +321,41 @@ private void CaptureThread() // Sensor-specific processing implemented by derived class if (!this.shutdown) { + // Post the map of calibration points (used for computing camera intrinsics) if requested + if (this.configuration.OutputCalibrationPointsMap && + (originatingTime - this.CalibrationPointsMap.LastEnvelope.OriginatingTime) > this.configuration.OutputCalibrationPointsMapMinInterval) + { + this.CalibrationPointsMap.Post(this.GetCalibrationPointsMap(), originatingTime); + } + + // Post the camera intrinsics if requested + if (this.configuration.OutputCameraIntrinsics && + (originatingTime - this.CameraIntrinsics.LastEnvelope.OriginatingTime) > this.configuration.OutputMinInterval) + { + this.CameraIntrinsics.Post(this.cameraIntrinsics, originatingTime); + } + + // compute the camera pose if needed + if (this.configuration.RequiresPose()) + { + var timestamp = PerceptionTimestampHelper.FromSystemRelativeTargetTime(TimeSpan.FromTicks((long)frameTicks)); + var rigNodeLocation = this.RigNodeLocator.TryLocateAtTimestamp(timestamp, MixedReality.WorldSpatialCoordinateSystem); + + // The rig node may not always be locatable, so we need a null check + if (rigNodeLocation != null) + { + // Compute the camera pose from the rig node location + this.cameraPose = this.ToCameraPose(rigNodeLocation); + } + } + + // Post the camera pose if requested + if (this.configuration.OutputPose && + (originatingTime - this.Pose.LastEnvelope.OriginatingTime) > this.configuration.OutputMinInterval) + { + this.Pose.Post(this.cameraPose, originatingTime); + } + this.ProcessSensorFrame(sensorFrame, resolution, frameTicks, originatingTime); } } @@ -282,40 +377,65 @@ private void CheckConsentAndThrow(ResearchModeSensorConsent consent) case ResearchModeSensorConsent.DeniedByUser: throw new UnauthorizedAccessException("Access to the camera was denied by the user"); case ResearchModeSensorConsent.NotDeclaredByApp: - throw new UnauthorizedAccessException("Camera capability was not declared in the app manifest"); + throw new UnauthorizedAccessException("Webcam capability was not declared in the app manifest"); case ResearchModeSensorConsent.UserPromptRequired: throw new UnauthorizedAccessException("Permission to access to the camera must be requested first"); } } - /// - /// Computes the camera intrinsics from a lookup table mapping image points to 3D points in camera space. - /// - /// The image width for the camera. - /// The image height for the camera. - /// The list of 3D camera points to use for calibration. - /// The list of corresponding 2D image points. - /// The camera intrinsics. - private ICameraIntrinsics ComputeCameraIntrinsics(int width, int height, List cameraPoints, List imagePoints) + private CalibrationPointsMap ComputeCalibrationPointsMap() { - // Initialize a starting camera matrix - var initialCameraMatrix = MathNet.Numerics.LinearAlgebra.Matrix.Build.Dense(3, 3); - var initialDistortion = MathNet.Numerics.LinearAlgebra.Vector.Build.Dense(2); - initialCameraMatrix[0, 0] = 250; // fx - initialCameraMatrix[1, 1] = 250; // fy - initialCameraMatrix[0, 2] = width / 2.0; // cx - initialCameraMatrix[1, 2] = height / 2.0; // cy - initialCameraMatrix[2, 2] = 1; - CalibrationExtensions.CalibrateCameraIntrinsics( - cameraPoints, - imagePoints, - initialCameraMatrix, - initialDistortion, - out var computedCameraMatrix, - out var computedDistortionCoefficients, - false); - - return new CameraIntrinsics(width, height, computedCameraMatrix, computedDistortionCoefficients, depthPixelSemantics: DepthPixelSemantics.DistanceToPoint); + int width, height; + switch (this.cameraSensor.GetSensorType()) + { + case ResearchModeSensorType.DepthLongThrow: + width = 320; + height = 288; + break; + case ResearchModeSensorType.DepthAhat: + width = 512; + height = 512; + break; + case ResearchModeSensorType.LeftFront: + case ResearchModeSensorType.RightFront: + case ResearchModeSensorType.LeftLeft: + case ResearchModeSensorType.RightRight: + width = 640; + height = 480; + break; + default: + throw new InvalidOperationException("Invalid research mode camera for computing calibration."); + } + + // Compute a lookup table of calibration points + double[] cameraUnitPlanePoints = new double[width * height * 2]; + + int ci = 0; + for (int y = 0; y < height; y++) + { + for (int x = 0; x < width; x++) + { + // Check the return value for success (HRESULT == S_OK) + if (this.cameraSensor.MapImagePointToCameraUnitPlane(new Point(x + 0.5, y + 0.5), out var xy) == 0) + { + // Add the camera space mapping for the image pixel + cameraUnitPlanePoints[ci++] = xy.X; + cameraUnitPlanePoints[ci++] = xy.Y; + } + else + { + cameraUnitPlanePoints[ci++] = double.NaN; + cameraUnitPlanePoints[ci++] = double.NaN; + } + } + } + + return new CalibrationPointsMap() + { + Width = width, + Height = height, + CameraUnitPlanePoints = cameraUnitPlanePoints, + }; } } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ResearchModeCameraConfiguration.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ResearchModeCameraConfiguration.cs new file mode 100644 index 000000000..0e525f017 --- /dev/null +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ResearchModeCameraConfiguration.cs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.MixedReality +{ + using System; + using HoloLens2ResearchMode; + + /// + /// Base class for configurations for components derived from . + /// + public abstract class ResearchModeCameraConfiguration + { + /// + /// Gets or sets a value indicating whether the camera intrinsics are emitted. + /// + public bool OutputCameraIntrinsics { get; set; } = true; + + /// + /// Gets or sets a value indicating whether the camera pose stream is emitted. + /// + public bool OutputPose { get; set; } = true; + + /// + /// Gets or sets a value indicating whether the original map of points for calibration are emitted. + /// + public bool OutputCalibrationPointsMap { get; set; } = false; + + /// + /// Gets or sets the minimum interval between posting calibration map messages. + /// + public TimeSpan OutputCalibrationPointsMapMinInterval { get; set; } = TimeSpan.FromSeconds(20); + + /// + /// Gets or sets the minumum interval between posting frames. + /// + /// This value can be used to reduce the emitting framerate of the camera. + /// The default value of results in the highest possible framerate. + public TimeSpan OutputMinInterval { get; set; } = TimeSpan.Zero; + + /// + /// Gets or sets the sensor type. + /// + internal ResearchModeSensorType SensorType { get; set; } + + /// + /// Gets a value indicating whether the configuration requires computing the calibration points map. + /// + /// True if the configuration requires computing the calibration points map. + internal abstract bool RequiresCalibrationPointsMap(); + + /// + /// Gets a value indicating whether the configuration requires computing the camera intrinsics. + /// + /// True if the configuration requires computing the camera intrinsics. + internal abstract bool RequiresCameraIntrinsics(); + + /// + /// Gets a value indicating whether the configuration requires computing the camera pose. + /// + /// True if the configuration requires computing the camera pose. + internal abstract bool RequiresPose(); + } +} diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ResearchModeImu.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ResearchModeImu.cs index b235beea4..98307c7ce 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ResearchModeImu.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/ResearchModeImu.cs @@ -17,6 +17,7 @@ namespace Microsoft.Psi.MixedReality /// public abstract class ResearchModeImu : IProducer<(Vector3D Sample, DateTime OriginatingTime)[]>, ISourceComponent { + private readonly string name; private readonly ResearchModeSensorDevice sensorDevice; private readonly ResearchModeImuSensor imuSensor; private readonly Task requestImuAccessTask; @@ -30,8 +31,10 @@ public abstract class ResearchModeImu : IProducer<(Vector3D Sample, DateTime Ori /// /// The pipeline to add the component to. /// The research mode sensor type. - public ResearchModeImu(Pipeline pipeline, ResearchModeSensorType sensorType) + /// An optional name for the component. + public ResearchModeImu(Pipeline pipeline, ResearchModeSensorType sensorType, string name = nameof(ResearchModeImu)) { + this.name = name; this.Pipeline = pipeline; this.sensorDevice = new ResearchModeSensorDevice(); this.requestImuAccessTask = this.sensorDevice.RequestIMUAccessAsync().AsTask(); @@ -71,6 +74,9 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) notifyCompleted(); } + /// + public override string ToString() => this.name; + /// /// Processes a sensor frame received from the sensor. /// @@ -87,20 +93,35 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) /// Function getting sample nanoseconds. protected void PostSamples(IResearchModeSensorFrame sensorFrame, T[] samples, Func toValueFn, Func toNanos) { - var frameTicks = sensorFrame.GetTimeStamp().HostTicks; + if (samples.Length == 0) + { + return; + } + + // Frames of samples arrive with a time stamp in 100ns "host ticks", which we convert to pipeline time. + // Individual samples within the frames have "VinylHupTicks", extracted by the toNanos function, which + // are in nanoseconds and are rooted at an arbitrary epoch (HololLens boot). To compute the originating + // time of individual samples, we find the difference between a particular sample and the last sample + // in the frame and apply this as a relative (negative) offset to the frame originating time, which also + // corresponds with the last sample but is rooted in a proper epoch. + var frameOriginatingTime = this.Pipeline.GetCurrentTimeFromElapsedTicks((long)sensorFrame.GetTimeStamp().HostTicks); + var lastRecentSampleNanos = toNanos(samples.Last()); // nanoseconds of last sample (arbitrary epoch) var frameSamples = samples .Select(sample => { + var sensorTicksOffsetFromFrame = (long)(toNanos(sample) - lastRecentSampleNanos) / 100; // negative offset in tick from last sample + var sampleOriginatingTime = frameOriginatingTime.AddTicks(sensorTicksOffsetFromFrame); // sample originating time relative to frame var val = toValueFn(sample); - var sensorTicks = (toNanos(sample) - toNanos(samples[0])) / 100; // nanoseconds to ticks - var sampleOriginatingTime = this.Pipeline.GetCurrentTimeFromElapsedTicks((long)(frameTicks + sensorTicks)); return (new Vector3D(-val.Z, -val.X, val.Y) /* \psi basis */, sampleOriginatingTime); }) .Where(sample => sample.sampleOriginatingTime > this.lastSampleOriginatingTime) .ToArray(); - this.lastSampleOriginatingTime = frameSamples.Last().sampleOriginatingTime; - var frameOriginatingTime = this.Pipeline.GetCurrentTimeFromElapsedTicks((long)frameTicks); - this.Out.Post(frameSamples, frameOriginatingTime); + + if (frameSamples.Length > 0) + { + this.lastSampleOriginatingTime = frameSamples.Last().sampleOriginatingTime; + this.Out.Post(frameSamples, frameOriginatingTime); + } } private void CaptureThread() diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/SceneUnderstanding.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/SceneUnderstanding.cs index 62e49e5dd..828083cc9 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/SceneUnderstanding.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/SceneUnderstanding.cs @@ -32,31 +32,38 @@ public class SceneUnderstanding : Generator, IProducer, I /// /// The pipeline to add the component to. /// The configuration for the component. - public SceneUnderstanding(Pipeline pipeline, SceneUnderstandingConfiguration configuration = null) - : base(pipeline, true) + /// An optional name for the component. + public SceneUnderstanding(Pipeline pipeline, SceneUnderstandingConfiguration configuration = null, string name = nameof(SceneUnderstanding)) + : base(pipeline, true, name) { - this.pipeline = pipeline; - // requires Spatial Perception capability if (!SceneObserver.IsSupported()) { throw new Exception("SceneObserver is not supported."); } - this.configuration = configuration ??= new (); - this.placementRectangleSize = this.configuration.InitialPlacementRectangleSize; - this.PlacementRectangleSizeInput = pipeline.CreateReceiver<(int Height, int Width)>(this, this.UpdatePlacementRectangleSize, nameof(this.PlacementRectangleSizeInput)); - this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); + this.pipeline = pipeline; // Defer call to SK.AddStepper(this) to PipelineRun to ensure derived classes have finished construction! // Otherwise IStepper.Initialize() could get called before this object is fully constructed. - pipeline.PipelineRun += (_, _) => + this.pipeline.PipelineRun += (_, _) => { if (SK.AddStepper(this) == default) { throw new Exception($"Unable to add {this} as a Stepper to StereoKit."); } }; + + // Remove this stepper when pipeline is no longer running, otherwise Step() will continue to be called! + this.pipeline.PipelineCompleted += (_, _) => + { + SK.RemoveStepper(this); + }; + + this.configuration = configuration ??= new (); + this.placementRectangleSize = this.configuration.InitialPlacementRectangleSize; + this.PlacementRectangleSizeInput = pipeline.CreateReceiver<(int Height, int Width)>(this, this.UpdatePlacementRectangleSize, nameof(this.PlacementRectangleSizeInput)); + this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); } /// @@ -73,10 +80,7 @@ public SceneUnderstanding(Pipeline pipeline, SceneUnderstandingConfiguration con public bool Enabled => true; /// - public virtual bool Initialize() - { - return true; - } + public virtual bool Initialize() => true; /// public virtual void Step() @@ -165,7 +169,7 @@ Rectangle3D QuadToRectangle(SceneQuad quad, CoordinateSystem rectanglePose) var placementFromCenter = new Vec3(placement.X - (quad.Extents.X / 2f), placement.Y - (quad.Extents.Y / 2f), 0); return new Rectangle3D( - rectanglePose.Transform(placementFromCenter.ToPoint3D(false)), + rectanglePose.Transform(placementFromCenter.ToPoint3D()), rectanglePose.YAxis.Negate().Normalize(), rectanglePose.ZAxis.Normalize(), -w / 2, @@ -226,7 +230,7 @@ private static Mesh3D ToMesh3D(SceneMesh mesh, CoordinateSystem meshPose) mesh.GetVertexPositions(vertices); mesh.GetTriangleIndices(indices); - return new Mesh3D(vertices.Select(v => meshPose.Transform(v.ToPoint3D(false))).ToArray(), indices); + return new Mesh3D(vertices.Select(v => meshPose.Transform(v.ToPoint3D())).ToArray(), indices); } private void UpdatePlacementRectangleSize((int Width, int Height) size) @@ -236,7 +240,7 @@ private void UpdatePlacementRectangleSize((int Width, int Height) size) private CoordinateSystem GetWorldPose(SceneObject sceneObject) { - var posePsiBasis = new CoordinateSystem(sceneObject.GetLocationAsMatrix().ToMathNetMatrix().ChangeBasisHoloLensToPsi()); + var posePsiBasis = new CoordinateSystem(sceneObject.GetLocationAsMatrix().ToMathNetMatrix()); return posePsiBasis.TransformBy(this.scenePoseInWorld); } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/SerializedCameraIntrinsics.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/SerializedCameraIntrinsics.cs new file mode 100644 index 000000000..ba30e6065 --- /dev/null +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/SerializedCameraIntrinsics.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.MixedReality +{ + using System.Xml; + using System.Xml.Serialization; + + /// + /// Represents a camera's serialized camera intrinsics information. + /// + [XmlRoot] + public sealed class SerializedCameraIntrinsics + { + /// + /// Gets or sets the image width. + /// + [XmlElement] + public int ImageWidth { get; set; } + + /// + /// Gets or sets the image height. + /// + [XmlElement] + public int ImageHeight { get; set; } + + /// + /// Gets or sets the camera intrinsics transform matrix. + /// Intrinsics defines a 3x3 matrix stored in column-major order and assumes column-vectors + /// (i.e. Matrix * Point rather than Point * Matrix). + /// + [XmlArray] + public double[] Transform { get; set; } + + /// + /// Gets or sets the radial distortion coefficients. + /// + [XmlArray] + public double[] RadialDistortion { get; set; } = null; + + /// + /// Gets or sets the tangential distortion coefficients. + /// + [XmlArray] + public double[] TangentialDistortion { get; set; } = null; + + /// + /// Gets or sets a value indicating whether the closed form equation of the Brown-Conrady Distortion model + /// distorts or undistorts. i.e. if true then: + /// Xdistorted = Xundistorted * (1+K1*R2+K2*R3+... + /// otherwise: + /// Xundistorted = Xdistorted * (1+K1*R2+K2*R3+... + /// + [XmlElement] + public bool ClosedFormDistorts { get; set; } = true; + } +} diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/SpatialAnchorsSource.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/SpatialAnchorsSource.cs index dd2bb30a8..8a3ed007b 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/SpatialAnchorsSource.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/SpatialAnchorsSource.cs @@ -20,8 +20,9 @@ public class SpatialAnchorsSource : Generator, IProducer /// The pipeline to add the component to. /// The configuration for the component. - public SpatialAnchorsSource(Pipeline pipeline, TimeSpan interval) - : base(pipeline) + /// An optional name for the component. + public SpatialAnchorsSource(Pipeline pipeline, TimeSpan interval, string name = nameof(SpatialAnchorsSource)) + : base(pipeline, name: name) { this.interval = interval; this.Out = pipeline.CreateEmitter>(this, nameof(this.Out)); diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/VisibleLightCamera.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/VisibleLightCamera.cs index b0f517ed5..5990cf271 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/VisibleLightCamera.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/VisibleLightCamera.cs @@ -8,39 +8,24 @@ namespace Microsoft.Psi.MixedReality using HoloLens2ResearchMode; using Microsoft.Psi; using Microsoft.Psi.Imaging; - using Windows.Perception; + using Microsoft.Psi.Spatial.Euclidean; /// /// Visible light camera source component. /// public class VisibleLightCamera : ResearchModeCamera { - private readonly VisibleLightCameraConfiguration configuration; - private DateTime previousOriginatingTime = DateTime.MinValue; - /// /// Initializes a new instance of the class. /// /// The pipeline to add the component to. /// The configuration for this component. - public VisibleLightCamera(Pipeline pipeline, VisibleLightCameraConfiguration configuration = null) - : base( - pipeline, - (configuration ?? new VisibleLightCameraConfiguration()).Mode, - (configuration ?? new VisibleLightCameraConfiguration()).OutputCalibrationMap, - (configuration ?? new VisibleLightCameraConfiguration()).OutputCalibration) + /// An optional name for the component. + public VisibleLightCamera(Pipeline pipeline, VisibleLightCameraConfiguration configuration = null, string name = nameof(VisibleLightCamera)) + : base(pipeline, configuration ?? new VisibleLightCameraConfiguration(), name) { - this.configuration = configuration ?? new VisibleLightCameraConfiguration(); - - if (this.configuration.Mode != ResearchModeSensorType.LeftFront && - this.configuration.Mode != ResearchModeSensorType.LeftLeft && - this.configuration.Mode != ResearchModeSensorType.RightFront && - this.configuration.Mode != ResearchModeSensorType.RightRight) - { - throw new ArgumentException($"Initializing the camera in {this.configuration.Mode} mode is not supported."); - } - this.Image = pipeline.CreateEmitter>(this, nameof(this.Image)); + this.ImageCameraView = pipeline.CreateEmitter(this, nameof(this.ImageCameraView)); } /// @@ -48,43 +33,26 @@ public VisibleLightCamera(Pipeline pipeline, VisibleLightCameraConfiguration con /// public Emitter> Image { get; } + /// + /// Gets the grayscale image camera view. + /// + public Emitter ImageCameraView { get; } + + /// + /// Gets the visible light camera configuration. + /// + protected new VisibleLightCameraConfiguration Configuration => base.Configuration as VisibleLightCameraConfiguration; + /// protected override void ProcessSensorFrame(IResearchModeSensorFrame sensorFrame, ResearchModeSensorResolution resolution, ulong frameTicks, DateTime originatingTime) { - // If we're withing the specified min frame interval, return - if ((originatingTime - this.previousOriginatingTime) <= this.configuration.MinInterframeInterval) - { - return; - } - - if (this.configuration.OutputCalibrationMap && - (originatingTime - this.CalibrationPointsMap.LastEnvelope.OriginatingTime) > this.configuration.OutputCalibrationMapInterval) - { - // Post the calibration map created at the start - this.CalibrationPointsMap.Post(this.GetCalibrationPointsMap(), originatingTime); - } + var shouldOutputImage = this.Configuration.OutputImage && + (originatingTime - this.Image.LastEnvelope.OriginatingTime) > this.Configuration.OutputMinInterval; - if (this.configuration.OutputCalibration) - { - // Post the intrinsics computed at the start - this.CameraIntrinsics.Post(this.GetCameraIntrinsics(), originatingTime); - } + var shouldOutputImageCameraView = this.Configuration.OutputImageCameraView && + (originatingTime - this.ImageCameraView.LastEnvelope.OriginatingTime) > this.Configuration.OutputMinInterval; - if (this.configuration.OutputPose) - { - var timestamp = PerceptionTimestampHelper.FromSystemRelativeTargetTime(TimeSpan.FromTicks((long)frameTicks)); - var rigNodeLocation = this.RigNodeLocator.TryLocateAtTimestamp(timestamp, MixedReality.WorldSpatialCoordinateSystem); - - // The rig node may not always be locatable, so we need a null check - if (rigNodeLocation != null) - { - // Compute the camera pose from the rig node location - var cameraWorldPose = this.ToCameraPose(rigNodeLocation); - this.Pose.Post(cameraWorldPose, originatingTime); - } - } - - if (this.configuration.OutputImage) + if (shouldOutputImage || shouldOutputImageCameraView) { var vlcFrame = sensorFrame as ResearchModeSensorVlcFrame; var imageBuffer = vlcFrame.GetBuffer(); @@ -94,8 +62,17 @@ protected override void ProcessSensorFrame(IResearchModeSensorFrame sensorFrame, using var image = ImagePool.GetOrCreate(imageWidth, imageHeight, PixelFormat.Gray_8bpp); Debug.Assert(image.Resource.Size == imageBuffer.Length * sizeof(byte), "Image size does not match raw image buffer size!"); image.Resource.CopyFrom(imageBuffer); - this.Image.Post(image, originatingTime); - this.previousOriginatingTime = originatingTime; + + if (shouldOutputImage) + { + this.Image.Post(image, originatingTime); + } + + if (shouldOutputImageCameraView) + { + using var imageCameraView = new ImageCameraView(image, this.GetCameraIntrinsics(), this.GetCameraPose()); + this.ImageCameraView.Post(imageCameraView, originatingTime); + } } } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/VisibleLightCameraConfiguration.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/VisibleLightCameraConfiguration.cs index be09c7418..37d70b3be 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/VisibleLightCameraConfiguration.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.UniversalWindows/VisibleLightCameraConfiguration.cs @@ -9,43 +9,56 @@ namespace Microsoft.Psi.MixedReality /// /// Configuration for the component. /// - public class VisibleLightCameraConfiguration + public class VisibleLightCameraConfiguration : ResearchModeCameraConfiguration { /// - /// Gets or sets a value indicating whether the calibration settings are emitted. + /// Initializes a new instance of the class. /// - public bool OutputCalibration { get; set; } = true; + public VisibleLightCameraConfiguration() + { + this.VisibleLightSensorType = ResearchModeSensorType.LeftFront; + } /// - /// Gets or sets a value indicating whether the original map of points for calibration are emitted. + /// Gets or sets a value indicating whether the component emits grayscale images. /// - public bool OutputCalibrationMap { get; set; } = true; + public bool OutputImage { get; set; } = true; /// - /// Gets or sets the minimum interval between posting calibration map messages. + /// Gets or sets a value indicating whether the component emits grayscale image camera views. /// - public TimeSpan OutputCalibrationMapInterval { get; set; } = TimeSpan.FromSeconds(20); + public bool OutputImageCameraView { get; set; } = true; /// - /// Gets or sets a value indicating whether the camera pose stream is emitted. + /// Gets or sets the visible light sensor type. /// - public bool OutputPose { get; set; } = true; + public ResearchModeSensorType VisibleLightSensorType + { + get => this.SensorType; + set + { + if (value != ResearchModeSensorType.LeftLeft && + value != ResearchModeSensorType.LeftFront && + value != ResearchModeSensorType.RightFront && + value != ResearchModeSensorType.RightRight) + { + throw new ArgumentException($"{value} mode is not valid for {nameof(VisibleLightCameraConfiguration)}.{nameof(this.VisibleLightSensorType)}."); + } - /// - /// Gets or sets a value indicating whether the grayscale image stream is emitted. - /// - public bool OutputImage { get; set; } = true; + this.SensorType = value; + } + } - /// - /// Gets or sets the sensor selection. - /// - /// Valid values are: LeftFront, LeftLeft, RightFront, RightRight. - public ResearchModeSensorType Mode { get; set; } = ResearchModeSensorType.LeftFront; + /// + internal override bool RequiresCalibrationPointsMap() + => this.OutputCalibrationPointsMap || this.OutputCameraIntrinsics || this.OutputImageCameraView; - /// - /// Gets or sets the minumum inter-frame interval. - /// - /// This value can be user to reduce the emitting framerate of the visible light camera. - public TimeSpan MinInterframeInterval { get; set; } = TimeSpan.Zero; + /// + internal override bool RequiresCameraIntrinsics() + => this.OutputCameraIntrinsics || this.OutputImageCameraView; + + /// + internal override bool RequiresPose() + => this.OutputPose || this.OutputImageCameraView; } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.Visualization.Windows/HandVisualizationObject.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.Visualization.Windows/HandVisualizationObject.cs index 6341cb524..0dde5a74b 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.Visualization.Windows/HandVisualizationObject.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.Visualization.Windows/HandVisualizationObject.cs @@ -8,9 +8,9 @@ namespace Microsoft.Psi.MixedReality.Visualization using Microsoft.Psi.Visualization.VisualizationObjects; /// - /// Implements a visualization object for tracked hands. + /// Implements a visualization object for . /// - [VisualizationObject("Mixed Reality Tracked Hand")] + [VisualizationObject("Hand")] public class HandVisualizationObject : Point3DGraphVisualizationObject { /// diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.Visualization.Windows/HandVisualizationObjectAdapter.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.Visualization.Windows/HandVisualizationObjectAdapter.cs index d3df90963..961384fef 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality.Visualization.Windows/HandVisualizationObjectAdapter.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.Visualization.Windows/HandVisualizationObjectAdapter.cs @@ -13,9 +13,9 @@ namespace Microsoft.Psi.MixedReality.Visualization using Microsoft.Psi.Visualization.DataTypes; /// - /// Implements a stream adapter from single System.Tuple labeled rectangle into lists of System.Tuple labeled rectangles. + /// Implements a stream adapter for visualizing hands as graphs of 3D points. /// - [StreamAdapter] + [StreamAdapter("Hand to Graph")] public class HandVisualizationObjectAdapter : StreamAdapter> { private static Dictionary<(HandJointIndex Start, HandJointIndex End), bool> HandJointHierarchy { get; } = new List<(HandJointIndex, HandJointIndex)> diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality.Visualization.Windows/TrackedHandVisualizationObjectAdapter.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality.Visualization.Windows/TrackedHandVisualizationObjectAdapter.cs new file mode 100644 index 000000000..057543fa1 --- /dev/null +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality.Visualization.Windows/TrackedHandVisualizationObjectAdapter.cs @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.MixedReality.Visualization +{ + using System.Collections.Generic; + using System.Linq; + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.MixedReality; + using Microsoft.Psi.Visualization.Adapters; + using Microsoft.Psi.Visualization.Data; + using Microsoft.Psi.Visualization.DataTypes; + + /// + /// Implements a stream adapter for visualizing tracked hands as graphs of 3D points. + /// + [StreamAdapter("Tracked Hand to Graph")] + public class TrackedHandVisualizationObjectAdapter : StreamAdapter> + { + private static Dictionary<(HandJointIndex Start, HandJointIndex End), bool> HandJointHierarchy { get; } = new List<(HandJointIndex, HandJointIndex)> + { + (HandJointIndex.Wrist, HandJointIndex.Palm), + + (HandJointIndex.Wrist, HandJointIndex.ThumbMetacarpal), + (HandJointIndex.ThumbMetacarpal, HandJointIndex.ThumbProximal), + (HandJointIndex.ThumbProximal, HandJointIndex.ThumbDistal), + (HandJointIndex.ThumbDistal, HandJointIndex.ThumbTip), + + (HandJointIndex.Wrist, HandJointIndex.IndexMetacarpal), + (HandJointIndex.IndexMetacarpal, HandJointIndex.IndexProximal), + (HandJointIndex.IndexProximal, HandJointIndex.IndexIntermediate), + (HandJointIndex.IndexIntermediate, HandJointIndex.IndexDistal), + (HandJointIndex.IndexDistal, HandJointIndex.IndexTip), + + (HandJointIndex.Wrist, HandJointIndex.MiddleMetacarpal), + (HandJointIndex.MiddleMetacarpal, HandJointIndex.MiddleProximal), + (HandJointIndex.MiddleProximal, HandJointIndex.MiddleIntermediate), + (HandJointIndex.MiddleIntermediate, HandJointIndex.MiddleDistal), + (HandJointIndex.MiddleDistal, HandJointIndex.MiddleTip), + + (HandJointIndex.Wrist, HandJointIndex.RingMetacarpal), + (HandJointIndex.RingMetacarpal, HandJointIndex.RingProximal), + (HandJointIndex.RingProximal, HandJointIndex.RingIntermediate), + (HandJointIndex.RingIntermediate, HandJointIndex.RingDistal), + (HandJointIndex.RingDistal, HandJointIndex.RingTip), + + (HandJointIndex.Wrist, HandJointIndex.PinkyMetacarpal), + (HandJointIndex.PinkyMetacarpal, HandJointIndex.PinkyProximal), + (HandJointIndex.PinkyProximal, HandJointIndex.PinkyIntermediate), + (HandJointIndex.PinkyIntermediate, HandJointIndex.PinkyDistal), + (HandJointIndex.PinkyDistal, HandJointIndex.PinkyTip), + }.ToDictionary(j => j, j => true); + + /// + public override Graph GetAdaptedValue(Hand source, Envelope envelope) + { + if (source != null && source.IsTracked) + { + var dictionary = new Dictionary(); + + for (int jointIndex = 0; jointIndex < (int)HandJointIndex.MaxIndex; jointIndex++) + { + if (source.Joints[jointIndex] != null) + { + dictionary.Add((HandJointIndex)jointIndex, source.Joints[jointIndex].Origin); + } + } + + return new Graph(dictionary, HandJointHierarchy); + } + else + { + return null; + } + } + } +} diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/CalibrationPointsMap.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/CalibrationPointsMap.cs index 5f5787c26..aa80e3b8a 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality/CalibrationPointsMap.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/CalibrationPointsMap.cs @@ -10,37 +10,21 @@ namespace Microsoft.Psi.MixedReality /// For the image pixel each point corresponds to, (i,j), it was sampled at /// the center of the pixel, at position: (i+0.5, j+0.5). /// - public readonly struct CalibrationPointsMap + public sealed class CalibrationPointsMap { /// - /// Gets the sensor image width. + /// Gets or sets the sensor image width. /// - public readonly int Width; + public int Width { get; set; } /// - /// Gets the sensor image height. + /// Gets or sets the sensor image height. /// - public readonly int Height; + public int Height { get; set; } /// - /// Gets the set of XY points on the camera unit plane, one for the center of each image pixel. + /// Gets or sets the set of XY points on the camera unit plane, one for the center of each image pixel. /// - public readonly float[] CameraUnitPlanePoints; - - /// - /// Initializes a new instance of the struct. - /// - /// The sensor image width. - /// The sensor image height. - /// The set of XY points on the camera unit plane. - /// These points are laid out row-wise, X then Y, repeating. - /// For the image pixel each point corresponds to, (i,j), it was sampled at - /// the center of the pixel, at position: (i+0.5, j+0.5). - public CalibrationPointsMap(int width, int height, float[] cameraUnitPlanePoints) - { - this.Width = width; - this.Height = height; - this.CameraUnitPlanePoints = cameraUnitPlanePoints; - } + public double[] CameraUnitPlanePoints { get; set; } } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/EyesSensor.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/EyesSensor.cs index dc2b1667b..db8cdac94 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality/EyesSensor.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/EyesSensor.cs @@ -12,7 +12,7 @@ namespace Microsoft.Psi.MixedReality /// Source component that surfaces eye tracking information on a stream. /// /// Applications using this component must enable the Gaze Input capability in Package.appxmanifest. - public class EyesSensor : StereoKitComponent, IProducer, ISourceComponent + public class EyesSensor : StereoKitComponent, IProducer, ISourceComponent { private readonly Pipeline pipeline; private readonly TimeSpan interval; @@ -24,19 +24,20 @@ public class EyesSensor : StereoKitComponent, IProducer, ISour /// /// The pipeline to add the component to. /// Optional interval at which to poll eye tracking information (default 1/60th second). - public EyesSensor(Pipeline pipeline, TimeSpan interval = default) - : base(pipeline) + /// An optional name for the component. + public EyesSensor(Pipeline pipeline, TimeSpan interval = default, string name = nameof(EyesSensor)) + : base(pipeline, name) { this.pipeline = pipeline; - this.interval = interval == default ? TimeSpan.Zero : interval; - this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); + this.interval = interval == default ? TimeSpan.FromTicks(1) : interval; // minimum interval of one-tick + this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); this.EyesTracked = pipeline.CreateEmitter(this, nameof(this.EyesTracked)); } /// - /// Gets the stream of tracked eyes pose. + /// Gets the stream of tracked eyes pose as a . /// - public Emitter Out { get; private set; } + public Emitter Out { get; private set; } /// /// Gets the stream of whether eyes are currently tracked. @@ -60,14 +61,13 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) /// public override void Step() { - var currentTime = this.pipeline.GetCurrentTime(); - if (this.active && currentTime - this.Out.LastEnvelope.OriginatingTime >= this.interval) + // Get the current time from OpenXR + var currentSampleTime = this.pipeline.GetCurrentTimeFromOpenXr(); + + if (this.active && currentSampleTime - this.Out.LastEnvelope.OriginatingTime >= this.interval) { - var eyes = Input.Eyes; - var eyesTracked = Input.EyesTracked; - var originatingTime = this.pipeline.GetCurrentTime(); - this.Out.Post(eyes.ToPsiCoordinateSystem(), originatingTime); - this.EyesTracked.Post(eyesTracked.IsActive(), originatingTime); + this.Out.Post(PsiInput.Eyes, currentSampleTime); + this.EyesTracked.Post(Input.EyesTracked.IsActive(), currentSampleTime); } } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Hand.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Hand.cs index cd4129cb5..810ab81f2 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality/Hand.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/Hand.cs @@ -4,10 +4,9 @@ namespace Microsoft.Psi.MixedReality { using MathNet.Spatial.Euclidean; - using StereoKit; /// - /// Represents a tracked hand. + /// Represents one of the user's hands. /// public class Hand { @@ -52,48 +51,5 @@ public Hand(bool isTracked, bool isPinched, bool isGripped, CoordinateSystem[] j /// The joint index. /// The corresponding joint. public CoordinateSystem this[HandJointIndex handJointIndex] => this.Joints[(int)handJointIndex]; - - /// - /// Constructs a object from a StereoKit hand. - /// - /// The StereoKit hand. - /// The constructed object. - public static Hand FromStereoKitHand(StereoKit.Hand hand) - { - // note: StereoKit thumbs have no Root, but \psi thumbs have no Intermediate - var joints = new CoordinateSystem[(int)HandJointIndex.MaxIndex]; - joints[(int)HandJointIndex.Palm] = hand.palm.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.Wrist] = hand.wrist.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.ThumbMetacarpal] = hand[FingerId.Thumb, JointId.KnuckleMajor].Pose.ToPsiCoordinateSystem(); // treating as proximal - joints[(int)HandJointIndex.ThumbProximal] = hand[FingerId.Thumb, JointId.KnuckleMid].Pose.ToPsiCoordinateSystem(); // treating as intermediate - joints[(int)HandJointIndex.ThumbDistal] = hand[FingerId.Thumb, JointId.KnuckleMinor].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.ThumbTip] = hand[FingerId.Thumb, JointId.Tip].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.IndexMetacarpal] = hand[FingerId.Index, JointId.Root].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.IndexProximal] = hand[FingerId.Index, JointId.KnuckleMajor].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.IndexIntermediate] = hand[FingerId.Index, JointId.KnuckleMid].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.IndexDistal] = hand[FingerId.Index, JointId.KnuckleMinor].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.IndexTip] = hand[FingerId.Index, JointId.Tip].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.MiddleMetacarpal] = hand[FingerId.Middle, JointId.Root].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.MiddleProximal] = hand[FingerId.Middle, JointId.KnuckleMajor].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.MiddleIntermediate] = hand[FingerId.Middle, JointId.KnuckleMid].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.MiddleDistal] = hand[FingerId.Middle, JointId.KnuckleMinor].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.MiddleTip] = hand[FingerId.Middle, JointId.Tip].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.RingMetacarpal] = hand[FingerId.Ring, JointId.Root].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.RingProximal] = hand[FingerId.Ring, JointId.KnuckleMajor].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.RingIntermediate] = hand[FingerId.Ring, JointId.KnuckleMid].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.RingDistal] = hand[FingerId.Ring, JointId.KnuckleMinor].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.RingTip] = hand[FingerId.Ring, JointId.Tip].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.PinkyMetacarpal] = hand[FingerId.Little, JointId.Root].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.PinkyProximal] = hand[FingerId.Little, JointId.KnuckleMajor].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.PinkyIntermediate] = hand[FingerId.Little, JointId.KnuckleMid].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.PinkyDistal] = hand[FingerId.Little, JointId.KnuckleMinor].Pose.ToPsiCoordinateSystem(); - joints[(int)HandJointIndex.PinkyTip] = hand[FingerId.Little, JointId.Tip].Pose.ToPsiCoordinateSystem(); - - return new Hand( - hand.IsTracked, - hand.IsPinched, - hand.IsGripped, - joints); - } } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Handle.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Handle.cs index 86b61febf..a5f20ce40 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality/Handle.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/Handle.cs @@ -11,7 +11,7 @@ namespace Microsoft.Psi.MixedReality /// /// Component that represents a movable UI handle. /// - public class Handle : StereoKitComponent, IProducer, ISourceComponent + public class Handle : StereoKitRenderer, IProducer, ISourceComponent { private readonly Pipeline pipeline; private readonly string id; @@ -33,7 +33,7 @@ public Handle(Pipeline pipeline, CoordinateSystem initialPose, Vector3D bounds, { this.pipeline = pipeline; this.id = Guid.NewGuid().ToString(); - this.pose = initialPose.ToStereoKitPose(); + this.pose = initialPose.TransformBy(StereoKitTransforms.WorldToStereoKit).ToStereoKitPose(); this.bounds = new Bounds(new Vec3((float)bounds.Y, (float)bounds.Z, (float)bounds.X)); // psi -> SK coordinates this.show = showHandle; this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); @@ -44,17 +44,6 @@ public Handle(Pipeline pipeline, CoordinateSystem initialPose, Vector3D bounds, /// public Emitter Out { get; private set; } - /// - public override void Step() - { - var originatingTime = this.pipeline.GetCurrentTime(); - if (this.active) - { - UI.Handle(this.id, ref this.pose, this.bounds, this.show); - this.Out.Post(this.pose.ToPsiCoordinateSystem(), originatingTime); - } - } - /// public void Start(Action notifyCompletionTime) { @@ -68,5 +57,15 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) this.active = false; notifyCompleted(); } + + /// + protected override void Render() + { + if (this.active) + { + UI.Handle(this.id, ref this.pose, this.bounds, this.show); + this.Out.Post(this.pose.ToCoordinateSystem(), this.pipeline.GetCurrentTime()); + } + } } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/HandsSensor.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/HandsSensor.cs index d558f084f..4c837339d 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality/HandsSensor.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/HandsSensor.cs @@ -5,69 +5,47 @@ namespace Microsoft.Psi.MixedReality { using System; using Microsoft.Psi.Components; - using StereoKit; /// /// Source component that produces streams containing information about the tracked hands. /// - public class HandsSensor : StereoKitComponent, ISourceComponent + public class HandsSensor : StereoKitComponent, ISourceComponent, IProducer<(Hand Left, Hand Right)> { private readonly Pipeline pipeline; private readonly TimeSpan interval; private bool active; - private bool visible = true; - private bool solid = true; - private Material material = Default.MaterialHand; /// /// Initializes a new instance of the class. /// /// The pipeline to add the component to. /// Optional interval at which to poll hand information (default 1/60th second). - /// Optional value indicating whether hands should be rendered (default true). - /// Optional value indicating whether hands participate in StereoKit physics (default true). - /// Optional material used to render the hands (default ). - public HandsSensor(Pipeline pipeline, TimeSpan interval = default, bool visible = true, bool solid = false, Material material = null) - : base(pipeline) + /// An optional name for the component. + public HandsSensor(Pipeline pipeline, TimeSpan interval = default, string name = nameof(HandsSensor)) + : base(pipeline, name) { this.pipeline = pipeline; - this.interval = interval == default ? TimeSpan.Zero : interval; - this.visible = visible; - this.solid = solid; - this.material = material ?? Default.MaterialHand; + this.interval = interval == default ? TimeSpan.FromTicks(1) : interval; // minimum interval of one-tick + + this.Out = pipeline.CreateEmitter<(Hand Left, Hand Right)>(this, nameof(this.Out)); this.Left = pipeline.CreateEmitter(this, nameof(this.Left)); this.Right = pipeline.CreateEmitter(this, nameof(this.Right)); - this.Visible = pipeline.CreateReceiver(this, v => this.visible = v, nameof(this.Visible)); - this.Solid = pipeline.CreateReceiver(this, s => this.solid = s, nameof(this.Solid)); - this.Material = pipeline.CreateReceiver(this, m => this.material = m, nameof(this.Material)); } + /// + public Emitter<(Hand Left, Hand Right)> Out { get; } + /// /// Gets the stream of left hand information. /// public Emitter Left { get; } /// - /// Gets the stream of left hand information. + /// Gets the stream of right hand information. /// public Emitter Right { get; } - /// - /// Gets the receiver of a value indicating whether hands should be rendered. - /// - public Receiver Visible { get; } - - /// - /// Gets the receiver of a value indicating whether hands participate in physics. - /// - public Receiver Solid { get; } - - /// - /// Gets the receiver of the material used to render the hands. - /// - public Receiver Material { get; } - /// public void Start(Action notifyCompletionTime) { @@ -85,22 +63,17 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) /// public override void Step() { - var currentTime = this.pipeline.GetCurrentTime(); - if (this.active && currentTime - this.Left.LastEnvelope.OriginatingTime >= this.interval) - { - var left = Input.Hand(Handed.Left); - var right = Input.Hand(Handed.Right); - var originatingTime = this.pipeline.GetCurrentTime(); + // Get the current time from OpenXR + var currentSampleTime = this.pipeline.GetCurrentTimeFromOpenXr(); - left.Visible = this.visible; - left.Solid = this.solid; - left.Material = this.material; - this.Left.Post(Hand.FromStereoKitHand(left), originatingTime); + if (this.active && currentSampleTime - this.Out.LastEnvelope.OriginatingTime >= this.interval) + { + var leftHand = PsiInput.LeftHand; + var rightHand = PsiInput.RightHand; - right.Visible = this.visible; - right.Solid = this.solid; - right.Material = this.material; - this.Right.Post(Hand.FromStereoKitHand(right), originatingTime); + this.Left.Post(leftHand, currentSampleTime); + this.Right.Post(rightHand, currentSampleTime); + this.Out.Post((leftHand, rightHand), currentSampleTime); } } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/HeadSensor.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/HeadSensor.cs index 68f88fb57..e83e39871 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality/HeadSensor.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/HeadSensor.cs @@ -23,11 +23,12 @@ public class HeadSensor : StereoKitComponent, IProducer, ISour /// /// The pipeline to add the component to. /// Optional interval at which to poll head information (default 1/60th second). - public HeadSensor(Pipeline pipeline, TimeSpan interval = default) - : base(pipeline) + /// An optional name for the component. + public HeadSensor(Pipeline pipeline, TimeSpan interval = default, string name = nameof(HeadSensor)) + : base(pipeline, name) { this.pipeline = pipeline; - this.interval = interval == default ? TimeSpan.Zero : interval; + this.interval = interval == default ? TimeSpan.FromTicks(1) : interval; // minimum interval of one-tick this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); } @@ -39,12 +40,12 @@ public HeadSensor(Pipeline pipeline, TimeSpan interval = default) /// public override void Step() { - var currentTime = this.pipeline.GetCurrentTime(); - if (this.active && currentTime - this.Out.LastEnvelope.OriginatingTime >= this.interval) + // Get the current time from OpenXR + var currentSampleTime = this.pipeline.GetCurrentTimeFromOpenXr(); + + if (this.active && currentSampleTime - this.Out.LastEnvelope.OriginatingTime >= this.interval) { - var head = Input.Head; - var originatingTime = this.pipeline.GetCurrentTime(); - this.Out.Post(head.ToPsiCoordinateSystem(), originatingTime); + this.Out.Post(PsiInput.Head, currentSampleTime); } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Microphone.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Microphone.cs index fc7c6c8f5..c5d447d21 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality/Microphone.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/Microphone.cs @@ -28,8 +28,9 @@ public class Microphone : StereoKitComponent, IProducer, ISourceCom /// /// The pipeline to add the component to. /// The configuration for the component. - public Microphone(Pipeline pipeline, MicrophoneConfiguration configuration = null) - : base(pipeline) + /// An optional name for the component. + public Microphone(Pipeline pipeline, MicrophoneConfiguration configuration = null, string name = nameof(Microphone)) + : base(pipeline, name) { this.pipeline = pipeline; this.configuration = configuration ?? new MicrophoneConfiguration(); diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Operators.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Operators.cs index a47ad92a2..a0aae9dd7 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality/Operators.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/Operators.cs @@ -7,8 +7,10 @@ namespace Microsoft.Psi.MixedReality using System.Numerics; using MathNet.Numerics.LinearAlgebra.Double; using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Spatial.Euclidean; using StereoKit; using StereoKitColor = StereoKit.Color; + using StereoKitColor32 = StereoKit.Color32; using SystemDrawingColor = System.Drawing.Color; /// @@ -20,99 +22,60 @@ public static partial class Operators private static readonly CoordinateSystem HoloLensBasisInverted = HoloLensBasis.Invert(); /// - /// Compute a change of basis for the given matrix. From HoloLens basis to \psi basis. - /// - /// The given matrix in HoloLens basis. - /// The converted matrix with \psi basis. - /// /// - /// The HoloLens basis assumes that Forward=-Z, Left=-X, and Up=Y. - /// The \psi basis assumes that Forward=X, Left=Y, and Up=Z. - /// - public static DenseMatrix ChangeBasisHoloLensToPsi(this DenseMatrix holoLensMatrix) - { - return HoloLensBasisInverted * holoLensMatrix * HoloLensBasis; - } - - /// - /// Compute a change of basis for the given matrix. From \psi basis to HoloLens basis. - /// - /// The given matrix in \psi basis. - /// The converted matrix with HoloLens basis. - /// /// - /// The HoloLens basis assumes that Forward=-Z, Left=-X, and Up=Y. - /// The \psi basis assumes that Forward=X, Left=Y, and Up=Z. - /// - public static DenseMatrix ChangeBasisPsiToHoloLens(this DenseMatrix psiMatrix) - { - return HoloLensBasis * psiMatrix * HoloLensBasisInverted; - } - - /// - /// Converts a pose to a \psi , - /// changing basis from HoloLens to \psi and transforming from StereoKit coordinates to world coordinates. + /// Converts a to a , + /// changing basis from HoloLens to MathNet. /// /// The to be converted. /// The . /// /// The HoloLens basis assumes that Forward=-Z, Left=-X, and Up=Y. - /// The \psi basis assumes that Forward=X, Left=Y, and Up=Z. - /// "StereoKit coordinates" means "in relation to the pose of the headset at startup". - /// "World coordinates" means "in relation to the world spatial anchor". + /// The MathNet basis assumes that Forward=X, Left=Y, and Up=Z. /// - public static CoordinateSystem ToPsiCoordinateSystem(this StereoKit.Matrix stereoKitMatrix) + public static CoordinateSystem ToCoordinateSystem(this StereoKit.Matrix stereoKitMatrix) { Matrix4x4 systemMatrix = stereoKitMatrix; - var mathNetMatrix = systemMatrix.ToMathNetMatrix().ChangeBasisHoloLensToPsi(); - var coordinateSystem = new CoordinateSystem(mathNetMatrix); - return coordinateSystem.TransformBy(StereoKitTransforms.StereoKitStartingPose); + return new CoordinateSystem(systemMatrix.ToMathNetMatrix()); } /// - /// Converts a StereoKit to a \psi , - /// changing basis from HoloLens to \psi and transforming from StereoKit coordinates to world coordinates. + /// Converts a StereoKit to a , + /// changing basis from HoloLens to MathNet. /// /// The StereoKit to be converted. /// The . /// /// The HoloLens basis assumes that Forward=-Z, Left=-X, and Up=Y. - /// The \psi basis assumes that Forward=X, Left=Y, and Up=Z. - /// "StereoKit coordinates" means "in relation to the pose of the headset at startup". - /// "World coordinates" means "in relation to the world spatial anchor". + /// The MathNet basis assumes that Forward=X, Left=Y, and Up=Z. /// - public static CoordinateSystem ToPsiCoordinateSystem(this Pose pose) + public static CoordinateSystem ToCoordinateSystem(this Pose pose) { - return pose.ToMatrix().ToPsiCoordinateSystem(); + return pose.ToMatrix().ToCoordinateSystem(); } /// - /// Converts a pose to a pose, - /// changing basis from \psi to HoloLens and transforming from world coordinates to StereoKit coordinates. + /// Converts a to a , + /// changing basis from MathNet to HoloLens. /// - /// The pose to be converted. + /// The to be converted. /// The . /// /// The HoloLens basis assumes that Forward=-Z, Left=-X, and Up=Y. - /// The \psi basis assumes that Forward=X, Left=Y, and Up=Z. - /// "StereoKit coordinates" means "in relation to the pose of the headset at startup". - /// "World coordinates" means "in relation to the world spatial anchor". + /// The MathNet basis assumes that Forward=X, Left=Y, and Up=Z. /// public static StereoKit.Matrix ToStereoKitMatrix(this CoordinateSystem coordinateSystem) { - var mathNetMatrix = coordinateSystem.TransformBy(StereoKitTransforms.StereoKitStartingPoseInverse).ChangeBasisPsiToHoloLens(); - return new StereoKit.Matrix(mathNetMatrix.ToSystemNumericsMatrix()); + return new StereoKit.Matrix(coordinateSystem.ToHoloLensSystemMatrix()); } /// /// Converts a pose to a StereoKit , - /// changing basis from \psi to HoloLens and transforming from world coordinates to StereoKit coordinates. + /// changing basis from MathNet to HoloLens. /// /// The pose to be converted. /// The . /// /// The HoloLens basis assumes that Forward=-Z, Left=-X, and Up=Y. - /// The \psi basis assumes that Forward=X, Left=Y, and Up=Z. - /// "StereoKit coordinates" means "in relation to the pose of the headset at startup". - /// "World coordinates" means "in relation to the world spatial anchor". + /// The MathNet basis assumes that Forward=X, Left=Y, and Up=Z. /// public static Pose ToStereoKitPose(this CoordinateSystem coordinateSystem) { @@ -120,128 +83,48 @@ public static Pose ToStereoKitPose(this CoordinateSystem coordinateSystem) } /// - /// Converts a to a . - /// - /// The System.Numerics matrix. - /// The MathNet dense matrix. - public static DenseMatrix ToMathNetMatrix(this Matrix4x4 systemNumericsMatrix) - { - // Values are stored column-major. - var values = new double[] - { - systemNumericsMatrix.M11, - systemNumericsMatrix.M12, - systemNumericsMatrix.M13, - systemNumericsMatrix.M14, - systemNumericsMatrix.M21, - systemNumericsMatrix.M22, - systemNumericsMatrix.M23, - systemNumericsMatrix.M24, - systemNumericsMatrix.M31, - systemNumericsMatrix.M32, - systemNumericsMatrix.M33, - systemNumericsMatrix.M34, - systemNumericsMatrix.M41, - systemNumericsMatrix.M42, - systemNumericsMatrix.M43, - systemNumericsMatrix.M44, - }; - - return new DenseMatrix(4, 4, values); - } - - /// - /// Converts a to a . - /// - /// The MathNet dense matrix. - /// The System.Numerics matrix. - public static Matrix4x4 ToSystemNumericsMatrix(this DenseMatrix mathNetMatrix) - { - var values = mathNetMatrix.Values; - return new Matrix4x4( - (float)values[0], - (float)values[1], - (float)values[2], - (float)values[3], - (float)values[4], - (float)values[5], - (float)values[6], - (float)values[7], - (float)values[8], - (float)values[9], - (float)values[10], - (float)values[11], - (float)values[12], - (float)values[13], - (float)values[14], - (float)values[15]); - } - - /// - /// Convert to , changing the basis from \psi to HoloLens. + /// Convert to , changing the basis from MathNet to HoloLens. /// /// to be converted. - /// If true, transform from world coordinates to StereoKit coordinates. /// . /// /// The HoloLens basis assumes that Forward=-Z, Left=-X, and Up=Y. - /// The \psi basis assumes that Forward=X, Left=Y, and Up=Z. - /// "StereoKit coordinates" means "in relation to the pose of the headset at startup". - /// "World coordinates" means "in relation to the world spatial anchor". + /// The MathNet basis assumes that Forward=X, Left=Y, and Up=Z. /// - public static Vec3 ToVec3(this Point3D point3d, bool transformWorldToStereoKit = true) + public static Vec3 ToVec3(this Point3D point3d) { - if (transformWorldToStereoKit) - { - point3d = StereoKitTransforms.StereoKitStartingPoseInverse.Transform(point3d); - } - - // Change of basis happening here: + // Change of basis happening in place here. return new Vec3(-(float)point3d.Y, (float)point3d.Z, -(float)point3d.X); } /// - /// Convert to , changing the basis from HoloLens to \psi. + /// Convert to , changing the basis from HoloLens to MathNet. /// /// to be converted. - /// If true, transform from StereoKit coordinates to world coordinates. /// . /// /// The HoloLens basis assumes that Forward=-Z, Left=-X, and Up=Y. - /// The \psi basis assumes that Forward=X, Left=Y, and Up=Z. - /// "StereoKit coordinates" means "in relation to the pose of the headset at startup". - /// "World coordinates" means "in relation to the world spatial anchor". + /// The MathNet basis assumes that Forward=X, Left=Y, and Up=Z. /// - public static Point3D ToPoint3D(this Vec3 vec3, bool transformStereoKitToWorld = true) + public static Point3D ToPoint3D(this Vec3 vec3) { - var point3D = new Point3D(-vec3.z, -vec3.x, vec3.y); - - if (transformStereoKitToWorld) - { - return StereoKitTransforms.StereoKitStartingPose.Transform(point3D); - } - else - { - return point3D; - } + // Change of basis happening in place here. + return new Point3D(-vec3.z, -vec3.x, vec3.y); } /// - /// Convert to , changing the basis from HoloLens to \psi. + /// Convert to , changing the basis from HoloLens to MathNet. /// /// to be converted. - /// If true, transform from StereoKit coordinates to world coordinates. /// . /// /// The HoloLens basis assumes that Forward=-Z, Left=-X, and Up=Y. - /// The \psi basis assumes that Forward=X, Left=Y, and Up=Z. - /// "StereoKit coordinates" means "in relation to the pose of the headset at startup". - /// "World coordinates" means "in relation to the world spatial anchor". + /// The MathNet basis assumes that Forward=X, Left=Y, and Up=Z. /// - public static Point3D ToPoint3D(this Vector3 vector3, bool transformStereoKitToWorld = true) + public static Point3D ToPoint3D(this Vector3 vector3) { Vec3 v = vector3; - return v.ToPoint3D(transformStereoKitToWorld); + return v.ToPoint3D(); } /// @@ -252,20 +135,108 @@ public static Point3D ToPoint3D(this Vector3 vector3, bool transformStereoKitToW public static StereoKitColor ToStereoKitColor(this SystemDrawingColor color) => new ((float)color.R / 255, (float)color.G / 255, (float)color.B / 255, (float)color.A / 255); + /// + /// Converts a specified to a . + /// + /// The . + /// The corresponding . + public static StereoKitColor32 ToStereoKitColor32(this SystemDrawingColor color) + => new (color.R, color.G, color.B, color.A); + /// /// Convert stream of frames of IMU samples to flattened stream of samples within. /// /// Stream of IMU frames. + /// An optional delivery policy. + /// An optional name for the stream operator. /// Stream of IMU samples. - public static IProducer SelectManyImuSamples(this IProducer<(Vector3D Sample, DateTime OriginatingTime)[]> source) + public static IProducer SelectManyImuSamples( + this IProducer<(Vector3D Sample, DateTime OriginatingTime)[]> source, + DeliveryPolicy<(Vector3D Sample, DateTime OriginatingTime)[]> deliveryPolicy = null, + string name = nameof(SelectManyImuSamples)) + => source.Process<(Vector3D Sample, DateTime OriginatingTime)[], Vector3D>( + (samples, envelope, emitter) => + { + foreach (var sample in samples) + { + emitter.Post(sample.Sample, sample.OriginatingTime); + } + }, + deliveryPolicy, + name); + + /// + /// Gets the pipeline current time from OpenXR. + /// + /// The pipeline to get the current time for. + /// The current OpenXR time. + public static DateTime GetCurrentTimeFromOpenXr(this Pipeline pipeline) + { + long currentSampleTicks = TimeHelper.ConvertXrTimeToHnsTicks(Backend.OpenXR.Time); + return pipeline.GetCurrentTimeFromElapsedTicks(currentSampleTicks); + } + + /// + /// Converts a MathNet to a HoloLens . + /// + /// The MathNet dense matrix. + /// The HoloLens System.Numerics matrix. + /// The HoloLens basis assumes that Forward=-Z, Left=-X, and Up=Y. + /// The MathNet basis assumes that Forward=X, Left=Y, and Up=Z. + internal static Matrix4x4 ToHoloLensSystemMatrix(this DenseMatrix mathNetMatrix) + { + var holoLensMatrix = HoloLensBasis * mathNetMatrix * HoloLensBasisInverted; + return new Matrix4x4( + (float)holoLensMatrix.Values[0], + (float)holoLensMatrix.Values[1], + (float)holoLensMatrix.Values[2], + (float)holoLensMatrix.Values[3], + (float)holoLensMatrix.Values[4], + (float)holoLensMatrix.Values[5], + (float)holoLensMatrix.Values[6], + (float)holoLensMatrix.Values[7], + (float)holoLensMatrix.Values[8], + (float)holoLensMatrix.Values[9], + (float)holoLensMatrix.Values[10], + (float)holoLensMatrix.Values[11], + (float)holoLensMatrix.Values[12], + (float)holoLensMatrix.Values[13], + (float)holoLensMatrix.Values[14], + (float)holoLensMatrix.Values[15]); + } + + /// + /// Converts a HoloLens to a MathNet . + /// + /// The System.Numerics matrix. + /// The MathNet dense matrix. + /// The HoloLens basis assumes that Forward=-Z, Left=-X, and Up=Y. + /// The MathNet basis assumes that Forward=X, Left=Y, and Up=Z. + internal static DenseMatrix ToMathNetMatrix(this Matrix4x4 holoLensMatrix) { - return source.Process<(Vector3D Sample, DateTime OriginatingTime)[], Vector3D>((samples, envelope, emitter) => + // Values are stored column-major. + var values = new double[] { - foreach (var sample in samples) - { - emitter.Post(sample.Sample, sample.OriginatingTime); - } - }); + holoLensMatrix.M11, + holoLensMatrix.M12, + holoLensMatrix.M13, + holoLensMatrix.M14, + holoLensMatrix.M21, + holoLensMatrix.M22, + holoLensMatrix.M23, + holoLensMatrix.M24, + holoLensMatrix.M31, + holoLensMatrix.M32, + holoLensMatrix.M33, + holoLensMatrix.M34, + holoLensMatrix.M41, + holoLensMatrix.M42, + holoLensMatrix.M43, + holoLensMatrix.M44, + }; + + var mathNetMatrix = new DenseMatrix(4, 4, values); + return HoloLensBasisInverted * mathNetMatrix * HoloLensBasis; } } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Properties/AssemblyInfo.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..5935712fd --- /dev/null +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/Properties/AssemblyInfo.cs @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Microsoft.Psi.MixedReality.UniversalWindows")] diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/PsiInput.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/PsiInput.cs new file mode 100644 index 000000000..1bc17dfd3 --- /dev/null +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/PsiInput.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.MixedReality +{ + using MathNet.Spatial.Euclidean; + using StereoKit; + + /// + /// Implements static properties for accessing inputs (head, eyes, and hands) with \psi conventions and types. + /// + public static class PsiInput + { + /// + /// Gets the current head pose as a \psi coordinate system. + /// + /// This input is also emitted on a stream by the component. + public static CoordinateSystem Head => Input.Head.ToPsi(); + + /// + /// Gets the current pose of the eyes as a expressed in \psi coordinates. + /// + /// This input is also emitted on a stream by the component. + public static Ray3D Eyes + { + get + { + var cs = Input.Eyes.ToPsi(); + return new Ray3D(cs.Origin, cs.XAxis); + } + } + + /// + /// Gets the left hand with \psi coordinate systems for all joint poses. + /// + /// This input is also emitted on a stream by the component. + public static Hand LeftHand => Input.Hand(Handed.Left).ToPsi(); + + /// + /// Gets the right hand with \psi coordinate systems for all joint poses. + /// + /// This input is also emitted on a stream by the component. + public static Hand RightHand => Input.Hand(Handed.Right).ToPsi(); + + private static Hand ToPsi(this StereoKit.Hand stereoKitHand) + { + var joints = new CoordinateSystem[(int)HandJointIndex.MaxIndex]; + + // All joint poses will be null if the hand is not tracked. + if (stereoKitHand.IsTracked) + { + // note: StereoKit thumbs have no Root, but \psi thumbs have no Intermediate + joints[(int)HandJointIndex.Palm] = stereoKitHand.palm.ToPsi(); + joints[(int)HandJointIndex.Wrist] = stereoKitHand.wrist.ToPsi(); + joints[(int)HandJointIndex.ThumbMetacarpal] = stereoKitHand[FingerId.Thumb, JointId.KnuckleMajor].Pose.ToPsi(); // treating as proximal + joints[(int)HandJointIndex.ThumbProximal] = stereoKitHand[FingerId.Thumb, JointId.KnuckleMid].Pose.ToPsi(); // treating as intermediate + joints[(int)HandJointIndex.ThumbDistal] = stereoKitHand[FingerId.Thumb, JointId.KnuckleMinor].Pose.ToPsi(); + joints[(int)HandJointIndex.ThumbTip] = stereoKitHand[FingerId.Thumb, JointId.Tip].Pose.ToPsi(); + joints[(int)HandJointIndex.IndexMetacarpal] = stereoKitHand[FingerId.Index, JointId.Root].Pose.ToPsi(); + joints[(int)HandJointIndex.IndexProximal] = stereoKitHand[FingerId.Index, JointId.KnuckleMajor].Pose.ToPsi(); + joints[(int)HandJointIndex.IndexIntermediate] = stereoKitHand[FingerId.Index, JointId.KnuckleMid].Pose.ToPsi(); + joints[(int)HandJointIndex.IndexDistal] = stereoKitHand[FingerId.Index, JointId.KnuckleMinor].Pose.ToPsi(); + joints[(int)HandJointIndex.IndexTip] = stereoKitHand[FingerId.Index, JointId.Tip].Pose.ToPsi(); + joints[(int)HandJointIndex.MiddleMetacarpal] = stereoKitHand[FingerId.Middle, JointId.Root].Pose.ToPsi(); + joints[(int)HandJointIndex.MiddleProximal] = stereoKitHand[FingerId.Middle, JointId.KnuckleMajor].Pose.ToPsi(); + joints[(int)HandJointIndex.MiddleIntermediate] = stereoKitHand[FingerId.Middle, JointId.KnuckleMid].Pose.ToPsi(); + joints[(int)HandJointIndex.MiddleDistal] = stereoKitHand[FingerId.Middle, JointId.KnuckleMinor].Pose.ToPsi(); + joints[(int)HandJointIndex.MiddleTip] = stereoKitHand[FingerId.Middle, JointId.Tip].Pose.ToPsi(); + joints[(int)HandJointIndex.RingMetacarpal] = stereoKitHand[FingerId.Ring, JointId.Root].Pose.ToPsi(); + joints[(int)HandJointIndex.RingProximal] = stereoKitHand[FingerId.Ring, JointId.KnuckleMajor].Pose.ToPsi(); + joints[(int)HandJointIndex.RingIntermediate] = stereoKitHand[FingerId.Ring, JointId.KnuckleMid].Pose.ToPsi(); + joints[(int)HandJointIndex.RingDistal] = stereoKitHand[FingerId.Ring, JointId.KnuckleMinor].Pose.ToPsi(); + joints[(int)HandJointIndex.RingTip] = stereoKitHand[FingerId.Ring, JointId.Tip].Pose.ToPsi(); + joints[(int)HandJointIndex.PinkyMetacarpal] = stereoKitHand[FingerId.Little, JointId.Root].Pose.ToPsi(); + joints[(int)HandJointIndex.PinkyProximal] = stereoKitHand[FingerId.Little, JointId.KnuckleMajor].Pose.ToPsi(); + joints[(int)HandJointIndex.PinkyIntermediate] = stereoKitHand[FingerId.Little, JointId.KnuckleMid].Pose.ToPsi(); + joints[(int)HandJointIndex.PinkyDistal] = stereoKitHand[FingerId.Little, JointId.KnuckleMinor].Pose.ToPsi(); + joints[(int)HandJointIndex.PinkyTip] = stereoKitHand[FingerId.Little, JointId.Tip].Pose.ToPsi(); + } + + return new Hand(stereoKitHand.IsTracked, stereoKitHand.IsPinched, stereoKitHand.IsGripped, joints); + } + + private static CoordinateSystem ToPsi(this Pose pose) => pose.ToCoordinateSystem().TransformBy(StereoKitTransforms.StereoKitToWorld); + } +} diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Box3DStereoKitRenderer.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Box3DStereoKitRenderer.cs new file mode 100644 index 000000000..acc211b2e --- /dev/null +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Box3DStereoKitRenderer.cs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.MixedReality +{ + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Spatial.Euclidean; + using StereoKit; + using Color = System.Drawing.Color; + + /// + /// Component that visually renders a 3D box. + /// + public class Box3DStereoKitRenderer : MeshStereoKitRenderer, IConsumer + { + private readonly bool roundedEdges = false; + private readonly float roundedEdgeRadius; + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Box color. + /// Whether to render as wireframe only. + /// Visibility. + /// An optional name for the component. + public Box3DStereoKitRenderer(Pipeline pipeline, Color color, bool wireframe = false, bool visible = true, string name = nameof(Box3DStereoKitRenderer)) + : base(pipeline, null, color, wireframe, visible, name) + { + this.In = pipeline.CreateReceiver(this, this.UpdateMesh, nameof(this.In)); + } + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Box to render. + /// Box color. + /// Whether to render as wireframe only. + /// Visibility. + /// An optional name for the component. + public Box3DStereoKitRenderer(Pipeline pipeline, Box3D box3D, Color color, bool wireframe = false, bool visible = true, string name = nameof(Box3DStereoKitRenderer)) + : this(pipeline, color, wireframe, visible, name) + { + this.UpdateMesh(box3D); + } + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Box color. + /// Render with rounded edges, with given radius. + /// Whether to render as wireframe only. + /// Visibility. + /// An optional name for the component. + public Box3DStereoKitRenderer(Pipeline pipeline, Color color, float roundedEdgeRadius, bool wireframe = false, bool visible = true, string name = nameof(Box3DStereoKitRenderer)) + : this(pipeline, color, wireframe, visible, name) + { + this.roundedEdges = true; + this.roundedEdgeRadius = roundedEdgeRadius; + } + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Box to render. + /// Box color. + /// Render with rounded edges, with given radius. + /// Whether to render as wireframe only. + /// Visibility. + /// An optional name for the component. + public Box3DStereoKitRenderer(Pipeline pipeline, Box3D box3D, Color color, float roundedEdgeRadius, bool wireframe = false, bool visible = true, string name = nameof(Box3DStereoKitRenderer)) + : this(pipeline, color, wireframe, visible, name) + { + this.roundedEdges = true; + this.roundedEdgeRadius = roundedEdgeRadius; + this.UpdateMesh(box3D); + } + + /// + /// Gets the receiver for the box to render. + /// + public Receiver In { get; private set; } + + private void UpdateMesh(Box3D box3D) + { + this.Mesh ??= this.roundedEdges ? Mesh.GenerateRoundedCube(Vec3.One, this.roundedEdgeRadius) : Mesh.Cube; + + var scale = Matrix.S(new Vec3((float)box3D.LengthY, (float)box3D.LengthZ, (float)box3D.LengthX)); + var pose = new CoordinateSystem(box3D.Center, box3D.XAxis, box3D.YAxis, box3D.ZAxis); + this.MeshTransform = scale * pose.ToStereoKitMatrix(); + } + } +} diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/EncodedImageRectangle3DStereoKitRenderer.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/EncodedImageRectangle3DStereoKitRenderer.cs new file mode 100644 index 000000000..bdf47743e --- /dev/null +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/EncodedImageRectangle3DStereoKitRenderer.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.MixedReality +{ + using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; + using StereoKit; + + /// + /// Component that visually renders an encoded image on a 3D rectangle. + /// + public class EncodedImageRectangle3DStereoKitRenderer : Rectangle3DStereoKitRenderer + { + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Visibility. + /// An optional name for the component. + public EncodedImageRectangle3DStereoKitRenderer(Pipeline pipeline, bool visible = true, string name = nameof(EncodedImageRectangle3DStereoKitRenderer)) + : base(pipeline, System.Drawing.Color.White, false, visible, name) + { + this.Image = pipeline.CreateReceiver>(this, this.UpdateImage, nameof(this.Image)); + } + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Rectangle to render. + /// Visibility. + /// An optional name for the component. + public EncodedImageRectangle3DStereoKitRenderer(Pipeline pipeline, Rectangle3D rectangle3D, bool visible = true, string name = nameof(EncodedImageRectangle3DStereoKitRenderer)) + : base(pipeline, rectangle3D, System.Drawing.Color.White, false, visible, name) + { + this.Image = pipeline.CreateReceiver>(this, this.UpdateImage, nameof(this.Image)); + } + + /// + /// Gets the receiver for encoded images to map onto the rectangle surface. + /// + public Receiver> Image { get; private set; } + + /// + /// Gets the receiver for the rectangle to draw the image on. + /// + public Receiver Rectangle => this.In; + + private void UpdateImage(Shared image) + { + this.Material[MatParamName.DiffuseTex] = Tex.FromMemory(image.Resource.GetBuffer()); + } + } +} diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/HandsStereoKitRenderer.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/HandsStereoKitRenderer.cs new file mode 100644 index 000000000..947b5552a --- /dev/null +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/HandsStereoKitRenderer.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.MixedReality +{ + using StereoKit; + using Color = System.Drawing.Color; + + /// + /// Component that controls rendering of the hands in StereoKit. + /// + public class HandsStereoKitRenderer + { + private readonly string name; + private readonly Material material = Default.MaterialHand; + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Initial value indicating whether hands should be rendered (default true). + /// Initial value indicating whether hands participate in StereoKit physics (default false). + /// An optional name for the component. + public HandsStereoKitRenderer(Pipeline pipeline, bool visible = true, bool solid = false, string name = nameof(HandsStereoKitRenderer)) + { + this.name = name; + this.ReceiveVisible(visible); + this.ReceiveSolid(solid); + + this.Visible = pipeline.CreateReceiver(this, this.ReceiveVisible, nameof(this.Visible)); + this.Solid = pipeline.CreateReceiver(this, this.ReceiveSolid, nameof(this.Solid)); + this.Color = pipeline.CreateReceiver(this, this.ReceiveColor, nameof(this.Color)); + } + + /// + /// Gets the receiver of a value indicating whether hands should be rendered. + /// + public Receiver Visible { get; } + + /// + /// Gets the receiver of a value indicating whether hands participate in the StereoKit physics system. + /// + public Receiver Solid { get; } + + /// + /// Gets the receiver of the color used to render the hands. + /// + public Receiver Color { get; } + + /// + public override string ToString() => this.name; + + private void ReceiveColor(Color color) + { + this.material[MatParamName.ColorTint] = color.ToStereoKitColor(); + Input.HandMaterial(Handed.Max, this.material); + } + + private void ReceiveVisible(bool visible) + { + Input.HandVisible(Handed.Max, visible); + } + + private void ReceiveSolid(bool solid) + { + Input.HandSolid(Handed.Max, solid); + } + } +} diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Mesh3DListStereoKitRenderer.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Mesh3DListStereoKitRenderer.cs deleted file mode 100644 index d6f4ad4f3..000000000 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Mesh3DListStereoKitRenderer.cs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace Microsoft.Psi.MixedReality -{ - using System.Collections.Generic; - using System.Linq; - using MathNet.Spatial.Euclidean; - using Microsoft.Psi.Spatial.Euclidean; - using StereoKit; - - /// - /// Component that visually renders a list of meshes. - /// - public class Mesh3DListStereoKitRenderer : ModelBasedStereoKitRenderer, IConsumer> - { - /// - /// Initializes a new instance of the class. - /// - /// The pipeline to add the component to. - /// Geometry pose. - /// Geometry scale. - /// Material color. - /// Visibility. - /// Whether to render as model as wireframe only. - public Mesh3DListStereoKitRenderer(Pipeline pipeline, CoordinateSystem pose, Vector3D scale, System.Drawing.Color color, bool visible = true, bool wireframe = false) - : base(pipeline, pose, scale, color, visible, wireframe) - { - this.In = pipeline.CreateReceiver>(this, this.UpdateMeshes, nameof(this.In)); - } - - /// - /// Initializes a new instance of the class. - /// - /// The pipeline to add the component to. - /// Visibility. - /// Whether to render as model as wireframe only. - public Mesh3DListStereoKitRenderer(Pipeline pipeline, bool visible = true, bool wireframe = true) - : this(pipeline, new CoordinateSystem(), new Vector3D(1, 1, 1), System.Drawing.Color.White, visible, wireframe) - { - } - - /// - /// Gets the receiver for meshes. - /// - public Receiver> In { get; private set; } - - /// - public override bool Initialize() - { - base.Initialize(); - this.Material.FaceCull = Cull.None; - return true; - } - - private void UpdateMeshes(List meshes) - { - static Mesh ToStereoKitMesh(Mesh3D mesh3d) - { - var verts = mesh3d.Vertices.Select(v => new Vertex(new Vec3((float)-v.Y, (float)v.Z, (float)-v.X), Vec3.One)).ToArray(); // TODO: surface normal? - var mesh = new Mesh(); - mesh.SetInds(mesh3d.TriangleIndices); - mesh.SetVerts(verts); - return mesh; - } - - var model = new Model(); - foreach (var mesh in meshes) - { - model.AddNode(null, Matrix.Identity, ToStereoKitMesh(mesh), this.Material); - } - - this.Model = model; - } - } -} diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Mesh3DStereoKitRenderer.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Mesh3DStereoKitRenderer.cs new file mode 100644 index 000000000..e4cd42213 --- /dev/null +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Mesh3DStereoKitRenderer.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.MixedReality +{ + using System.Linq; + using Microsoft.Psi.Spatial.Euclidean; + using StereoKit; + using Color = System.Drawing.Color; + + /// + /// Component that visually renders a . + /// + public class Mesh3DStereoKitRenderer : MeshStereoKitRenderer, IConsumer + { + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Mesh color. + /// Whether to render as wireframe only. + /// Visibility. + /// An optional name for the component. + public Mesh3DStereoKitRenderer(Pipeline pipeline, Color color, bool wireframe = false, bool visible = true, string name = nameof(Mesh3DStereoKitRenderer)) + : base(pipeline, null, color, wireframe, visible, name) + { + this.In = pipeline.CreateReceiver(this, this.UpdateMesh, nameof(this.In)); + } + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Mesh to render. + /// Mesh color. + /// Whether to render as wireframe only. + /// Visibility. + /// An optional name for the component. + public Mesh3DStereoKitRenderer(Pipeline pipeline, Mesh3D mesh3D, Color color, bool wireframe = false, bool visible = true, string name = nameof(Mesh3DStereoKitRenderer)) + : this(pipeline, color, wireframe, visible, name) + { + this.UpdateMesh(mesh3D); + } + + /// + /// Gets the receiver for the mesh. + /// + public Receiver In { get; private set; } + + private void UpdateMesh(Mesh3D mesh3D) + { + this.Mesh ??= new Mesh(); + this.Mesh.SetVerts(mesh3D.Vertices.Select(p => new Vertex(p.ToVec3(), Vec3.One)).ToArray()); + this.Mesh.SetInds(mesh3D.TriangleIndices); + } + } +} diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/MeshStereoKitRenderer.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/MeshStereoKitRenderer.cs index 1e3cd394a..35ad84155 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/MeshStereoKitRenderer.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/MeshStereoKitRenderer.cs @@ -7,40 +7,112 @@ namespace Microsoft.Psi.MixedReality using System.Reflection; using MathNet.Spatial.Euclidean; using StereoKit; + using Color = System.Drawing.Color; /// - /// Component that visually renders a mesh. + /// Component that visually renders a single mesh. /// - public class MeshStereoKitRenderer : ModelBasedStereoKitRenderer + public class MeshStereoKitRenderer : StereoKitRenderer { - private readonly Mesh mesh; + private Matrix pose = Matrix.Identity; + private Matrix scale = Matrix.Identity; /// /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - /// Geometry mesh. + /// The mesh to render. /// Geometry pose. /// Geometry scale. /// Material color. + /// Whether to render mesh as wireframe only. /// Visibility. - /// Whether to render as model as wireframe only. - public MeshStereoKitRenderer(Pipeline pipeline, Mesh mesh, CoordinateSystem pose, Vector3D scale, System.Drawing.Color color, bool visible = true, bool wireframe = false) - : base(pipeline, pose, scale, color, visible, wireframe) + /// An optional name for the component. + public MeshStereoKitRenderer(Pipeline pipeline, Mesh mesh, CoordinateSystem pose, Vector3D scale, Color color, bool wireframe = false, bool visible = true, string name = nameof(MeshStereoKitRenderer)) + : base(pipeline, name) { - this.mesh = mesh; + this.Mesh = mesh; + this.ReceivePose(pose); + this.ReceiveScale(scale); + + // initialize the material + this.Material = Default.Material.Copy(); + this.Material.FaceCull = Cull.None; + this.ReceiveColor(color); + this.ReceiveWireframe(wireframe); + this.IsVisible = visible; + + this.Pose = pipeline.CreateReceiver(this, this.ReceivePose, nameof(this.Pose)); + this.Scale = pipeline.CreateReceiver(this, this.ReceiveScale, nameof(this.Scale)); + this.Visible = pipeline.CreateReceiver(this, this.ReceiveVisible, nameof(this.Visible)); + this.Color = pipeline.CreateReceiver(this, this.ReceiveColor, nameof(this.Color)); + this.Wireframe = pipeline.CreateReceiver(this, this.ReceiveWireframe, nameof(this.Wireframe)); } /// /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - /// Geometry mesh. - public MeshStereoKitRenderer(Pipeline pipeline, Mesh mesh) - : this(pipeline, mesh, new CoordinateSystem(), new Vector3D(1, 1, 1), System.Drawing.Color.White) + /// The mesh to render. + /// Material color. + /// Whether to render mesh as wireframe only. + /// Visibility. + /// An optional name for the component. + public MeshStereoKitRenderer(Pipeline pipeline, Mesh mesh, Color color, bool wireframe = false, bool visible = true, string name = nameof(MeshStereoKitRenderer)) + : this(pipeline, mesh, null, new Vector3D(1, 1, 1), color, wireframe, visible, name) { } + /// + /// Gets receiver for geometry pose (in \psi basis). + /// + public Receiver Pose { get; private set; } + + /// + /// Gets receiver for geometry scale (in \psi basis). + /// + public Receiver Scale { get; private set; } + + /// + /// Gets receiver for visibility. + /// + public Receiver Visible { get; private set; } + + /// + /// Gets receiver for material color. + /// + public Receiver Color { get; private set; } + + /// + /// Gets receiver for wireframe indicator. + /// + public Receiver Wireframe { get; private set; } + + /// + /// Gets or sets the mesh to be rendered. + /// + protected Mesh Mesh { get; set; } = null; + + /// + /// Gets the overall render transform (scale * pose). + /// + protected Matrix RenderTransform { get; private set; } = Matrix.Identity; + + /// + /// Gets or sets the transform for drawing the mesh (relative to the overall render transform). + /// + protected Matrix MeshTransform { get; set; } = Matrix.Identity; + + /// + /// Gets the material used for rendering the mesh. + /// + protected Material Material { get; } + + /// + /// Gets a value indicating whether the renderer should be currently visibile or not. + /// + protected bool IsVisible { get; private set; } + /// /// Get a mesh from an embedded resource asset. /// @@ -55,11 +127,46 @@ public static Mesh CreateMeshFromEmbeddedResource(string name) } /// - public override bool Initialize() + protected override void Render() + { + if (this.IsVisible && this.Mesh is not null) + { + Hierarchy.Push(this.RenderTransform); + this.Mesh.Draw(this.Material, this.MeshTransform); + Hierarchy.Pop(); + } + } + + /// + /// Update visibility. + /// + /// Desired visibility. + /// Message envelope. + protected virtual void ReceiveVisible(bool visible, Envelope envelope) + { + this.IsVisible = visible; + } + + private void ReceivePose(CoordinateSystem pose) + { + this.pose = pose is null ? Matrix.Identity : pose.ToStereoKitMatrix(); + this.RenderTransform = this.scale * this.pose; + } + + private void ReceiveScale(Vector3D scale) + { + this.scale = Matrix.S(new Vec3((float)scale.Y, (float)scale.Z, (float)scale.X)); + this.RenderTransform = this.scale * this.pose; + } + + private void ReceiveColor(Color color) + { + this.Material[MatParamName.ColorTint] = color.ToStereoKitColor(); + } + + private void ReceiveWireframe(bool wireframe) { - base.Initialize(); - this.Model = Model.FromMesh(this.mesh, this.Material); - return true; + this.Material.Wireframe = wireframe; } } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/ModelBasedStereoKitRenderer.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/ModelBasedStereoKitRenderer.cs deleted file mode 100644 index 10377e658..000000000 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/ModelBasedStereoKitRenderer.cs +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace Microsoft.Psi.MixedReality -{ - using MathNet.Spatial.Euclidean; - using StereoKit; - - /// - /// Base class for StereoKit model-based rendering components. - /// - public abstract class ModelBasedStereoKitRenderer : StereoKitComponent - { - /// - /// Initializes a new instance of the class. - /// - /// The pipeline to add the component to. - /// Geometry pose. - /// Geometry scale. - /// Material color. - /// Visibility. - /// Whether to render as model as wireframe only. - public ModelBasedStereoKitRenderer(Pipeline pipeline, CoordinateSystem pose, Vector3D scale, System.Drawing.Color color, bool visible = true, bool wireframe = false) - : base(pipeline) - { - this.Color = color; - this.Visible = visible; - this.Wireframe = wireframe; - - // Convert pose and scale to StereoKit basis. - this.PoseMatrix = pose.ToStereoKitMatrix(); - this.Scale = new Vec3((float)scale.Y, (float)scale.Z, (float)scale.X); - - this.ColorInput = pipeline.CreateReceiver(this, this.ReceiveColor, nameof(this.ColorInput)); - this.PoseInput = pipeline.CreateReceiver(this, this.ReceivePose, nameof(this.PoseInput)); - this.ScaleInput = pipeline.CreateReceiver(this, this.ReceiveScale, nameof(this.ScaleInput)); - this.VisibleInput = pipeline.CreateReceiver(this, this.ReceiveVisible, nameof(this.VisibleInput)); - this.WireframeInput = pipeline.CreateReceiver(this, this.ReceiveWireframe, nameof(this.WireframeInput)); - } - - /// - /// Gets receiver for material color. - /// - public Receiver ColorInput { get; private set; } - - /// - /// Gets receiver for geometry pose (in \psi basis). - /// - public Receiver PoseInput { get; private set; } - - /// - /// Gets receiver for geometry scale (in \psi basis). - /// - public Receiver ScaleInput { get; private set; } - - /// - /// Gets receiver for visibility. - /// - public Receiver VisibleInput { get; private set; } - - /// - /// Gets receiver for wireframe indicator. - /// - public Receiver WireframeInput { get; private set; } - - /// - /// Gets or sets material. - /// - protected Material Material { get; set; } - - /// - /// Gets or sets geometry model. - /// - protected Model Model { get; set; } - - /// - /// Gets or sets the model transform. - /// - protected Matrix ModelTransform { get; set; } - - /// - /// Gets or sets the color. - /// - protected System.Drawing.Color Color { get; set; } - - /// - /// Gets or sets the pose as a Matrix (in StereoKit basis). - /// - protected Matrix PoseMatrix { get; set; } - - /// - /// Gets or sets the scale (in StereoKit basis). - /// - protected Vec3 Scale { get; set; } - - /// - /// Gets or sets a value indicating whether the renderer is visible. - /// - protected bool Visible { get; set; } - - /// - /// Gets or sets a value indicating whether to render the model as wireframe-only. - /// - protected bool Wireframe { get; set; } - - /// - public override bool Initialize() - { - this.UpdateMaterial(); - this.UpdateModelTransform(); - - return true; - } - - /// - public override void Step() - { - if (this.Visible) - { - this.Model?.Draw(this.ModelTransform); - } - } - - /// - /// Updates the material based on the other properties. - /// - protected virtual void UpdateMaterial() - { - this.Material ??= Default.Material.Copy(); - this.Material[MatParamName.ColorTint] = this.Color.ToStereoKitColor(); - this.Material.Wireframe = this.Wireframe; - - if (this.Model != null) - { - this.Model.Visuals[0].Material = this.Material; - } - } - - /// - /// Updates the model transform. - /// - protected virtual void UpdateModelTransform() - { - this.ModelTransform = Matrix.S(this.Scale) * this.PoseMatrix; - } - - private void ReceiveColor(System.Drawing.Color color) - { - this.Color = color; - this.UpdateMaterial(); - } - - private void ReceivePose(CoordinateSystem pose) - { - this.PoseMatrix = pose.ToStereoKitMatrix(); - this.UpdateModelTransform(); - } - - private void ReceiveScale(Vector3D scale) - { - this.Scale = new Vec3((float)scale.Y, (float)scale.Z, (float)scale.X); - this.UpdateModelTransform(); - } - - private void ReceiveVisible(bool visible) - { - this.Visible = visible; - } - - private void ReceiveWireframe(bool wireframe) - { - this.Wireframe = wireframe; - this.UpdateMaterial(); - } - } -} diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Rectangle3DListStereoKitRenderer.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Rectangle3DListStereoKitRenderer.cs deleted file mode 100644 index b9bdaa2a1..000000000 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Rectangle3DListStereoKitRenderer.cs +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace Microsoft.Psi.MixedReality -{ - using System.Collections.Generic; - using MathNet.Spatial.Euclidean; - using Microsoft.Psi.Spatial.Euclidean; - using StereoKit; - - /// - /// Component that visually renders a list of 3D rectangles. - /// - public class Rectangle3DListStereoKitRenderer : ModelBasedStereoKitRenderer, IConsumer> - { - /// - /// Initializes a new instance of the class. - /// - /// The pipeline to add the component to. - /// Geometry pose. - /// Material color. - /// Visibility. - /// Whether to render as model as wireframe only. - public Rectangle3DListStereoKitRenderer(Pipeline pipeline, CoordinateSystem pose, System.Drawing.Color color, bool visible = true, bool wireframe = false) - : base(pipeline, pose, new Vector3D(1, 1, 1), color, visible, wireframe) - { - this.In = pipeline.CreateReceiver>(this, this.UpdateRectangles, nameof(this.In)); - } - - /// - /// Initializes a new instance of the class. - /// - /// The pipeline to add the component to. - /// Initial visibility. - public Rectangle3DListStereoKitRenderer(Pipeline pipeline, bool visible = true) - : this(pipeline, new CoordinateSystem(), System.Drawing.Color.White, visible) - { - } - - /// - /// Gets the receiver for rectangles. - /// - public Receiver> In { get; private set; } - - /// - public override bool Initialize() - { - base.Initialize(); - this.Material.FaceCull = Cull.None; - return true; - } - - private static Vertex[] ConvertToStereoKitVertices(Rectangle3D rect) - { - // Convert rectangle points and normal into StereoKit vertices. We only need to change basis, - // and do not need to change from world to StereoKit coordinates, because that is already done - // by the parent Model that these mesh vertices will be attached to. - var normal = (rect.BottomRight - rect.BottomLeft).CrossProduct(rect.TopLeft - rect.BottomLeft).Normalize(); - var stereoKitNormal = normal.ToPoint3D().ToVec3(false); - - return new Vertex[] - { - new Vertex(rect.TopLeft.ToVec3(false), stereoKitNormal), - new Vertex(rect.TopRight.ToVec3(false), stereoKitNormal), - new Vertex(rect.BottomLeft.ToVec3(false), stereoKitNormal), - new Vertex(rect.BottomRight.ToVec3(false), stereoKitNormal), - }; - } - - private void UpdateRectangles(List rectangles) - { - static Mesh ToQuadMesh(Rectangle3D rect) - { - var mesh = new Mesh(); - mesh.SetVerts(ConvertToStereoKitVertices(rect)); - mesh.SetInds(new uint[] { 3, 1, 0, 2, 3, 0 }); // two triangles from corner vertices - return mesh; - } - - var model = new Model(); - foreach (var rect in rectangles) - { - model.AddNode(null, Matrix.Identity, ToQuadMesh(rect), this.Material); - } - - this.Model = model; - } - } -} diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Rectangle3DStereoKitRenderer.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Rectangle3DStereoKitRenderer.cs new file mode 100644 index 000000000..2574481a5 --- /dev/null +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/Rectangle3DStereoKitRenderer.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.MixedReality +{ + using Microsoft.Psi.Spatial.Euclidean; + using StereoKit; + using Color = System.Drawing.Color; + + /// + /// Component that visually renders a 3D rectangle. + /// + public class Rectangle3DStereoKitRenderer : MeshStereoKitRenderer, IConsumer + { + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Rectangle color. + /// Whether to render as wireframe only. + /// Visibility. + /// An optional name for the component. + public Rectangle3DStereoKitRenderer(Pipeline pipeline, Color color, bool wireframe = false, bool visible = true, string name = nameof(Rectangle3DStereoKitRenderer)) + : base(pipeline, null, color, wireframe, visible, name) + { + this.In = pipeline.CreateReceiver(this, this.UpdateMesh, nameof(this.In)); + } + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Rectangle to render. + /// Rectangle color. + /// Whether to render as wireframe only. + /// Visibility. + /// An optional name for the component. + public Rectangle3DStereoKitRenderer(Pipeline pipeline, Rectangle3D rectangle3D, Color color, bool wireframe = false, bool visible = true, string name = nameof(Rectangle3DStereoKitRenderer)) + : this(pipeline, color, wireframe, visible, name) + { + this.UpdateMesh(rectangle3D); + } + + /// + /// Gets the receiver for the rectangle to render. + /// + public Receiver In { get; private set; } + + private void UpdateMesh(Rectangle3D rectangle3D) + { + this.Mesh ??= Mesh.Quad; + + var pose = rectangle3D.GetCenteredCoordinateSystem().ToStereoKitMatrix(); + var scale = Matrix.S(new Vec3((float)rectangle3D.Width, (float)rectangle3D.Height, 1)); + this.MeshTransform = scale * pose; + } + } +} diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/StereoKitRenderer.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/StereoKitRenderer.cs new file mode 100644 index 000000000..970e7800e --- /dev/null +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/StereoKitRenderer.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.MixedReality +{ + using StereoKit; + + /// + /// Base class for StereoKit rendering components. + /// + /// This class ensures that rendering of \psi objects occurs in the correct world frame. + public abstract class StereoKitRenderer : StereoKitComponent + { + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// An optional name for the component. + public StereoKitRenderer(Pipeline pipeline, string name = nameof(StereoKitRenderer)) + : base(pipeline, name) + { + } + + /// + public override void Step() + { + Hierarchy.Push(StereoKitTransforms.WorldHierarchy); + this.Render(); + Hierarchy.Pop(); + } + + /// + /// All rendering/drawing that needs to happen on each frame. + /// + protected abstract void Render(); + } +} diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/TextStereoKitRenderer.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/TextStereoKitRenderer.cs new file mode 100644 index 000000000..06151297b --- /dev/null +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/TextStereoKitRenderer.cs @@ -0,0 +1,212 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.MixedReality +{ + using MathNet.Spatial.Euclidean; + using StereoKit; + using Color = System.Drawing.Color; + + /// + /// Component that visually renders text. + /// + public class TextStereoKitRenderer : StereoKitRenderer + { + private readonly Model billboardFillModel; + private readonly Material billboardFillMaterial; + private readonly (Vec3 startPoint, Vec3 endPoint)[] billboardBorderLines; + private readonly TextStereoKitRendererConfiguration configuration; + + private Matrix pose; + private TextStyle textStyle; + private StereoKit.Color billboardBorderColor; + private Matrix billboardFillTransform; + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// Configuration to use. + /// An optional name for the component. + public TextStereoKitRenderer(Pipeline pipeline, TextStereoKitRendererConfiguration configuration = null, string name = nameof(TextStereoKitRenderer)) + : base(pipeline, name) + { + this.configuration = configuration ?? new TextStereoKitRendererConfiguration(); + this.pose = this.configuration.Pose.ToStereoKitMatrix(); + + this.Pose = pipeline.CreateReceiver(this, this.ReceivePose, nameof(this.Pose)); + this.Visible = pipeline.CreateReceiver(this, v => this.configuration.Visible = v, nameof(this.Visible)); + this.BillboardSize = pipeline.CreateReceiver(this, this.ReceiveBillboardSize, nameof(this.BillboardSize)); + this.Text = pipeline.CreateReceiver(this, t => this.configuration.Text = t, nameof(this.Text)); + this.TextColor = pipeline.CreateReceiver(this, this.ReceiveTextColor, nameof(this.TextColor)); + + if (this.configuration.DrawBillboardFill) + { + // Initialize the billboard fill model and material + this.billboardFillMaterial = Material.Default.Copy(); + this.billboardFillMaterial[MatParamName.ColorTint] = this.configuration.BillboardFillColor.ToStereoKitColor(); + this.billboardFillMaterial.FaceCull = Cull.None; + this.billboardFillModel = Model.FromMesh(Mesh.Quad, this.billboardFillMaterial); + + this.BillboardFillColor = pipeline.CreateReceiver(this, this.ReceiveBillboardFillColor, nameof(this.BillboardFillColor)); + } + + if (this.configuration.DrawBillboardBorder) + { + this.billboardBorderLines = new (Vec3, Vec3)[4]; + this.billboardBorderColor = this.configuration.BillboardBorderColor.ToStereoKitColor(); + this.BillboardBorderColor = pipeline.CreateReceiver(this, this.ReceiveBillboardBorderColor, nameof(this.BillboardBorderColor)); + } + + this.UpdateTextStyle(); + this.UpdateBillboardSize(); + } + + /// + /// Gets receiver for visibility. + /// + public Receiver Visible { get; private set; } + + /// + /// Gets receiver for text pose (in \psi basis). + /// + public Receiver Pose { get; private set; } + + /// + /// Gets the receiver for the text content. + /// + public Receiver Text { get; } + + /// + /// Gets the receiver for the text color. + /// + public Receiver TextColor { get; } + + /// + /// Gets the receiver for billboard fill color. + /// + public Receiver BillboardFillColor { get; } + + /// + /// Gets the receiver for billboard border color. + /// + public Receiver BillboardBorderColor { get; } + + /// + /// Gets the receiver for the billboard size (width and height). + /// + public Receiver BillboardSize { get; } + + /// + protected override void Render() + { + if (this.configuration.Visible) + { + Hierarchy.Push(this.pose); + + if (this.configuration.DrawBillboardFill) + { + this.billboardFillModel.Draw(this.billboardFillTransform); + } + + if (this.configuration.DrawBillboardBorder) + { + foreach (var (lineStart, lineEnd) in this.billboardBorderLines) + { + Lines.Add(lineStart, lineEnd, this.billboardBorderColor, this.configuration.BillboardBorderThickness); + } + } + + if (!string.IsNullOrEmpty(this.configuration.Text)) + { + // Render the text + StereoKit.Text.Add( + this.configuration.Text, + Matrix.Identity, + this.configuration.DesiredTextSize ?? StereoKit.Text.Size(this.configuration.Text), + this.configuration.TextFit, + this.textStyle, + this.configuration.TextPosition, + this.configuration.TextAlign, + offZ: this.configuration.DrawBillboardFill ? -this.configuration.TextOffsetFromBillboard : 0); + } + + Hierarchy.Pop(); + } + } + + private void ReceiveBillboardBorderColor(Color color) + { + this.configuration.BillboardBorderColor = color; + this.billboardBorderColor = this.configuration.BillboardBorderColor.ToStereoKitColor(); + } + + private void ReceiveBillboardFillColor(Color color) + { + this.configuration.BillboardFillColor = color; + this.billboardFillMaterial[MatParamName.ColorTint] = this.configuration.BillboardFillColor.ToStereoKitColor(); + this.billboardFillModel.Visuals[0].Material = this.billboardFillMaterial; + } + + private void ReceiveTextColor(Color color) + { + this.configuration.TextColor = color; + this.UpdateTextStyle(); + } + + private void UpdateTextStyle() + { + var colorGamma = this.configuration.TextColor.ToStereoKitColor().ToGamma(); + this.textStyle = StereoKit.Text.MakeStyle(this.configuration.TextFont, TextStyle.Default.CharHeight, colorGamma); + } + + private void ReceiveBillboardSize(Vec2 size) + { + this.configuration.BillboardSize = size; + this.UpdateBillboardSize(); + } + + private void UpdateBillboardSize() + { + if (this.configuration.DrawBillboardFill) + { + this.billboardFillTransform = Matrix.S(new Vec3(this.configuration.BillboardSize.x, this.configuration.BillboardSize.y, 1)); + } + + if (this.configuration.DrawBillboardBorder) + { + // Initialize the billboard border + var halfWidth = 0.5f * this.configuration.BillboardSize.x; + var halfHeight = 0.5f * this.configuration.BillboardSize.y; + var halfThickness = 0.5f * this.configuration.BillboardBorderThickness; + + // bottom edge + var p1 = new Vec3(-halfWidth - this.configuration.BillboardBorderThickness, -halfHeight - halfThickness, 0); + var p2 = new Vec3(halfWidth + this.configuration.BillboardBorderThickness, -halfHeight - halfThickness, 0); + + // right edge + var p3 = new Vec3(halfWidth + halfThickness, -halfHeight - this.configuration.BillboardBorderThickness, 0); + var p4 = new Vec3(halfWidth + halfThickness, halfHeight + this.configuration.BillboardBorderThickness, 0); + + // top edge + var p5 = new Vec3(halfWidth + this.configuration.BillboardBorderThickness, halfHeight + halfThickness, 0); + var p6 = new Vec3(-halfWidth - this.configuration.BillboardBorderThickness, halfHeight + halfThickness, 0); + + // left edge + var p7 = new Vec3(-halfWidth - halfThickness, halfHeight + this.configuration.BillboardBorderThickness, 0); + var p8 = new Vec3(-halfWidth - halfThickness, -halfHeight - this.configuration.BillboardBorderThickness, 0); + + this.billboardBorderLines[0] = (p1, p2); // bottom edge + this.billboardBorderLines[1] = (p3, p4); // right edge + this.billboardBorderLines[2] = (p5, p6); // top edge + this.billboardBorderLines[3] = (p7, p8); // left edge + } + } + + private void ReceivePose(CoordinateSystem pose) + { + this.configuration.Pose = pose; + this.pose = pose.ToStereoKitMatrix(); + } + } +} \ No newline at end of file diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/TextStereoKitRendererConfiguration.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/TextStereoKitRendererConfiguration.cs new file mode 100644 index 000000000..d8fbb6f5a --- /dev/null +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/Renderers/TextStereoKitRendererConfiguration.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.MixedReality +{ + using MathNet.Spatial.Euclidean; + using StereoKit; + using Color = System.Drawing.Color; + + /// + /// Represents the configuration for a component. + /// + /// This configuration object is used when initializing a new renderer instance, but many of the parameters can + /// be subsequently changed based on streaming inputs to the component. + public class TextStereoKitRendererConfiguration + { + /// + /// Gets or sets the text to render. + /// + public string Text { get; set; } = null; + + /// + /// Gets or sets the pose of the text. + /// + public CoordinateSystem Pose { get; set; } = new CoordinateSystem(); + + /// + /// Gets or sets the color of the text. + /// + public Color TextColor { get; set; } = Color.White; + + /// + /// Gets or sets the font to use for the text style. + /// + public Font TextFont { get; set; } = Font.Default; + + /// + /// Gets or sets how the text's bounding rectangle should be positioned relative to the overall pose. + /// + public TextAlign TextPosition { get; set; } = TextAlign.Center; + + /// + /// Gets or sets how the text should be aligned within the text's bounding rectangle. + /// + public TextAlign TextAlign { get; set; } = TextAlign.Center; + + /// + /// Gets or sets the desired size (width, height) to draw the text (in meters). If null, the size is auto computed. + /// + public Vec2? DesiredTextSize { get; set; } = null; + + /// + /// Gets or sets how the text should behave when one of its size dimensions conflicts with the provided parameter. + /// + public TextFit TextFit { get; set; } = TextFit.Exact; + + /// + /// Gets or sets a value indicating whether to draw a billboard behind the text. + /// + public bool DrawBillboardFill { get; set; } = false; + + /// + /// Gets or sets the depth offset from the billboard that text should be drawn (in meters). + /// + /// When set high enough, alleviates weird aliasing effects of the text intersecting with the billboard fill (if drawn). + /// The farther away the user is, the more likely the text will render intersected with the billboard fill. + /// + public float TextOffsetFromBillboard { get; set; } = 0.01f; + + /// + /// Gets or sets the size (width, height) to draw the billboard (in meters). + /// + public Vec2 BillboardSize { get; set; } = Vec2.Zero; + + /// + /// Gets or sets the fill color of the billboard. + /// + public Color BillboardFillColor { get; set; } = Color.Black; + + /// + /// Gets or sets a value indicating whether to draw a border around the billboard. + /// + public bool DrawBillboardBorder { get; set; } = false; + + /// + /// Gets or sets the color of the billboard border. + /// + public Color BillboardBorderColor { get; set; } = Color.White; + + /// + /// Gets or sets the thickness of the billboard border (in meters). + /// + public float BillboardBorderThickness { get; set; } = 0.01f; + + /// + /// Gets or sets a value indicating whether or not the renderer should be visible. + /// + public bool Visible { get; set; } = true; + } +} \ No newline at end of file diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/SpatialSound.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/SpatialSound.cs index dba94dcce..ac45a4f51 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality/SpatialSound.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/SpatialSound.cs @@ -26,8 +26,9 @@ public class SpatialSound : StereoKitComponent, IConsumer /// The pipeline to add the component to. /// Initial position of spatial sound. /// Intial audio volume (0-1, default 1). - public SpatialSound(Pipeline pipeline, Point3D initialPosition, double initialVolume = 1) - : base(pipeline) + /// An optional name for the component. + public SpatialSound(Pipeline pipeline, Point3D initialPosition, double initialVolume = 1, string name = nameof(SpatialSound)) + : base(pipeline, name) { this.position = new CoordinateSystem(initialPosition, UnitVector3D.XAxis, UnitVector3D.YAxis, UnitVector3D.ZAxis).ToStereoKitMatrix().Translation; this.volume = (float)initialVolume; @@ -89,26 +90,19 @@ private void UpdateAudio(AudioBuffer audio) private void UpdatePosition(Point3D position) { - var p = new CoordinateSystem(position, UnitVector3D.XAxis, UnitVector3D.YAxis, UnitVector3D.ZAxis).ToStereoKitMatrix().Translation; + this.position = position.TransformBy(StereoKitTransforms.WorldToStereoKit).ToVec3(); if (this.playing) { - this.soundInst.Position = p; - } - else - { - this.position = p; + this.soundInst.Position = this.position; } } private void UpdateVolume(double volume) { + this.volume = (float)volume; if (this.playing) { - this.soundInst.Volume = (float)volume; - } - else - { - this.volume = (float)volume; + this.soundInst.Volume = this.volume; } } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/StereoKitComponent.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/StereoKitComponent.cs index 592fcca4f..d81817d0c 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality/StereoKitComponent.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/StereoKitComponent.cs @@ -12,12 +12,17 @@ namespace Microsoft.Psi.MixedReality /// public abstract class StereoKitComponent : IStepper { + private readonly string name; + /// /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - public StereoKitComponent(Pipeline pipeline) + /// An optional name for the component. + public StereoKitComponent(Pipeline pipeline, string name = nameof(StereoKitComponent)) { + this.name = name; + // Defer call to SK.AddStepper(this) to PipelineRun to ensure derived classes have finished construction! // Otherwise IStepper.Initialize() could get called before this object is fully constructed. pipeline.PipelineRun += (_, _) => @@ -50,5 +55,8 @@ public virtual void Step() public virtual void Shutdown() { } + + /// + public override string ToString() => this.name; } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/StereoKitTransforms.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/StereoKitTransforms.cs index ba0b6ab30..fa1e2d6a5 100644 --- a/Sources/MixedReality/Microsoft.Psi.MixedReality/StereoKitTransforms.cs +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/StereoKitTransforms.cs @@ -4,6 +4,7 @@ namespace Microsoft.Psi.MixedReality { using MathNet.Spatial.Euclidean; + using StereoKit; /// /// Static StereoKit transforms which are applied in/out of StereoKit from \psi. @@ -11,13 +12,22 @@ namespace Microsoft.Psi.MixedReality public static class StereoKitTransforms { /// - /// Gets or sets the starting pose of StereoKit (the headset) in the world (in \psi basis). + /// Gets the "world hierarchy" for rendering. + /// Push this matrix onto StereoKit's stack to render content coherently in the world. /// - public static CoordinateSystem StereoKitStartingPose { get; set; } = new CoordinateSystem(); + /// + /// This matrix is pushed automatically by the base class for new rendering components. + /// + public static Matrix WorldHierarchy { get; internal set; } = Matrix.Identity; /// - /// Gets or sets the inverse of the StereoKit starting pose in the world. + /// Gets or sets the transform from StereoKit to the world. /// - public static CoordinateSystem StereoKitStartingPoseInverse { get; set; } = new CoordinateSystem(); + internal static CoordinateSystem StereoKitToWorld { get; set; } = new CoordinateSystem(); + + /// + /// Gets or sets the the transform from the world to StereoKit. + /// + internal static CoordinateSystem WorldToStereoKit { get; set; } = new CoordinateSystem(); } } diff --git a/Sources/MixedReality/Microsoft.Psi.MixedReality/TimeHelper.cs b/Sources/MixedReality/Microsoft.Psi.MixedReality/TimeHelper.cs new file mode 100644 index 000000000..572aa12bf --- /dev/null +++ b/Sources/MixedReality/Microsoft.Psi.MixedReality/TimeHelper.cs @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.MixedReality +{ + using System; + using System.Runtime.InteropServices; + using StereoKit; + + /// + /// Provides helper methods for converting between OpenXR and platform times. + /// + internal static class TimeHelper + { + private static readonly double QpcToHns; + private static XR_xrConvertTimeToWin32PerformanceCounterKHR openXrConvertTimeToWin32PerformanceCounterKHR; + + /// + /// Initializes static members of the class. + /// + static TimeHelper() + { + NativeMethods.QueryPerformanceFrequency(out long frequency); + QpcToHns = 10000000.0 / frequency; + } + + private delegate int XR_xrConvertTimeToWin32PerformanceCounterKHR(ulong instance, long time, out long performanceCount); + + /// + /// Converts an OpenXR time value as returned from Backend.OpenXR.Time to 100 ns ticks based on the performance counter. + /// + /// The OpenXR time value. + /// The equivalent performance counter value in 100 ns ticks. + internal static long ConvertXrTimeToHnsTicks(long openXrTime) + { + return (long)(ConvertXrTimeToWin32PerformanceCounter(openXrTime) * QpcToHns); + } + + /// + /// Converts an OpenXR time value as returned from Backend.OpenXR.Time to a performance counter value as if generated by the QueryPerformanceCounter function. + /// + /// The OpenXR time value. + /// The equivalent performance counter value. + private static long ConvertXrTimeToWin32PerformanceCounter(long openXrTime) + { + // Initialize the delegate on first use + if (openXrConvertTimeToWin32PerformanceCounterKHR == null) + { + if (!SK.IsInitialized) + { + throw new InvalidOperationException("Attempting to convert OpenXR time before StereoKit is initialized. Ensure that SK.Initialize() has been called first."); + } + + if (Backend.XRType != BackendXRType.OpenXR) + { + throw new InvalidOperationException("Cannot convert OpenXR time. Backend XR type is not OpenXR."); + } + + openXrConvertTimeToWin32PerformanceCounterKHR = Backend.OpenXR.GetFunction("xrConvertTimeToWin32PerformanceCounterKHR"); + } + + // Get the raw performance counter value + openXrConvertTimeToWin32PerformanceCounterKHR(Backend.OpenXR.Instance, openXrTime, out long performanceCount); + + return performanceCount; + } + + /// + /// Provides native APIs used by the class. + /// + private static class NativeMethods + { + /// + /// Retrieves the current value of the performance counter. + /// + /// A variable that receives the current performance-counter value, in counts. + /// True if the function succeeds, false otherwise. + [DllImport("kernel32.dll")] + internal static extern bool QueryPerformanceCounter(out long performanceCount); + + /// + /// Retrieves the frequency of the performance counter. + /// + /// A variable that receives the current performance-counter frequency, in counts per second. + /// True if the function succeeds, false otherwise. + [DllImport("kernel32.dll")] + internal static extern bool QueryPerformanceFrequency(out long frequency); + } + } +} diff --git a/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/Properties/AssemblyInfo.cs b/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/Properties/AssemblyInfo.cs index b5a90029e..b90a3070f 100644 --- a/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/Properties/AssemblyInfo.cs +++ b/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/Properties/AssemblyInfo.cs @@ -35,6 +35,6 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("0.16.92.1")] -[assembly: AssemblyFileVersion("0.16.92.1")] -[assembly: AssemblyInformationalVersion("0.16.92.1-beta")] +[assembly: AssemblyVersion("0.17.52.1")] +[assembly: AssemblyFileVersion("0.17.52.1")] +[assembly: AssemblyInformationalVersion("0.17.52.1-beta")] diff --git a/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/RealSenseSensor.cs b/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/RealSenseSensor.cs index fcbcad405..227317e12 100644 --- a/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/RealSenseSensor.cs +++ b/Sources/RealSense/Microsoft.Psi.RealSense.Windows.x64/RealSenseSensor.cs @@ -14,8 +14,9 @@ namespace Microsoft.Psi.RealSense.Windows /// public class RealSenseSensor : ISourceComponent, IDisposable { + private readonly Pipeline pipeline; + private readonly string name; private bool shutdown; - private Pipeline pipeline; private RealSenseDevice device; private Thread thread; @@ -23,8 +24,10 @@ public class RealSenseSensor : ISourceComponent, IDisposable /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - public RealSenseSensor(Pipeline pipeline) + /// An optional name for the component. + public RealSenseSensor(Pipeline pipeline, string name = nameof(RealSenseSensor)) { + this.name = name; this.shutdown = false; this.ColorImage = pipeline.CreateEmitter>(this, "ColorImage"); this.DepthImage = pipeline.CreateEmitter>(this, "DepthImage"); @@ -85,16 +88,19 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) notifyCompleted(); } + /// + public override string ToString() => this.name; + private void ThreadProc() { - Imaging.PixelFormat pixelFormat = Imaging.PixelFormat.BGR_24bpp; + Imaging.PixelFormat pixelFormat = PixelFormat.BGR_24bpp; switch (this.device.GetColorBpp()) { case 24: - pixelFormat = Imaging.PixelFormat.BGR_24bpp; + pixelFormat = PixelFormat.BGR_24bpp; break; case 32: - pixelFormat = Imaging.PixelFormat.BGRX_32bpp; + pixelFormat = PixelFormat.BGRX_32bpp; break; default: throw new NotSupportedException("Expected 24bpp or 32bpp image."); @@ -105,16 +111,20 @@ private void ThreadProc() switch (this.device.GetDepthBpp()) { case 16: - pixelFormat = Imaging.PixelFormat.Gray_16bpp; + pixelFormat = PixelFormat.Gray_16bpp; break; case 8: - pixelFormat = Imaging.PixelFormat.Gray_8bpp; + pixelFormat = PixelFormat.Gray_8bpp; break; default: throw new NotSupportedException("Expected 8bpp or 16bpp image."); } - var depthImage = DepthImagePool.GetOrCreate((int)this.device.GetDepthWidth(), (int)this.device.GetDepthHeight()); + var depthImage = DepthImagePool.GetOrCreate( + (int)this.device.GetDepthWidth(), + (int)this.device.GetDepthHeight(), + DepthValueSemantics.DistanceToPlane, + 0.001); uint depthImageSize = this.device.GetDepthHeight() * this.device.GetDepthStride(); while (!this.shutdown) { diff --git a/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/AssemblyInfo.cpp b/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/AssemblyInfo.cpp index d9d49e6ae..ddb5bf116 100644 --- a/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/AssemblyInfo.cpp +++ b/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/AssemblyInfo.cpp @@ -31,9 +31,9 @@ using namespace System::Security::Permissions; // You can specify all the value or you can default the Revision and Build Numbers // by using the '*' as shown below: -[assembly:AssemblyVersionAttribute("0.16.92.1")]; -[assembly:AssemblyFileVersionAttribute("0.16.92.1")]; -[assembly:AssemblyInformationalVersionAttribute("0.16.92.1-beta")]; +[assembly:AssemblyVersionAttribute("0.17.52.1")]; +[assembly:AssemblyFileVersionAttribute("0.17.52.1")]; +[assembly:AssemblyInformationalVersionAttribute("0.17.52.1-beta")]; [assembly:ComVisible(false)]; diff --git a/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/Microsoft.Psi.RealSense_Interop.Windows.x64.vcxproj b/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/Microsoft.Psi.RealSense_Interop.Windows.x64.vcxproj index df033293d..4c4e5a8b2 100644 --- a/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/Microsoft.Psi.RealSense_Interop.Windows.x64.vcxproj +++ b/Sources/RealSense/Microsoft.Psi.RealSense_Interop.Windows.x64/Microsoft.Psi.RealSense_Interop.Windows.x64.vcxproj @@ -16,7 +16,7 @@ v4.7.2 ManagedCProj MicrosoftPsiRealSenseWindows_InteropWindowsx64 - 10.0.18362.0 + 10.0.19041.0 Microsoft.Psi.RealSense_Interop.Windows.x64 diff --git a/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/Operators.cs b/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/Operators.cs index de19c8d42..9d7c3472d 100644 --- a/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/Operators.cs +++ b/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/Operators.cs @@ -4,7 +4,6 @@ namespace Microsoft.Psi.Interop.Rendezvous { using System; - using System.Linq; using Microsoft.Psi.Interop.Serialization; using Microsoft.Psi.Interop.Transport; using Microsoft.Psi.Remoting; @@ -20,11 +19,12 @@ public static class Operators /// Type of data stream. /// from which to create endpoint. /// Address with which to create endpoint. + /// The name of the rendezvous stream. /// Rendezvous endpoint. - public static Rendezvous.Endpoint ToRendezvousEndpoint(this TcpWriter writer, string address) + public static Rendezvous.Endpoint ToRendezvousEndpoint(this TcpWriter writer, string address, string streamName) { // Each TcpWriter is an endpoint emitting a single stream - return new Rendezvous.TcpSourceEndpoint(address, writer.Port, new[] { new Rendezvous.Stream(writer.Name, typeof(T)) }); + return new Rendezvous.TcpSourceEndpoint(address, writer.Port, new[] { new Rendezvous.Stream(streamName, typeof(T)) }); } /// @@ -34,11 +34,18 @@ public static Rendezvous.Endpoint ToRendezvousEndpoint(this TcpWriter writ /// from which to create . /// The pipeline to add the component to. /// The deserializer to use to deserialize messages. + /// An optional deallocator for the data. /// An optional parameter indicating whether to use originating times received from the source over the network or to re-timestamp with the current pipeline time upon receiving. - /// An optional name for the TCP source. + /// An optional name for the TCP source component. /// . - public static TcpSource ToTcpSource(this Rendezvous.TcpSourceEndpoint endpoint, Pipeline pipeline, IFormatDeserializer deserializer, bool useSourceOriginatingTimes = true, string name = null) - => new (pipeline, endpoint.Host, endpoint.Port, deserializer, useSourceOriginatingTimes, name); + public static TcpSource ToTcpSource( + this Rendezvous.TcpSourceEndpoint endpoint, + Pipeline pipeline, + IFormatDeserializer deserializer, + Action deallocator = null, + bool useSourceOriginatingTimes = true, + string name = nameof(TcpSource)) + => new (pipeline, endpoint.Host, endpoint.Port, deserializer, deallocator, useSourceOriginatingTimes, name); /// /// Create a rendezvous endpoint from a . @@ -79,9 +86,7 @@ public static Rendezvous.Endpoint ToRendezvousEndpoint(this RemoteClockExporter /// Flag indicating whether or not to post with originating times received over the socket. If false, we ignore them and instead use pipeline's current time. /// . public static NetMQSource ToNetMQSource(this Rendezvous.NetMQSourceEndpoint endpoint, Pipeline pipeline, string topic, IFormatDeserializer deserializer, bool useSourceOriginatingTimes = true) - { - return new NetMQSource(pipeline, topic, endpoint.Address, deserializer, useSourceOriginatingTimes); - } + => new NetMQSource(pipeline, topic, endpoint.Address, deserializer, useSourceOriginatingTimes); /// /// Create a rendezvous endpoint from a . diff --git a/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/Rendezvous.cs b/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/Rendezvous.cs index 7e7a630d2..94b57769c 100644 --- a/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/Rendezvous.cs +++ b/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/Rendezvous.cs @@ -361,9 +361,11 @@ public class Process /// /// Unique name by which to refer to the process. /// Process endpoints. - public Process(string name, IEnumerable endpoints) + /// Optional process version (allowing negotiation of client compatibility). + public Process(string name, IEnumerable endpoints, string version = null) { this.Name = name; + this.Version = version ?? string.Empty; this.endpoints = endpoints.ToList(); } @@ -371,8 +373,9 @@ public Process(string name, IEnumerable endpoints) /// Initializes a new instance of the class. /// /// Unique name by which to refer to the process. - public Process(string name) - : this(name, Enumerable.Empty()) + /// Optional process version (allowing negotiation of client compatibility). + public Process(string name, string version = null) + : this(name, Enumerable.Empty(), version) { } @@ -381,6 +384,11 @@ public Process(string name) /// public string Name { get; private set; } + /// + /// Gets the process version. + /// + public string Version { get; private set; } + /// /// Gets the endpoints. /// diff --git a/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousClient.cs b/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousClient.cs index ed3b7aeaf..58cb028cb 100644 --- a/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousClient.cs +++ b/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousClient.cs @@ -79,7 +79,7 @@ public void Start() this.active = true; new Thread(new ThreadStart(this.ReadFromServer)) { IsBackground = true }.Start(); } - catch (Exception ex) + catch (SocketException ex) { Trace.WriteLine($"Failed to connect to {nameof(RendezvousServer)} (retrying): {ex.Message}"); } diff --git a/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousClient.py b/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousClient.py index 0dfba5b92..0b20b67ad 100644 --- a/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousClient.py +++ b/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousClient.py @@ -6,7 +6,7 @@ # Client which connects to a RendezvousServer and relays rendezvous information. class RendezvousClient: - PROTOCOL_VERSION = 1 + PROTOCOL_VERSION = 2 def __init__(self, host, port = 13331): self.serverAddress = (host, port) @@ -83,9 +83,10 @@ def createRemoteClockExporterEndpoint(host, port): 'port': port, 'streams': [] } - def createProcess(name, endpoints): + def createProcess(name, endpoints, version): return { 'name': name, - 'endpoints': endpoints } + 'endpoints': endpoints, + 'version': version } def start(self, processAddedCallback = None, processRemovedCallback = None): self.socket.connect(self.serverAddress) @@ -112,6 +113,7 @@ def stop(self): def addProcess(self, process): self.__sendByte(1) # add process self.__sendString(process['name']) + self.__sendString(process['version']) self.__sendInt(len(process['endpoints'])) for e in process['endpoints']: self.__sendByte(e['endpoint']) @@ -174,11 +176,12 @@ def __readEndpoint(self): def __readProcess(self): name = self.__readString() + version = self.__readString() numEndpoints = self.__readInt() endpoints = [] for _ in range(numEndpoints): endpoints.append(self.__readEndpoint()) - return RendezvousClient.createProcess(name, endpoints) + return RendezvousClient.createProcess(name, endpoints, version) rendezvous = {} diff --git a/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousRelay.cs b/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousRelay.cs index 73c55dd0b..04ffc8197 100644 --- a/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousRelay.cs +++ b/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousRelay.cs @@ -41,6 +41,7 @@ protected static void WriteAddProcess(Rendezvous.Process process, BinaryWriter w { writer.Write((byte)1); // add writer.Write(process.Name); + writer.Write(process.Version); writer.Write(process.Endpoints.Count()); foreach (var endpoint in process.Endpoints) { @@ -160,7 +161,9 @@ protected bool ReadProcessUpdate(BinaryReader reader) /// Process. private static Rendezvous.Process ReadProcess(BinaryReader reader) { - var process = new Rendezvous.Process(reader.ReadString()); + var processName = reader.ReadString(); + var processVersion = reader.ReadString(); + var process = new Rendezvous.Process(processName, processVersion); // read endpoint info var endpointCount = reader.ReadInt32(); diff --git a/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousServer.cs b/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousServer.cs index 6f020b65e..48bb9eacf 100644 --- a/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousServer.cs +++ b/Sources/Runtime/Microsoft.Psi.Interop/Rendezvous/RendezvousServer.cs @@ -25,7 +25,7 @@ public class RendezvousServer : RendezvousRelay, IDisposable /// /// Protocol version. /// - internal const short ProtocolVersion = 1; + internal const short ProtocolVersion = 2; private readonly int port; private readonly ConcurrentDictionary writers = new (); diff --git a/Sources/Runtime/Microsoft.Psi.Interop/Transport/FileSource.cs b/Sources/Runtime/Microsoft.Psi.Interop/Transport/FileSource.cs index 9e3eab022..9a8a9152c 100644 --- a/Sources/Runtime/Microsoft.Psi.Interop/Transport/FileSource.cs +++ b/Sources/Runtime/Microsoft.Psi.Interop/Transport/FileSource.cs @@ -22,30 +22,27 @@ public class FileSource : Generator /// The pipeline to add the component to. /// File name to which to persist. /// Format serializer with which messages are deserialized. - public FileSource(Pipeline pipeline, string filename, IPersistentFormatDeserializer deserializer) - : base(pipeline, EnumerateFile(filename, deserializer), GetStartTimeFromFile(filename, deserializer)) + /// An optional name for the component. + public FileSource(Pipeline pipeline, string filename, IPersistentFormatDeserializer deserializer, string name = nameof(FileSource)) + : base(pipeline, EnumerateFile(filename, deserializer), GetStartTimeFromFile(filename, deserializer), name: name) { } private static IEnumerator<(T, DateTime)> EnumerateFile(string filename, IPersistentFormatDeserializer deserializer) { - using (FileStream stream = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) + using var stream = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read); + foreach (var record in deserializer.DeserializeRecords(stream)) { - foreach (var record in deserializer.DeserializeRecords(stream)) - { - yield return ((T)record.Item1, record.Item2); - } + yield return ((T)record.Item1, record.Item2); } } private static DateTime GetStartTimeFromFile(string filename, IPersistentFormatDeserializer deserializer) { DateTime startTime; - using (FileStream stream = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - (_, startTime) = deserializer.DeserializeRecords(stream).First(); - return startTime; - } + using var stream = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read); + (_, startTime) = deserializer.DeserializeRecords(stream).First(); + return startTime; } } } diff --git a/Sources/Runtime/Microsoft.Psi.Interop/Transport/FileWriter.cs b/Sources/Runtime/Microsoft.Psi.Interop/Transport/FileWriter.cs index 41bbd9038..8c406ce71 100644 --- a/Sources/Runtime/Microsoft.Psi.Interop/Transport/FileWriter.cs +++ b/Sources/Runtime/Microsoft.Psi.Interop/Transport/FileWriter.cs @@ -13,6 +13,7 @@ namespace Microsoft.Psi.Interop.Transport /// Message type. public class FileWriter : IConsumer, IDisposable { + private readonly string name; private FileStream file; private bool first = true; private dynamic state; @@ -23,8 +24,10 @@ public class FileWriter : IConsumer, IDisposable /// The pipeline to add the component to. /// File name to which to persist. /// Format serializer with which messages are serialized. - public FileWriter(Pipeline pipeline, string filename, IPersistentFormatSerializer serializer) + /// An optional name for the component. + public FileWriter(Pipeline pipeline, string filename, IPersistentFormatSerializer serializer, string name = nameof(FileWriter)) { + this.name = name; this.file = File.Create(filename); this.In = pipeline.CreateReceiver(this, (m, e) => this.WriteRecord(m, e, serializer), nameof(this.In)); this.In.Unsubscribed += _ => serializer.PersistFooter(this.file, this.state); @@ -43,6 +46,9 @@ public void Dispose() } } + /// + public override string ToString() => this.name; + private void WriteRecord(T message, Envelope envelope, IPersistentFormatSerializer serializer) { if (this.first) diff --git a/Sources/Runtime/Microsoft.Psi.Interop/Transport/NetMQSource.cs b/Sources/Runtime/Microsoft.Psi.Interop/Transport/NetMQSource.cs index 01bb9f3dc..eb4eb014e 100644 --- a/Sources/Runtime/Microsoft.Psi.Interop/Transport/NetMQSource.cs +++ b/Sources/Runtime/Microsoft.Psi.Interop/Transport/NetMQSource.cs @@ -20,6 +20,7 @@ public class NetMQSource : IProducer, ISourceComponent, IDisposable private readonly string address; private readonly IFormatDeserializer deserializer; private readonly Pipeline pipeline; + private readonly string name; private readonly bool useSourceOriginatingTimes; private SubscriberSocket socket; @@ -33,9 +34,11 @@ public class NetMQSource : IProducer, ISourceComponent, IDisposable /// Connection string. /// Format deserializer with which messages are deserialized. /// Flag indicating whether or not to post with originating times received over the socket. If false, we ignore them and instead use pipeline's current time. - public NetMQSource(Pipeline pipeline, string topic, string address, IFormatDeserializer deserializer, bool useSourceOriginatingTimes = true) + /// An optional name for the component. + public NetMQSource(Pipeline pipeline, string topic, string address, IFormatDeserializer deserializer, bool useSourceOriginatingTimes = true, string name = nameof(NetMQSource)) { this.pipeline = pipeline; + this.name = name; this.useSourceOriginatingTimes = useSourceOriginatingTimes; this.topic = topic; this.address = address; @@ -74,6 +77,9 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) notifyCompleted(); } + /// + public override string ToString() => this.name; + private void Stop() { if (this.socket != null) diff --git a/Sources/Runtime/Microsoft.Psi.Interop/Transport/NetMQWriter.cs b/Sources/Runtime/Microsoft.Psi.Interop/Transport/NetMQWriter.cs index c20001086..aabfb6c8a 100644 --- a/Sources/Runtime/Microsoft.Psi.Interop/Transport/NetMQWriter.cs +++ b/Sources/Runtime/Microsoft.Psi.Interop/Transport/NetMQWriter.cs @@ -16,6 +16,7 @@ namespace Microsoft.Psi.Interop.Transport public class NetMQWriter : IDisposable { private readonly Pipeline pipeline; + private readonly string name; private readonly IFormatSerializer serializer; private readonly Dictionary topics = new (); @@ -27,9 +28,11 @@ public class NetMQWriter : IDisposable /// The pipeline to add the component to. /// Connection string. /// Format serializer with which messages are serialized. - public NetMQWriter(Pipeline pipeline, string address, IFormatSerializer serializer) + /// An optional name for the component. + public NetMQWriter(Pipeline pipeline, string address, IFormatSerializer serializer, string name = nameof(NetMQWriter)) { this.pipeline = pipeline; + this.name = name; this.Address = address; this.serializer = serializer; this.socket = new PublisherSocket(); @@ -71,6 +74,9 @@ public void Dispose() } } + /// + public override string ToString() => this.name; + private void Receive(T message, Envelope envelope, string topic) { var (bytes, index, length) = this.serializer.SerializeMessage(message, envelope.OriginatingTime); diff --git a/Sources/Runtime/Microsoft.Psi.Interop/Transport/NetMQWriter{T}.cs b/Sources/Runtime/Microsoft.Psi.Interop/Transport/NetMQWriter{T}.cs index b15f5a2dc..80fa9e2cc 100644 --- a/Sources/Runtime/Microsoft.Psi.Interop/Transport/NetMQWriter{T}.cs +++ b/Sources/Runtime/Microsoft.Psi.Interop/Transport/NetMQWriter{T}.cs @@ -3,10 +3,7 @@ namespace Microsoft.Psi.Interop.Transport { - using System; using Microsoft.Psi.Interop.Serialization; - using NetMQ; - using NetMQ.Sockets; /// /// NetMQ (ZeroMQ) publisher component. @@ -21,8 +18,9 @@ public class NetMQWriter : NetMQWriter, IConsumer /// Topic name. /// Connection string. /// Format serializer with which messages are serialized. - public NetMQWriter(Pipeline pipeline, string topic, string address, IFormatSerializer serializer) - : base(pipeline, address, serializer) + /// An optional name for the component. + public NetMQWriter(Pipeline pipeline, string topic, string address, IFormatSerializer serializer, string name = nameof(NetMQWriter)) + : base(pipeline, address, serializer, name) { this.In = this.AddTopic(topic); } diff --git a/Sources/Runtime/Microsoft.Psi.Interop/Transport/TcpSource.cs b/Sources/Runtime/Microsoft.Psi.Interop/Transport/TcpSource.cs index 514302b59..476bbce71 100644 --- a/Sources/Runtime/Microsoft.Psi.Interop/Transport/TcpSource.cs +++ b/Sources/Runtime/Microsoft.Psi.Interop/Transport/TcpSource.cs @@ -18,10 +18,10 @@ namespace Microsoft.Psi.Interop.Transport /// The type of the messages. public class TcpSource : IProducer, ISourceComponent, IDisposable { - private static readonly bool IsDisposableT = typeof(IDisposable).IsAssignableFrom(typeof(T)); private readonly Pipeline pipeline; private readonly string address; private readonly int port; + private readonly Action deallocator; private readonly string name; private readonly TcpClient client; private readonly IFormatDeserializer deserializer; @@ -38,15 +38,31 @@ public class TcpSource : IProducer, ISourceComponent, IDisposable /// The address of the remote server. /// The port on which to connect. /// The deserializer to use to deserialize messages. + /// An optional deallocator for the data. /// An optional parameter indicating whether to use originating times from the source received over the network or to re-timestamp with the current pipeline time upon receiving. - /// An optional name for the TCP source. - public TcpSource(Pipeline pipeline, string address, int port, IFormatDeserializer deserializer, bool useSourceOriginatingTimes = true, string name = null) + /// An optional name for the component. + public TcpSource( + Pipeline pipeline, + string address, + int port, + IFormatDeserializer deserializer, + Action deallocator = null, + bool useSourceOriginatingTimes = true, + string name = nameof(TcpSource)) { this.pipeline = pipeline; this.client = new TcpClient(); this.address = address; this.port = port; this.deserializer = deserializer; + this.deallocator = deallocator ?? (d => + { + if (d is IDisposable disposable) + { + disposable.Dispose(); + } + }); + this.useSourceOriginatingTimes = useSourceOriginatingTimes; this.name = name; this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); @@ -79,7 +95,7 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) } /// - public override string ToString() => this.name ?? base.ToString(); + public override string ToString() => this.name; /// /// Reads a data frame into the frame buffer. Will re-allocate the frame buffer if necessary. @@ -140,12 +156,7 @@ private void ReadFrames() lastTimestamp = timestamp, (message, timestamp) = this.ReadNextFrame(reader)) { this.Out.Post(message, timestamp); - - if (IsDisposableT) - { - // message is deep-cloned on Post, so dispose it if IDisposable - ((IDisposable)message).Dispose(); - } + this.deallocator(message); } } catch diff --git a/Sources/Runtime/Microsoft.Psi.Interop/Transport/TcpWriter.cs b/Sources/Runtime/Microsoft.Psi.Interop/Transport/TcpWriter.cs index 601506f0e..47b01fae2 100644 --- a/Sources/Runtime/Microsoft.Psi.Interop/Transport/TcpWriter.cs +++ b/Sources/Runtime/Microsoft.Psi.Interop/Transport/TcpWriter.cs @@ -18,6 +18,7 @@ namespace Microsoft.Psi.Interop.Transport public class TcpWriter : IConsumer, IDisposable { private readonly IFormatSerializer serializer; + private readonly string name; private TcpListener listener; private NetworkStream networkStream; @@ -26,24 +27,19 @@ public class TcpWriter : IConsumer, IDisposable /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - /// Name by which to refer to the data stream. /// The connection port. /// The serializer to use to serialize messages. - public TcpWriter(Pipeline pipeline, string name, int port, IFormatSerializer serializer) + /// An optional name for the component. + public TcpWriter(Pipeline pipeline, int port, IFormatSerializer serializer, string name = nameof(TcpWriter)) { this.serializer = serializer; - this.Name = name; + this.name = name; this.Port = port; this.In = pipeline.CreateReceiver(this, this.Receive, nameof(this.In)); this.listener = new TcpListener(IPAddress.Any, port); this.Start(); } - /// - /// Gets the name by which to refer to the data stream. - /// - public string Name { get; private set; } - /// /// Gets the connection port. /// @@ -60,6 +56,9 @@ public void Dispose() this.listener = null; } + /// + public override string ToString() => this.name; + private void Receive(T message, Envelope envelope) { (var bytes, int offset, int count) = this.serializer.SerializeMessage(message, envelope.OriginatingTime); diff --git a/Sources/Runtime/Microsoft.Psi/Common/Interpolators/AdjacentValuesInterpolator.cs b/Sources/Runtime/Microsoft.Psi/Common/Interpolators/AdjacentValuesInterpolator.cs index 30f07ca82..0a49b5fb9 100644 --- a/Sources/Runtime/Microsoft.Psi/Common/Interpolators/AdjacentValuesInterpolator.cs +++ b/Sources/Runtime/Microsoft.Psi/Common/Interpolators/AdjacentValuesInterpolator.cs @@ -23,6 +23,7 @@ public class AdjacentValuesInterpolator : ReproducibleInterpolator interpolator; private readonly bool orDefault; private readonly TOut defaultValue; + private readonly string name; /// /// Initializes a new instance of the class. @@ -31,11 +32,15 @@ public class AdjacentValuesInterpolator : ReproducibleInterpolator /// Indicates whether to output a default value when no result is found. /// An optional default value to use. - public AdjacentValuesInterpolator(Func interpolator, bool orDefault, TOut defaultValue = default) + /// An optional name for the interpolator (defaults to AdjacentValuesInterpolator). + public AdjacentValuesInterpolator(Func interpolator, bool orDefault, TOut defaultValue = default, string name = null) { this.interpolator = interpolator; this.orDefault = orDefault; this.defaultValue = defaultValue; + + name ??= "AdjacentValues"; + this.name = this.orDefault ? $"{nameof(Reproducible)}.{name}OrDefault" : $"{nameof(Reproducible)}.{name}"; } /// @@ -111,5 +116,8 @@ public override InterpolationResult Interpolate(DateTime interpolationTime return InterpolationResult.InsufficientData(); } } + + /// + public override string ToString() => this.name; } } diff --git a/Sources/Runtime/Microsoft.Psi/Common/Interpolators/FirstAvailableInterpolator.cs b/Sources/Runtime/Microsoft.Psi/Common/Interpolators/FirstAvailableInterpolator.cs index 1cce37345..4974e9a4d 100644 --- a/Sources/Runtime/Microsoft.Psi/Common/Interpolators/FirstAvailableInterpolator.cs +++ b/Sources/Runtime/Microsoft.Psi/Common/Interpolators/FirstAvailableInterpolator.cs @@ -19,6 +19,7 @@ public sealed class FirstAvailableInterpolator : GreedyInterpolator private readonly RelativeTimeInterval relativeTimeInterval; private readonly bool orDefault; private readonly T defaultValue; + private readonly string name; /// /// Initializes a new instance of the class. @@ -41,6 +42,9 @@ public FirstAvailableInterpolator(RelativeTimeInterval relativeTimeInterval, boo this.relativeTimeInterval = relativeTimeInterval; this.orDefault = orDefault; this.defaultValue = defaultValue; + this.name = + (this.orDefault ? $"{nameof(Available)}.{nameof(Available.FirstOrDefault)}" : $"{nameof(Available)}.{nameof(Available.First)}") + + this.relativeTimeInterval.ToString(); } /// @@ -101,5 +105,8 @@ public override InterpolationResult Interpolate(DateTime interpolationTime, I InterpolationResult.DoesNotExist(windowLeft); } } + + /// + public override string ToString() => this.name; } } diff --git a/Sources/Runtime/Microsoft.Psi/Common/Interpolators/FirstReproducibleInterpolator.cs b/Sources/Runtime/Microsoft.Psi/Common/Interpolators/FirstReproducibleInterpolator.cs index 19b8bbe7e..6a99e084f 100644 --- a/Sources/Runtime/Microsoft.Psi/Common/Interpolators/FirstReproducibleInterpolator.cs +++ b/Sources/Runtime/Microsoft.Psi/Common/Interpolators/FirstReproducibleInterpolator.cs @@ -21,6 +21,7 @@ public sealed class FirstReproducibleInterpolator : ReproducibleInterpolator< private readonly RelativeTimeInterval relativeTimeInterval; private readonly bool orDefault; private readonly T defaultValue; + private readonly string name; /// /// Initializes a new instance of the class. @@ -43,6 +44,9 @@ public FirstReproducibleInterpolator(RelativeTimeInterval relativeTimeInterval, this.relativeTimeInterval = relativeTimeInterval; this.orDefault = orDefault; this.defaultValue = defaultValue; + this.name = + (this.orDefault ? $"{nameof(Reproducible)}.{nameof(Reproducible.FirstOrDefault)}" : $"{nameof(Reproducible)}.{nameof(Reproducible.First)}") + + this.relativeTimeInterval.ToString(); } /// @@ -128,5 +132,8 @@ public override InterpolationResult Interpolate(DateTime interpolationTime, I return InterpolationResult.InsufficientData(); } } + + /// + public override string ToString() => this.name; } } \ No newline at end of file diff --git a/Sources/Runtime/Microsoft.Psi/Common/Interpolators/LastAvailableInterpolator.cs b/Sources/Runtime/Microsoft.Psi/Common/Interpolators/LastAvailableInterpolator.cs index e325ae7d8..0b1907d68 100644 --- a/Sources/Runtime/Microsoft.Psi/Common/Interpolators/LastAvailableInterpolator.cs +++ b/Sources/Runtime/Microsoft.Psi/Common/Interpolators/LastAvailableInterpolator.cs @@ -19,6 +19,7 @@ public sealed class LastAvailableInterpolator : GreedyInterpolator private readonly RelativeTimeInterval relativeTimeInterval; private readonly bool orDefault; private readonly T defaultValue; + private readonly string name; /// /// Initializes a new instance of the class. @@ -31,6 +32,9 @@ public LastAvailableInterpolator(RelativeTimeInterval relativeTimeInterval, bool this.relativeTimeInterval = relativeTimeInterval; this.orDefault = orDefault; this.defaultValue = defaultValue; + this.name = + (this.orDefault ? $"{nameof(Available)}.{nameof(Available.LastOrDefault)}" : $"{nameof(Available)}.{nameof(Available.Last)}") + + this.relativeTimeInterval.ToString(); } /// @@ -90,5 +94,8 @@ public override InterpolationResult Interpolate(DateTime interpolationTime, I InterpolationResult.DoesNotExist(windowLeft); } } + + /// + public override string ToString() => this.name; } } \ No newline at end of file diff --git a/Sources/Runtime/Microsoft.Psi/Common/Interpolators/LastReproducibleInterpolator.cs b/Sources/Runtime/Microsoft.Psi/Common/Interpolators/LastReproducibleInterpolator.cs index 0e883da87..5288b9549 100644 --- a/Sources/Runtime/Microsoft.Psi/Common/Interpolators/LastReproducibleInterpolator.cs +++ b/Sources/Runtime/Microsoft.Psi/Common/Interpolators/LastReproducibleInterpolator.cs @@ -21,6 +21,7 @@ public sealed class LastReproducibleInterpolator : ReproducibleInterpolator /// Initializes a new instance of the class. @@ -33,6 +34,9 @@ public LastReproducibleInterpolator(RelativeTimeInterval relativeTimeInterval, b this.relativeTimeInterval = relativeTimeInterval; this.orDefault = orDefault; this.defaultValue = defaultValue; + this.name = + (this.orDefault ? $"{nameof(Reproducible)}.{nameof(Available.LastOrDefault)}" : $"{nameof(Reproducible)}.{nameof(Available.Last)}") + + this.relativeTimeInterval.ToString(); } /// @@ -135,5 +139,8 @@ public override InterpolationResult Interpolate(DateTime interpolationTime, I return InterpolationResult.InsufficientData(); } } + + /// + public override string ToString() => this.name; } } diff --git a/Sources/Runtime/Microsoft.Psi/Common/Interpolators/NearestAvailableInterpolator.cs b/Sources/Runtime/Microsoft.Psi/Common/Interpolators/NearestAvailableInterpolator.cs index 8fdaeba69..543926024 100644 --- a/Sources/Runtime/Microsoft.Psi/Common/Interpolators/NearestAvailableInterpolator.cs +++ b/Sources/Runtime/Microsoft.Psi/Common/Interpolators/NearestAvailableInterpolator.cs @@ -19,6 +19,7 @@ public sealed class NearestAvailableInterpolator : GreedyInterpolator private readonly RelativeTimeInterval relativeTimeInterval; private readonly bool orDefault; private readonly T defaultValue; + private readonly string name; /// /// Initializes a new instance of the class. @@ -31,6 +32,17 @@ public NearestAvailableInterpolator(RelativeTimeInterval relativeTimeInterval, b this.relativeTimeInterval = relativeTimeInterval; this.orDefault = orDefault; this.defaultValue = defaultValue; + + if (this.relativeTimeInterval == RelativeTimeInterval.Zero) + { + this.name = this.orDefault ? $"{nameof(Available)}.{nameof(Available.ExactOrDefault)}" : $"{nameof(Available)}.{nameof(Available.Exact)}"; + } + else + { + this.name = + (this.orDefault ? $"{nameof(Available)}.{nameof(Available.NearestOrDefault)}" : $"{nameof(Available)}.{nameof(Available.Nearest)}") + + this.relativeTimeInterval.ToString(); + } } /// @@ -92,5 +104,8 @@ public override InterpolationResult Interpolate(DateTime interpolationTime, I InterpolationResult.DoesNotExist(upperBound); } } + + /// + public override string ToString() => this.name; } } \ No newline at end of file diff --git a/Sources/Runtime/Microsoft.Psi/Common/Interpolators/NearestReproducibleInterpolator.cs b/Sources/Runtime/Microsoft.Psi/Common/Interpolators/NearestReproducibleInterpolator.cs index 1a82c3112..0c01c9c24 100644 --- a/Sources/Runtime/Microsoft.Psi/Common/Interpolators/NearestReproducibleInterpolator.cs +++ b/Sources/Runtime/Microsoft.Psi/Common/Interpolators/NearestReproducibleInterpolator.cs @@ -21,6 +21,7 @@ public sealed class NearestReproducibleInterpolator : ReproducibleInterpolato private readonly RelativeTimeInterval relativeTimeInterval; private readonly bool orDefault; private readonly T defaultValue; + private readonly string name; /// /// Initializes a new instance of the class. @@ -33,6 +34,17 @@ public NearestReproducibleInterpolator(RelativeTimeInterval relativeTimeInterval this.relativeTimeInterval = relativeTimeInterval; this.orDefault = orDefault; this.defaultValue = defaultValue; + + if (this.relativeTimeInterval == RelativeTimeInterval.Zero) + { + this.name = this.orDefault ? $"{nameof(Reproducible)}.{nameof(Reproducible.ExactOrDefault)}" : $"{nameof(Reproducible)}.{nameof(Reproducible.Exact)}"; + } + else + { + this.name = + (this.orDefault ? $"{nameof(Reproducible)}.{nameof(Reproducible.NearestOrDefault)}" : $"{nameof(Reproducible)}.{nameof(Reproducible.Nearest)}") + + this.relativeTimeInterval.ToString(); + } } /// @@ -144,5 +156,8 @@ public override InterpolationResult Interpolate(DateTime interpolationTime, I return InterpolationResult.InsufficientData(); } } + + /// + public override string ToString() => this.name; } } diff --git a/Sources/Runtime/Microsoft.Psi/Common/Intervals/RelativeTimeInterval.cs b/Sources/Runtime/Microsoft.Psi/Common/Intervals/RelativeTimeInterval.cs index 901d343d3..3195f09f4 100644 --- a/Sources/Runtime/Microsoft.Psi/Common/Intervals/RelativeTimeInterval.cs +++ b/Sources/Runtime/Microsoft.Psi/Common/Intervals/RelativeTimeInterval.cs @@ -15,25 +15,25 @@ public class RelativeTimeInterval : Interval public static readonly RelativeTimeInterval Infinite = - new RelativeTimeInterval(TimeSpan.MinValue, false, false, TimeSpan.MaxValue, false, false); + new (TimeSpan.MinValue, false, false, TimeSpan.MaxValue, false, false); /// /// Canonical empty instance (bounded, non-inclusive, single point). /// public static readonly RelativeTimeInterval Empty = - new RelativeTimeInterval(TimeSpan.Zero, false, true, TimeSpan.Zero, false, true); + new (TimeSpan.Zero, false, true, TimeSpan.Zero, false, true); /// /// Zero interval (unbounded but inclusive, zero value). /// public static readonly RelativeTimeInterval Zero = - new RelativeTimeInterval(TimeSpan.Zero, true, true, TimeSpan.Zero, true, true); + new (TimeSpan.Zero, true, true, TimeSpan.Zero, true, true); private static readonly RelativeTimeInterval PastInterval = - new RelativeTimeInterval(TimeSpan.MinValue, false, false, TimeSpan.Zero, true, true); + new (TimeSpan.MinValue, false, false, TimeSpan.Zero, true, true); private static readonly RelativeTimeInterval FutureInterval = - new RelativeTimeInterval(TimeSpan.Zero, true, true, TimeSpan.MaxValue, false, false); + new (TimeSpan.Zero, true, true, TimeSpan.MaxValue, false, false); /// /// Initializes a new instance of the class. @@ -147,18 +147,13 @@ protected override TimeSpan SpanMaxValue /// Relative endpoints. /// Translated interval. public static TimeInterval operator +(DateTime origin, RelativeTimeInterval relative) - { - return new TimeInterval(origin, relative); - } + => new (origin, relative); /// /// Returns a relative time interval describing the past. The returned interval includes the present moment. /// /// A relative time interval describing the past. - public static RelativeTimeInterval Past() - { - return PastInterval; - } + public static RelativeTimeInterval Past() => PastInterval; /// /// Returns a relative time interval of a specified duration in the past. The returned interval includes the present moment. @@ -167,18 +162,13 @@ public static RelativeTimeInterval Past() /// Indicates if the interval should be inclusive of the left endpoint. /// A relative time interval of a specified duration in the past. public static RelativeTimeInterval Past(TimeSpan duration, bool inclusive = true) - { - return new RelativeTimeInterval(-duration, inclusive, true, TimeSpan.Zero, true, true); - } + => new (-duration, inclusive, true, TimeSpan.Zero, true, true); /// /// Returns a relative time interval describing the future. The returned interval includes the present moment. /// /// A relative time interval describing the future. - public static RelativeTimeInterval Future() - { - return FutureInterval; - } + public static RelativeTimeInterval Future() => FutureInterval; /// /// Returns a relative time interval of a specified duration in the future. The returned interval includes the present moment. @@ -187,9 +177,7 @@ public static RelativeTimeInterval Future() /// Indicates if the interval should be inclusive of the right endpoint. /// A relative time interval of a specified duration in the future. public static RelativeTimeInterval Future(TimeSpan duration, bool inclusive = true) - { - return new RelativeTimeInterval(TimeSpan.Zero, true, true, duration, inclusive, true); - } + => new (TimeSpan.Zero, true, true, duration, inclusive, true); /// /// Determine coverage from minimum left to maximum right. @@ -198,9 +186,7 @@ public static RelativeTimeInterval Future(TimeSpan duration, bool inclusive = tr /// Returns negative interval from max to min point when sequence is empty. /// Interval from minimum left to maximum right value. public static RelativeTimeInterval Coverage(IEnumerable intervals) - { - return Coverage(intervals, (left, right) => new RelativeTimeInterval(left, right), RelativeTimeInterval.Empty); - } + => Coverage(intervals, (left, right) => new RelativeTimeInterval(left, right), RelativeTimeInterval.Empty); /// /// Constructor helper for left-bound instances. @@ -209,9 +195,7 @@ public static RelativeTimeInterval Coverage(IEnumerable in /// Whether left point is inclusive. /// A left-bound instance of the class. public static RelativeTimeInterval LeftBounded(TimeSpan left, bool inclusive) - { - return new RelativeTimeInterval(left, inclusive, true, TimeSpan.MaxValue, false, false); - } + => new (left, inclusive, true, TimeSpan.MaxValue, false, false); /// /// Constructor helper for left-bound instances. @@ -219,10 +203,7 @@ public static RelativeTimeInterval LeftBounded(TimeSpan left, bool inclusive) /// Defaults to inclusive. /// Left bound point. /// A left-bound instance of the class. - public static RelativeTimeInterval LeftBounded(TimeSpan left) - { - return LeftBounded(left, true); - } + public static RelativeTimeInterval LeftBounded(TimeSpan left) => LeftBounded(left, true); /// /// Constructor helper for right-bound instances. @@ -231,9 +212,7 @@ public static RelativeTimeInterval LeftBounded(TimeSpan left) /// Whether right point is inclusive. /// A right-bound instance of the class. public static RelativeTimeInterval RightBounded(TimeSpan right, bool inclusive) - { - return new RelativeTimeInterval(TimeSpan.MinValue, false, false, right, inclusive, true); - } + => new (TimeSpan.MinValue, false, false, right, inclusive, true); /// /// Constructor helper for right-bound instances. @@ -241,10 +220,7 @@ public static RelativeTimeInterval RightBounded(TimeSpan right, bool inclusive) /// Defaults to inclusive. /// Right bound point. /// A right-bound instance of the class. - public static RelativeTimeInterval RightBounded(TimeSpan right) - { - return RightBounded(right, true); - } + public static RelativeTimeInterval RightBounded(TimeSpan right) => RightBounded(right, true); /// /// Translate by a span distance. @@ -253,9 +229,7 @@ public static RelativeTimeInterval RightBounded(TimeSpan right) /// Span by which to translate. /// Translated interval. public override RelativeTimeInterval Translate(TimeSpan span) - { - return this.Translate(span, (lp, li, lb, rp, ri, rb) => new RelativeTimeInterval(lp, li, lb, rp, ri, rb)); - } + => this.Translate(span, (lp, li, lb, rp, ri, rb) => new RelativeTimeInterval(lp, li, lb, rp, ri, rb)); /// /// Scale endpoints by span distances. @@ -264,9 +238,7 @@ public override RelativeTimeInterval Translate(TimeSpan span) /// Span by which to scale right. /// Scaled interval. public override RelativeTimeInterval Scale(TimeSpan left, TimeSpan right) - { - return this.Scale(left, right, (lp, li, lb, rp, ri, rb) => new RelativeTimeInterval(lp, li, lb, rp, ri, rb)); - } + => this.Scale(left, right, (lp, li, lb, rp, ri, rb) => new RelativeTimeInterval(lp, li, lb, rp, ri, rb)); /// /// Scale endpoints by factors. @@ -275,32 +247,35 @@ public override RelativeTimeInterval Scale(TimeSpan left, TimeSpan right) /// Factor by which to scale right. /// Scaled interval. public override RelativeTimeInterval Scale(float left, float right) - { - return this.Scale(left, right, (lp, li, lb, rp, ri, rb) => new RelativeTimeInterval(lp, li, lb, rp, ri, rb)); - } + => this.Scale(left, right, (lp, li, lb, rp, ri, rb) => new RelativeTimeInterval(lp, li, lb, rp, ri, rb)); /// public override bool Equals(object obj) - { - return (obj is RelativeTimeInterval other) && this.Equals(other); - } + => (obj is RelativeTimeInterval other) && this.Equals(other); /// public bool Equals(RelativeTimeInterval other) - { - return - (this.LeftEndpoint.Point, this.LeftEndpoint.Inclusive, this.LeftEndpoint.Bounded, this.RightEndpoint.Point, this.RightEndpoint.Inclusive, this.RightEndpoint.Bounded) == + => (this.LeftEndpoint.Point, this.LeftEndpoint.Inclusive, this.LeftEndpoint.Bounded, this.RightEndpoint.Point, this.RightEndpoint.Inclusive, this.RightEndpoint.Bounded) == (other.LeftEndpoint.Point, other.LeftEndpoint.Inclusive, other.LeftEndpoint.Bounded, other.RightEndpoint.Point, other.RightEndpoint.Inclusive, other.RightEndpoint.Bounded); - } /// - public override int GetHashCode() => ( - this.LeftEndpoint.Point, - this.LeftEndpoint.Inclusive, - this.LeftEndpoint.Bounded, - this.RightEndpoint.Point, - this.RightEndpoint.Inclusive, - this.RightEndpoint.Bounded).GetHashCode(); + public override int GetHashCode() + => (this.LeftEndpoint.Point, + this.LeftEndpoint.Inclusive, + this.LeftEndpoint.Bounded, + this.RightEndpoint.Point, + this.RightEndpoint.Inclusive, + this.RightEndpoint.Bounded).GetHashCode(); + + /// + public override string ToString() + { + var openParens = this.LeftEndpoint.Inclusive ? "[" : "("; + var min = this.TimeSpanToString(this.LeftEndpoint.Point); + var max = this.TimeSpanToString(this.RightEndpoint.Point); + var closeParens = this.RightEndpoint.Inclusive ? "]" : ")"; + return $"{openParens}{min},{max}{closeParens}"; + } /// /// Scale a span by a given factor. @@ -309,19 +284,14 @@ public bool Equals(RelativeTimeInterval other) /// Factor by which to scale. /// Scaled span. protected override TimeSpan ScaleSpan(TimeSpan span, double factor) - { - return new TimeSpan((long)Math.Round(span.Ticks * factor)); - } + => new ((long)Math.Round(span.Ticks * factor)); /// /// Negate span. /// /// Span to be negated. /// Negated span. - protected override TimeSpan NegateSpan(TimeSpan span) - { - return -span; - } + protected override TimeSpan NegateSpan(TimeSpan span) => -span; /// /// Translate point by given span. @@ -329,10 +299,7 @@ protected override TimeSpan NegateSpan(TimeSpan span) /// Point value. /// Span by which to translate. /// Translated point. - protected override TimeSpan TranslatePoint(TimeSpan point, TimeSpan span) - { - return point + span; - } + protected override TimeSpan TranslatePoint(TimeSpan point, TimeSpan span) => point + span; /// /// Determine span between two given points. @@ -340,10 +307,7 @@ protected override TimeSpan TranslatePoint(TimeSpan point, TimeSpan span) /// First point. /// Second point. /// Span between points. - protected override TimeSpan Difference(TimeSpan x, TimeSpan y) - { - return x - y; - } + protected override TimeSpan Difference(TimeSpan x, TimeSpan y) => x - y; /// /// Compare points. @@ -351,9 +315,30 @@ protected override TimeSpan Difference(TimeSpan x, TimeSpan y) /// First point. /// Second point. /// Less (-1), greater (+1) or equal (0). - protected override int ComparePoints(TimeSpan a, TimeSpan b) + protected override int ComparePoints(TimeSpan a, TimeSpan b) => a.CompareTo(b); + + private string TimeSpanToString(TimeSpan timeSpan) { - return a.CompareTo(b); + if (timeSpan == TimeSpan.Zero) + { + return "0"; + } + else if (timeSpan == TimeSpan.MinValue) + { + return double.NegativeInfinity.ToString(); + } + else if (timeSpan == TimeSpan.MaxValue) + { + return double.PositiveInfinity.ToString(); + } + else if (timeSpan.TotalMilliseconds < 1000 && timeSpan.TotalMilliseconds == Math.Floor(timeSpan.TotalMilliseconds)) + { + return $"{(int)timeSpan.TotalMilliseconds}ms"; + } + else + { + return timeSpan.ToString(); + } } } } \ No newline at end of file diff --git a/Sources/Runtime/Microsoft.Psi/Common/PsiStreamMetadata.cs b/Sources/Runtime/Microsoft.Psi/Common/PsiStreamMetadata.cs index 44051cb0d..fea6ab4e8 100644 --- a/Sources/Runtime/Microsoft.Psi/Common/PsiStreamMetadata.cs +++ b/Sources/Runtime/Microsoft.Psi/Common/PsiStreamMetadata.cs @@ -233,16 +233,13 @@ public T GetSupplementalMetadata() /// /// Sets supplemental stream metadata. /// - /// Type of supplemental metadata. - /// Supplemental metadata value. - /// Known serializers. - internal void SetSupplementalMetadata(T value, KnownSerializers serializers) + /// The serialized supplemental metadata bytes. + /// The supplemental metadata type name. + internal void SetSupplementalMetadata(string supplementalMetadataTypeName, byte[] supplementalMetadataBytes) { - this.SupplementalMetadataTypeName = typeof(T).AssemblyQualifiedName; - var handler = serializers.GetHandler(); - var writer = new BufferWriter(this.supplementalMetadataBytes); - handler.Serialize(writer, value, new SerializationContext(serializers)); - this.supplementalMetadataBytes = writer.Buffer; + this.SupplementalMetadataTypeName = supplementalMetadataTypeName; + this.supplementalMetadataBytes = new byte[supplementalMetadataBytes.Length]; + Array.Copy(supplementalMetadataBytes, this.supplementalMetadataBytes, supplementalMetadataBytes.Length); } /// diff --git a/Sources/Runtime/Microsoft.Psi/Components/Aggregator.cs b/Sources/Runtime/Microsoft.Psi/Components/Aggregator.cs index 107d2b6a4..25cd97ee2 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/Aggregator.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/Aggregator.cs @@ -13,19 +13,20 @@ namespace Microsoft.Psi.Components /// The output message type. public class Aggregator : ConsumerProducer, IDisposable { - private Func, TState> aggregator; + private readonly Func, TState> aggregator; private TState state; /// /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - /// Initial state. + /// Initial state. /// Aggregation function. - public Aggregator(Pipeline pipeline, TState init, Func, TState> aggregator) - : base(pipeline) + /// An optional name for this component. + public Aggregator(Pipeline pipeline, TState initialState, Func, TState> aggregator, string name = nameof(Aggregator)) + : base(pipeline, name) { - this.state = init; + this.state = initialState; this.aggregator = aggregator; } diff --git a/Sources/Runtime/Microsoft.Psi/Components/AsyncConsumerProducer.cs b/Sources/Runtime/Microsoft.Psi/Components/AsyncConsumerProducer.cs index 5dcadbeb1..70cbe2ec6 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/AsyncConsumerProducer.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/AsyncConsumerProducer.cs @@ -13,12 +13,16 @@ namespace Microsoft.Psi.Components /// The output message type. public abstract class AsyncConsumerProducer : IConsumerProducer { + private readonly string name; + /// /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - public AsyncConsumerProducer(Pipeline pipeline) + /// An optional name for the component. + public AsyncConsumerProducer(Pipeline pipeline, string name = nameof(AsyncConsumerProducer)) { + this.name = name; this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); this.In = pipeline.CreateAsyncReceiver(this, this.ReceiveAsync, nameof(this.In)); } @@ -29,6 +33,9 @@ public AsyncConsumerProducer(Pipeline pipeline) /// public Emitter Out { get; } + /// + public override string ToString() => this.name; + /// /// Async receiver to be implemented by subclass. /// diff --git a/Sources/Runtime/Microsoft.Psi/Components/ConsumerProducer.cs b/Sources/Runtime/Microsoft.Psi/Components/ConsumerProducer.cs index f8be5e949..5f18990e9 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/ConsumerProducer.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/ConsumerProducer.cs @@ -6,19 +6,23 @@ namespace Microsoft.Psi.Components /// /// This is the base class for any component that transforms an input type into an output type. /// Derive from thsi class if your component has more than one input or more than one output. - /// Otherwise, use one of the the - /// or operators. + /// Otherwise, use one of the the + /// or operators. /// /// The input message type. /// The output message type. public abstract class ConsumerProducer : IConsumerProducer { + private readonly string name; + /// /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - public ConsumerProducer(Pipeline pipeline) + /// An optional name for this component. + public ConsumerProducer(Pipeline pipeline, string name = nameof(ConsumerProducer)) { + this.name = name; this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); this.In = pipeline.CreateReceiver(this, this.Receive, nameof(this.In)); } @@ -33,6 +37,9 @@ public ConsumerProducer(Pipeline pipeline) /// public Emitter Out { get; } + /// + public override string ToString() => this.name; + /// /// Override this method to process the incomming message and potentially publish one or more output messages. /// The input message payload is only valid for the duration of the call. diff --git a/Sources/Runtime/Microsoft.Psi/Components/DynamicWindow.cs b/Sources/Runtime/Microsoft.Psi/Components/DynamicWindow.cs index cdbc9d03a..dcce4d2cf 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/DynamicWindow.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/DynamicWindow.cs @@ -1,156 +1,158 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace Microsoft.Psi.Components -{ - using System; +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Components +{ + using System; using System.Collections.Generic; - using System.Linq; - using Microsoft.Psi; - - /// - /// Component that implements a dynamic window stream operator. - /// - /// The type of messages on the window stream. - /// The type of messages on the input stream. - /// The type of messages on the output stream. - /// The component implements a dynamic window operator over a stream of data. Messages - /// on the incoming stream are used to compute a relative time - /// interval over the in input stream. The output is created by a function that has access - /// to the window message and the computed buffer of messages on the input stream. - public class DynamicWindow : ConsumerProducer - { - private readonly List> windowBuffer = new List>(); - private readonly List> inputBuffer = new List>(); - private readonly Func, (TimeInterval Window, DateTime ObsoleteTime)> dynamicWindowFunction; - private readonly Func, IEnumerable>, TOutput> outputCreator; - - private DateTime minimumObsoleteTime = DateTime.MinValue; - - /// - /// Initializes a new instance of the class. - /// - /// The pipeline to add the component to. - /// The function that creates the actual window to use at every point, and specified the time point previous to which no future windows will extend. - /// A function that creates output messages given a message on the window-defining stream and a buffer of messages on the source stream. - public DynamicWindow( - Pipeline pipeline, - Func, (TimeInterval, DateTime)> windowCreator, - Func, IEnumerable>, TOutput> outputCreator) - : base(pipeline) - { - this.dynamicWindowFunction = windowCreator; - this.outputCreator = outputCreator; - this.WindowIn = pipeline.CreateReceiver(this, this.ReceiveWindow, nameof(this.WindowIn)); - this.In.Unsubscribed += _ => this.Publish(true); - } - - /// - /// Gets the received for the input stream of window messages. - /// - public Receiver WindowIn { get; } - - /// - protected override void Receive(TInput data, Envelope envelope) - { - this.inputBuffer.Add(Message.Create(data.DeepClone(this.In.Recycler), envelope)); - this.Publish(false); - } - - private void ReceiveWindow(TWindow data, Envelope envelope) - { - this.windowBuffer.Add(Message.Create(data.DeepClone(this.WindowIn.Recycler), envelope)); - this.Publish(false); - } - - private void Publish(bool final) - { - while (this.TryPublish(final)) - { - } - } - - private bool TryPublish(bool final) - { - if (this.windowBuffer.Count == 0) - { - return false; - } - - (var timeInterval, var obsoleteTime) = this.dynamicWindowFunction(this.windowBuffer[0]); + using System.Linq; + using Microsoft.Psi; + + /// + /// Component that implements a dynamic window stream operator. + /// + /// The type of messages on the window stream. + /// The type of messages on the input stream. + /// The type of messages on the output stream. + /// The component implements a dynamic window operator over a stream of data. Messages + /// on the incoming stream are used to compute a relative time + /// interval over the in input stream. The output is created by a function that has access + /// to the window message and the computed buffer of messages on the input stream. + public class DynamicWindow : ConsumerProducer + { + private readonly List> windowBuffer = new (); + private readonly List> inputBuffer = new (); + private readonly Func, (TimeInterval Window, DateTime ObsoleteTime)> dynamicWindowFunction; + private readonly Func, IEnumerable>, TOutput> outputCreator; + + private DateTime minimumObsoleteTime = DateTime.MinValue; + + /// + /// Initializes a new instance of the class. + /// + /// The pipeline to add the component to. + /// The function that creates the actual window to use at every point, and specified the time point previous to which no future windows will extend. + /// A function that creates output messages given a message on the window-defining stream and a buffer of messages on the source stream. + /// An optional name for the component. + public DynamicWindow( + Pipeline pipeline, + Func, (TimeInterval, DateTime)> windowCreator, + Func, IEnumerable>, TOutput> outputCreator, + string name = nameof(DynamicWindow)) + : base(pipeline, name) + { + this.dynamicWindowFunction = windowCreator; + this.outputCreator = outputCreator; + this.WindowIn = pipeline.CreateReceiver(this, this.ReceiveWindow, nameof(this.WindowIn)); + this.In.Unsubscribed += _ => this.Publish(true); + } + + /// + /// Gets the received for the input stream of window messages. + /// + public Receiver WindowIn { get; } + + /// + protected override void Receive(TInput data, Envelope envelope) + { + this.inputBuffer.Add(Message.Create(data.DeepClone(this.In.Recycler), envelope)); + this.Publish(false); + } + + private void ReceiveWindow(TWindow data, Envelope envelope) + { + this.windowBuffer.Add(Message.Create(data.DeepClone(this.WindowIn.Recycler), envelope)); + this.Publish(false); + } + + private void Publish(bool final) + { + while (this.TryPublish(final)) + { + } + } + + private bool TryPublish(bool final) + { + if (this.windowBuffer.Count == 0) + { + return false; + } + + (var timeInterval, var obsoleteTime) = this.dynamicWindowFunction(this.windowBuffer[0]); if (timeInterval.IsNegative) { throw new ArgumentException("Dynamic window must be a positive time interval."); - } - + } + if (timeInterval.Left < this.minimumObsoleteTime) { throw new ArgumentException("Dynamic window must not extend before previous obsolete time."); - } - + } + if (!timeInterval.IsFinite) { - throw new ArgumentException("Dynamic window must be finite (bounded at both ends)."); - } - - if (!final && (this.inputBuffer.Count == 0 || this.inputBuffer[this.inputBuffer.Count - 1].OriginatingTime < timeInterval.RightEndpoint.Point)) + throw new ArgumentException("Dynamic window must be finite (bounded at both ends)."); + } + + if (!final && (this.inputBuffer.Count == 0 || this.inputBuffer[this.inputBuffer.Count - 1].OriginatingTime < timeInterval.RightEndpoint.Point)) + { + return false; + } + + // if we have enough data, find the index of where to start and where to end + var startIndex = this.inputBuffer.FindIndex(m => timeInterval.PointIsWithin(m.OriginatingTime)); + var endIndex = this.inputBuffer.FindLastIndex(m => timeInterval.PointIsWithin(m.OriginatingTime)); + + // if endIndex is -1 (all inputBuffer messages are after the time interval) + if (endIndex == -1) + { + // then post an empty buffer + this.PostAndClearObsoleteInputs(obsoleteTime, Enumerable.Empty>()); + return true; + } + else if (startIndex == -1) { + // o/w if the startIndex is -1 (all inputBuffer messages are before the time interval) + // we cannot post yet, we are still waiting for data messages in the temporal range of the + // entity, so return false return false; - } - - // if we have enough data, find the index of where to start and where to end - var startIndex = this.inputBuffer.FindIndex(m => timeInterval.PointIsWithin(m.OriginatingTime)); - var endIndex = this.inputBuffer.FindLastIndex(m => timeInterval.PointIsWithin(m.OriginatingTime)); - - // if endIndex is -1 (all inputBuffer messages are after the time interval) - if (endIndex == -1) - { - // then post an empty buffer - this.PostAndClearObsoleteInputs(obsoleteTime, Enumerable.Empty>()); - return true; - } - else if (startIndex == -1) - { - // o/w if the startIndex is -1 (all inputBuffer messages are before the time interval) - // we cannot post yet, we are still waiting for data messages in the temporal range of the - // entity, so return false - return false; - } - else if (endIndex >= startIndex) - { - // o/w if the endIndex is strictly larger than the start index, then we have some overlap - this.PostAndClearObsoleteInputs(obsoleteTime, this.inputBuffer.GetRange(startIndex, endIndex - startIndex + 1)); - return true; - } - else - { - // o/w if the endindex is strictly smaller than the startindex, that means the temporal interval - // is caught in between the two different indices (endindex -> startindex) - // in this case, we can post an empty buffer - this.PostAndClearObsoleteInputs(obsoleteTime, Enumerable.Empty>()); - return true; - } - } - - private void PostAndClearObsoleteInputs(DateTime obsoleteTime, IEnumerable> inputs) - { - // check that obsolete times don't backtrack + } + else if (endIndex >= startIndex) + { + // o/w if the endIndex is strictly larger than the start index, then we have some overlap + this.PostAndClearObsoleteInputs(obsoleteTime, this.inputBuffer.GetRange(startIndex, endIndex - startIndex + 1)); + return true; + } + else + { + // o/w if the endindex is strictly smaller than the startindex, that means the temporal interval + // is caught in between the two different indices (endindex -> startindex) + // in this case, we can post an empty buffer + this.PostAndClearObsoleteInputs(obsoleteTime, Enumerable.Empty>()); + return true; + } + } + + private void PostAndClearObsoleteInputs(DateTime obsoleteTime, IEnumerable> inputs) + { + // check that obsolete times don't backtrack if (obsoleteTime < this.minimumObsoleteTime) { throw new ArgumentException("Dynamic window with obsolete time prior to previous window."); - } - - this.minimumObsoleteTime = obsoleteTime; - - // post output - var sourceMessage = this.windowBuffer[0]; - var value = this.outputCreator(sourceMessage, inputs); - this.Out.Post(value, sourceMessage.OriginatingTime); - - // remove & recycle window and obsolete inputs - this.windowBuffer.RemoveAt(0); - this.WindowIn.Recycler.Recycle(sourceMessage.Data); - + } + + this.minimumObsoleteTime = obsoleteTime; + + // post output + var sourceMessage = this.windowBuffer[0]; + var value = this.outputCreator(sourceMessage, inputs); + this.Out.Post(value, sourceMessage.OriginatingTime); + + // remove & recycle window and obsolete inputs + this.windowBuffer.RemoveAt(0); + this.WindowIn.Recycler.Recycle(sourceMessage.Data); + if (this.inputBuffer.Any()) { var obsoleteIndex = this.inputBuffer.FindIndex(m => m.OriginatingTime >= obsoleteTime); @@ -168,7 +170,7 @@ private void PostAndClearObsoleteInputs(DateTime obsoleteTime, IEnumerable : IProducer, ISourceComponen private readonly Action unsubscribe; private readonly TEventHandler eventHandler; private readonly Pipeline pipeline; + private readonly string name; /// /// Initializes a new instance of the class. @@ -33,13 +34,16 @@ public class EventSource : IProducer, ISourceComponen /// handler of type that will be subscribed to the /// external event by the delegate. /// + /// An optional name for the component. public EventSource( Pipeline pipeline, Action subscribe, Action unsubscribe, - Func, TEventHandler> converter) + Func, TEventHandler> converter, + string name = nameof(EventSource)) { this.pipeline = pipeline; + this.name = name; this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); this.subscribe = subscribe; this.unsubscribe = unsubscribe; @@ -73,6 +77,9 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) notifyCompleted(); } + /// + public override string ToString() => this.name; + /// /// Posts a value on the output stream. /// diff --git a/Sources/Runtime/Microsoft.Psi/Components/Fuse.cs b/Sources/Runtime/Microsoft.Psi/Components/Fuse.cs index cce4de8d2..bbfefc3d4 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/Fuse.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/Fuse.cs @@ -6,7 +6,6 @@ namespace Microsoft.Psi.Components using System; using System.Collections.Generic; using System.Linq; - using Microsoft.Psi.Common.Interpolators; /// /// Component that fuses multiple streams based on a specified interpolator. @@ -18,7 +17,8 @@ namespace Microsoft.Psi.Components public class Fuse : IProducer { private readonly Pipeline pipeline; - private readonly Queue> primaryQueue = new Queue>(); // to be paired + private readonly string name; + private readonly Queue> primaryQueue = new (); // to be paired private readonly Interpolator interpolator; private readonly Func outputCreator; private readonly Func> secondarySelector; @@ -39,15 +39,18 @@ public class Fuse : IProducer /// Mapping function from messages to output. /// Number of secondary streams. /// Selector function mapping primary messages to a set of secondary stream indices. + /// An optional name for the component. public Fuse( Pipeline pipeline, Interpolator interpolator, Func outputCreator, int secondaryCount = 1, - Func> secondarySelector = null) + Func> secondarySelector = null, + string name = null) : base() { this.pipeline = pipeline; + this.name = name ?? $"Fuse({interpolator})"; this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); this.InPrimary = pipeline.CreateReceiver(this, this.ReceivePrimary, nameof(this.InPrimary)); this.interpolator = interpolator; @@ -83,6 +86,9 @@ public class Fuse : IProducer /// public IList> InSecondaries => this.inSecondaries; + /// + public override string ToString() => this.name; + /// /// Add input receiver. /// diff --git a/Sources/Runtime/Microsoft.Psi/Components/Generator.cs b/Sources/Runtime/Microsoft.Psi/Components/Generator.cs index c5ef1e331..7b2fc0f7b 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/Generator.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/Generator.cs @@ -28,6 +28,7 @@ public abstract class Generator : ISourceComponent private readonly Receiver loopBackIn; private readonly Emitter loopBackOut; private readonly Pipeline pipeline; + private readonly string name; private readonly PipelineElement node; private readonly bool isInfiniteSource; private bool stopped; @@ -42,12 +43,14 @@ public abstract class Generator : ISourceComponent /// The pipeline to add the component to. /// If true, mark this Generator instance as representing an infinite source (e.g., a live-running sensor). /// If false (default), it represents a finite source (e.g., Generating messages based on a finite file or IEnumerable). - public Generator(Pipeline pipeline, bool isInfiniteSource = false) + /// An optional name for the generator. + public Generator(Pipeline pipeline, bool isInfiniteSource = false, string name = nameof(Generator)) { this.loopBackOut = pipeline.CreateEmitter(this, nameof(this.loopBackOut)); this.loopBackIn = pipeline.CreateReceiver(this, this.Next, nameof(this.loopBackIn)); this.loopBackOut.PipeTo(this.loopBackIn, DeliveryPolicy.Unlimited); this.pipeline = pipeline; + this.name = name; this.node = pipeline.GetOrCreateNode(this); this.isInfiniteSource = isInfiniteSource; } @@ -99,6 +102,9 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) } } + /// + public override string ToString() => this.name; + /// /// Function that gets called to produce more data once the pipeline is ready to consume it. /// Override to post data to the appropriate stream. diff --git a/Sources/Runtime/Microsoft.Psi/Components/Generator{T}.cs b/Sources/Runtime/Microsoft.Psi/Components/Generator{T}.cs index c85952710..04d32d356 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/Generator{T}.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/Generator{T}.cs @@ -30,8 +30,9 @@ public class Generator : Generator, IProducer, IDisposable /// is non-null, the messages will have originating times that align with the specified time. /// If true, mark this Generator instance as representing an infinite source (e.g., a live-running sensor). /// If false (default), it represents a finite source (e.g., Generating messages based on a finite file or IEnumerable). - public Generator(Pipeline pipeline, IEnumerator enumerator, TimeSpan interval, DateTime? alignDateTime = null, bool isInfiniteSource = false) - : this(pipeline, CreateEnumerator(pipeline, enumerator, interval, alignDateTime), null, isInfiniteSource) + /// An optional name for the component. + public Generator(Pipeline pipeline, IEnumerator enumerator, TimeSpan interval, DateTime? alignDateTime = null, bool isInfiniteSource = false, string name = nameof(Generator)) + : this(pipeline, CreateEnumerator(pipeline, enumerator, interval, alignDateTime), null, isInfiniteSource, name) { } @@ -46,8 +47,9 @@ public Generator(Pipeline pipeline, IEnumerator enumerator, TimeSpan interval /// account any other components in the pipeline which may have proposed replay times. /// If true, mark this Generator instance as representing an infinite source (e.g., a live-running sensor). /// If false (default), it represents a finite source (e.g., Generating messages based on a finite file or IEnumerable). - public Generator(Pipeline pipeline, IEnumerator<(T, DateTime)> enumerator, DateTime? startTime = null, bool isInfiniteSource = false) - : base(pipeline, isInfiniteSource) + /// An optional name for the component. + public Generator(Pipeline pipeline, IEnumerator<(T, DateTime)> enumerator, DateTime? startTime = null, bool isInfiniteSource = false, string name = nameof(Generator)) + : base(pipeline, isInfiniteSource, name) { this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); this.enumerator = new Enumerator(enumerator); diff --git a/Sources/Runtime/Microsoft.Psi/Components/Join.cs b/Sources/Runtime/Microsoft.Psi/Components/Join.cs index 295fbedbd..519e58d31 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/Join.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/Join.cs @@ -23,13 +23,15 @@ public class Join : FuseMapping function from message pair to output. /// Number of secondary streams. /// Selector function mapping primary messages to secondary stream indices. + /// An optional name for the component. public Join( Pipeline pipeline, ReproducibleInterpolator interpolator, Func outputCreator, int secondaryCount = 1, - Func> secondarySelector = null) - : base(pipeline, interpolator, outputCreator, secondaryCount, secondarySelector) + Func> secondarySelector = null, + string name = null) + : base(pipeline, interpolator, outputCreator, secondaryCount, secondarySelector, name ?? $"Join({interpolator})") { } } diff --git a/Sources/Runtime/Microsoft.Psi/Components/Join{TPrimary,TSecondary,TOut}.cs b/Sources/Runtime/Microsoft.Psi/Components/Join{TPrimary,TSecondary,TOut}.cs index 836a6bd6b..2262c814b 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/Join{TPrimary,TSecondary,TOut}.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/Join{TPrimary,TSecondary,TOut}.cs @@ -22,13 +22,15 @@ public class Join : FuseMapping function from message pair to output. /// Number of secondary streams. /// Selector function mapping primary messages to secondary stream indices. + /// An optional name for the component. public Join( Pipeline pipeline, ReproducibleInterpolator interpolator, Func outputCreator, int secondaryCount = 1, - Func> secondarySelector = null) - : base(pipeline, interpolator, outputCreator, secondaryCount, secondarySelector) + Func> secondarySelector = null, + string name = null) + : base(pipeline, interpolator, outputCreator, secondaryCount, secondarySelector, name ?? $"Join({interpolator})") { } } diff --git a/Sources/Runtime/Microsoft.Psi/Components/Merge.cs b/Sources/Runtime/Microsoft.Psi/Components/Merge.cs index 89056fa6f..83be407c5 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/Merge.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/Merge.cs @@ -11,14 +11,17 @@ namespace Microsoft.Psi.Components public class Merge : IProducer> { private readonly Pipeline pipeline; + private readonly string name; /// /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - public Merge(Pipeline pipeline) + /// An optional name for this component. + public Merge(Pipeline pipeline, string name = nameof(Merge)) { this.pipeline = pipeline; + this.name = name; this.Out = pipeline.CreateEmitter>(this, nameof(this.Out)); } @@ -37,6 +40,9 @@ public Receiver AddInput(string name) return this.pipeline.CreateReceiver(this, this.Receive, name); } + /// + public override string ToString() => this.name; + private void Receive(T message, Envelope e) { this.Out.Post(Message.Create(message, e), this.pipeline.GetCurrentTime()); diff --git a/Sources/Runtime/Microsoft.Psi/Components/Merger.cs b/Sources/Runtime/Microsoft.Psi/Components/Merger.cs index 8e083dc5a..69e198524 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/Merger.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/Merger.cs @@ -15,17 +15,20 @@ public class Merger { private readonly Dictionary> inputs = new Dictionary>(); private readonly Pipeline pipeline; + private readonly string name; private readonly Action> action; - private readonly object syncRoot = new object(); + private readonly object syncRoot = new (); /// /// Initializes a new instance of the class. /// /// The pipeline to add the component to. /// Action invoked for each key/message. - public Merger(Pipeline pipeline, Action> action) + /// An optional name for the component. + public Merger(Pipeline pipeline, Action> action, string name = nameof(Merger)) { this.pipeline = pipeline; + this.name = name; this.action = action; } @@ -47,5 +50,8 @@ public Receiver Add(TKey key) return this.inputs[key] = this.pipeline.CreateReceiver(this, m => this.action(key, m), key.ToString()); } } + + /// + public override string ToString() => this.name; } } \ No newline at end of file diff --git a/Sources/Runtime/Microsoft.Psi/Components/Pair.cs b/Sources/Runtime/Microsoft.Psi/Components/Pair.cs index 1df1bdb8c..85c9e1d27 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/Pair.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/Pair.cs @@ -13,6 +13,7 @@ namespace Microsoft.Psi.Components /// The type of output message. public class Pair : IProducer { + private readonly string name; private readonly Func outputCreator; private bool secondaryValueReady = false; private TSecondary lastSecondaryValue = default; @@ -22,11 +23,13 @@ public class Pair : IProducer /// /// The pipeline to add the component to. /// Mapping function from primary/secondary stream values to output type. + /// An optional name for the component. public Pair( Pipeline pipeline, - Func outputCreator) - : base() + Func outputCreator, + string name = nameof(Pair)) { + this.name = name; this.outputCreator = outputCreator; this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); this.InPrimary = pipeline.CreateReceiver(this, this.ReceivePrimary, nameof(this.InPrimary)); @@ -39,11 +42,13 @@ public class Pair : IProducer /// The pipeline to add the component to. /// Mapping function from primary/secondary stream values to output type. /// An initial secondary value to be used until the first message arrives on the secondary stream. + /// An optional name for the component. public Pair( Pipeline pipeline, Func outputCreator, - TSecondary initialSecondaryValue) - : this(pipeline, outputCreator) + TSecondary initialSecondaryValue, + string name = nameof(Pair)) + : this(pipeline, outputCreator, name) { this.secondaryValueReady = true; this.lastSecondaryValue = initialSecondaryValue; @@ -64,6 +69,9 @@ public class Pair : IProducer /// public Receiver InSecondary { get; } + /// + public override string ToString() => this.name; + private void ReceivePrimary(TPrimary message, Envelope e) { // drop unless a secondary value has been received or using an initial value diff --git a/Sources/Runtime/Microsoft.Psi/Components/ParallelSparseDo.cs b/Sources/Runtime/Microsoft.Psi/Components/ParallelSparseDo.cs index 98d86de29..0187896fd 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/ParallelSparseDo.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/ParallelSparseDo.cs @@ -26,16 +26,16 @@ public class ParallelSparseDo : Subpipeline, IConsum /// A function that generates a dictionary of key-value pairs for each given input message. /// Action to perform in parallel. /// Predicate function determining whether and when (originating time) to terminate branches (defaults to when key no longer present), given the current key, message payload (dictionary) and originating time. - /// Name for this component (defaults to ParallelSparse). + /// An optional name for the component. /// Pipeline-level default delivery policy to be used by this component (defaults to if unspecified). public ParallelSparseDo( Pipeline pipeline, Func> splitter, Action> action, Func, DateTime, (bool, DateTime)> branchTerminationPolicy = null, - string name = null, + string name = nameof(ParallelSparseDo), DeliveryPolicy defaultDeliveryPolicy = null) - : base(pipeline, name ?? nameof(ParallelSparseDo), defaultDeliveryPolicy) + : base(pipeline, name, defaultDeliveryPolicy) { this.inConnector = this.CreateInputConnectorFrom(pipeline, nameof(this.inConnector)); var parallelSparseSplitter = new ParallelSparseSplitter(this, splitter, action, branchTerminationPolicy); diff --git a/Sources/Runtime/Microsoft.Psi/Components/ParallelSparseSelect.cs b/Sources/Runtime/Microsoft.Psi/Components/ParallelSparseSelect.cs index 2d73d114a..728ab5203 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/ParallelSparseSelect.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/ParallelSparseSelect.cs @@ -48,9 +48,9 @@ public class ParallelSparseSelect bool outputDefaultIfDropped = false, TBranchOut defaultValue = default, Func, DateTime, (bool, DateTime)> branchTerminationPolicy = null, - string name = null, + string name = nameof(ParallelSparseSelect), DeliveryPolicy defaultDeliveryPolicy = null) - : base(pipeline, name ?? nameof(ParallelSparseSelect), defaultDeliveryPolicy) + : base(pipeline, name, defaultDeliveryPolicy) { this.pipeline = pipeline; this.inConnector = this.CreateInputConnectorFrom(pipeline, nameof(this.inConnector)); diff --git a/Sources/Runtime/Microsoft.Psi/Components/ParallelSparseSplitter.cs b/Sources/Runtime/Microsoft.Psi/Components/ParallelSparseSplitter.cs index 828274cb1..c80f92427 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/ParallelSparseSplitter.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/ParallelSparseSplitter.cs @@ -18,6 +18,7 @@ namespace Microsoft.Psi.Components public class ParallelSparseSplitter : IConsumer { private readonly Pipeline pipeline; + private readonly string name; private readonly Dictionary> branches = new Dictionary>(); private readonly Dictionary keyToBranchMapping = new Dictionary(); private readonly Func> splitterFunction; @@ -36,14 +37,17 @@ public class ParallelSparseSplitter : IC /// Function mapping keyed input producers to output producers. /// Predicate function determining whether and when (originating time) to terminate branches (defaults to when key no longer present), given the current key. /// Action that connects the results of a parallel branch back to join. + /// An optional name for the component. public ParallelSparseSplitter( Pipeline pipeline, Func> splitter, Func, IProducer> transform, Func, DateTime, (bool, DateTime)> branchTerminationPolicy, - Action> connectToJoin) + Action> connectToJoin, + string name = nameof(ParallelSparseSplitter)) { this.pipeline = pipeline; + this.name = name; this.splitterFunction = splitter; this.parallelTransform = transform; this.branchTerminationPolicy = branchTerminationPolicy ?? BranchTerminationPolicy.WhenKeyNotPresent(); @@ -81,6 +85,9 @@ public class ParallelSparseSplitter : IC /// public Emitter> ActiveBranches { get; } + /// + public override string ToString() => this.name; + private void Receive(TIn input, Envelope e) { var keyedValues = this.splitterFunction(input); diff --git a/Sources/Runtime/Microsoft.Psi/Components/ParallelVariableLength.cs b/Sources/Runtime/Microsoft.Psi/Components/ParallelVariableLength.cs index b141834a7..7482a1d65 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/ParallelVariableLength.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/ParallelVariableLength.cs @@ -29,10 +29,10 @@ public class ParallelVariableLength : Subpipeline, IConsumer, /// /// The pipeline to add the component to. /// Function mapping keyed input producers to output producers. - /// Name for this component (defaults to ParallelVariableLength). + /// An optional name for the component. /// Pipeline-level default delivery policy to be used by this component (defaults to if unspecified). - public ParallelVariableLength(Pipeline pipeline, Action> action, string name = null, DeliveryPolicy defaultDeliveryPolicy = null) - : base(pipeline, name ?? nameof(ParallelVariableLength), defaultDeliveryPolicy) + public ParallelVariableLength(Pipeline pipeline, Action> action, string name = nameof(ParallelVariableLength), DeliveryPolicy defaultDeliveryPolicy = null) + : base(pipeline, name, defaultDeliveryPolicy) { this.parallelAction = action; this.inConnector = this.CreateInputConnectorFrom(pipeline, nameof(this.inConnector)); @@ -47,10 +47,10 @@ public ParallelVariableLength(Pipeline pipeline, Action> act /// Function mapping keyed input producers to output producers. /// When true, a result is produced even if a message is dropped in processing one of the input elements. In this case the corresponding output element is set to a default value. /// Default value to use when messages are dropped in processing one of the input elements. - /// Name for this component (defaults to ParallelVariableLength). + /// An optional name for the component. /// Pipeline-level default delivery policy to be used by this component (defaults to if unspecified). - public ParallelVariableLength(Pipeline pipeline, Func, IProducer> transform, bool outputDefaultIfDropped = false, TOut defaultValue = default, string name = null, DeliveryPolicy defaultDeliveryPolicy = null) - : base(pipeline, name ?? nameof(ParallelVariableLength), defaultDeliveryPolicy) + public ParallelVariableLength(Pipeline pipeline, Func, IProducer> transform, bool outputDefaultIfDropped = false, TOut defaultValue = default, string name = nameof(ParallelVariableLength), DeliveryPolicy defaultDeliveryPolicy = null) + : base(pipeline, name, defaultDeliveryPolicy) { this.parallelTransform = transform; this.inConnector = this.CreateInputConnectorFrom(pipeline, nameof(this.inConnector)); diff --git a/Sources/Runtime/Microsoft.Psi/Components/Processor.cs b/Sources/Runtime/Microsoft.Psi/Components/Processor.cs index bf4e0a52a..ba8733541 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/Processor.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/Processor.cs @@ -26,8 +26,9 @@ public class Processor : ConsumerProducer /// The pipeline to add the component to. /// A delegate that processes the input data and potentially publishes a result on the provided . /// An optional action to execute when the input stream closes. - public Processor(Pipeline pipeline, Action> transform, Action> onClose = null) - : base(pipeline) + /// An optional name for this component. + public Processor(Pipeline pipeline, Action> transform, Action> onClose = null, string name = nameof(Processor)) + : base(pipeline, name) { this.transform = transform; if (onClose != null) diff --git a/Sources/Runtime/Microsoft.Psi/Components/RelativeIndexWindow.cs b/Sources/Runtime/Microsoft.Psi/Components/RelativeIndexWindow.cs index d850e01b3..b9f6150b9 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/RelativeIndexWindow.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/RelativeIndexWindow.cs @@ -30,8 +30,9 @@ public class RelativeIndexWindow : ConsumerProducerThe pipeline to add the component to. /// The relative index interval over which to gather messages. /// Select output message from collected window of input messages. - public RelativeIndexWindow(Pipeline pipeline, IntInterval relativeIndexInterval, Func>, TOutput> selector) - : base(pipeline) + /// An optional name for the component. + public RelativeIndexWindow(Pipeline pipeline, IntInterval relativeIndexInterval, Func>, TOutput> selector, string name = nameof(RelativeIndexWindow)) + : base(pipeline, name) { if (relativeIndexInterval.IsNegative) { diff --git a/Sources/Runtime/Microsoft.Psi/Components/RelativeTimeWindow.cs b/Sources/Runtime/Microsoft.Psi/Components/RelativeTimeWindow.cs index 9e2dfd80d..e25ee4edb 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/RelativeTimeWindow.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/RelativeTimeWindow.cs @@ -28,8 +28,9 @@ public class RelativeTimeWindow : ConsumerProducerThe pipeline to add the component to. /// The relative time interval over which to gather messages. /// Select output message from collected window of input messages. - public RelativeTimeWindow(Pipeline pipeline, RelativeTimeInterval relativeTimeInterval, Func>, TOutput> selector) - : base(pipeline) + /// An optional name for the component. + public RelativeTimeWindow(Pipeline pipeline, RelativeTimeInterval relativeTimeInterval, Func>, TOutput> selector, string name = nameof(RelativeTimeWindow)) + : base(pipeline, name) { this.relativeTimeInterval = relativeTimeInterval; this.selector = selector; diff --git a/Sources/Runtime/Microsoft.Psi/Components/SerializerComponent.cs b/Sources/Runtime/Microsoft.Psi/Components/SerializerComponent.cs index b7dd9a93f..b0767b8d0 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/SerializerComponent.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/SerializerComponent.cs @@ -21,8 +21,9 @@ internal sealed class SerializerComponent : ConsumerProducer, Mess /// /// The pipeline to add the component to. /// Known serializers. - internal SerializerComponent(Pipeline pipeline, KnownSerializers serializers) - : base(pipeline) + /// An optional name for the component. + internal SerializerComponent(Pipeline pipeline, KnownSerializers serializers, string name = nameof(SerializerComponent)) + : base(pipeline, name) { this.context = new SerializationContext(serializers); this.handler = serializers.GetHandler(); diff --git a/Sources/Runtime/Microsoft.Psi/Components/SimpleConsumer.cs b/Sources/Runtime/Microsoft.Psi/Components/SimpleConsumer.cs index 89dfc5e08..d8b71c85a 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/SimpleConsumer.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/SimpleConsumer.cs @@ -9,12 +9,16 @@ namespace Microsoft.Psi.Components /// The input message type. public abstract class SimpleConsumer : IConsumer { + private readonly string name; + /// /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - public SimpleConsumer(Pipeline pipeline) + /// An optional name for this component. + public SimpleConsumer(Pipeline pipeline, string name = nameof(SimpleConsumer)) { + this.name = name; this.In = pipeline.CreateReceiver(this, this.Receive, nameof(this.In)); } @@ -26,5 +30,8 @@ public SimpleConsumer(Pipeline pipeline) /// /// Message received. public abstract void Receive(Message message); + + /// + public override string ToString() => this.name; } } \ No newline at end of file diff --git a/Sources/Runtime/Microsoft.Psi/Components/Splitter.cs b/Sources/Runtime/Microsoft.Psi/Components/Splitter.cs index 66bf2291f..cdcda28a1 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/Splitter.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/Splitter.cs @@ -13,9 +13,10 @@ namespace Microsoft.Psi.Components /// The type of key to use when identifying the correct output. public class Splitter : IConsumer { - private readonly Dictionary> outputs = new Dictionary>(); + private readonly Dictionary> outputs = new (); private readonly Func outputSelector; private readonly Pipeline pipeline; + private readonly string name; private readonly Receiver input; /// @@ -23,9 +24,11 @@ public class Splitter : IConsumer /// /// The pipeline to add the component to. /// Selector function identifying the output. - public Splitter(Pipeline pipeline, Func outputSelector) + /// An optional name for the component. + public Splitter(Pipeline pipeline, Func outputSelector, string name = nameof(Splitter)) { this.pipeline = pipeline; + this.name = name; this.outputSelector = outputSelector; this.input = pipeline.CreateReceiver(this, this.Receive, nameof(this.In)); } @@ -48,6 +51,9 @@ public Emitter Add(TKey key) return this.outputs[key] = this.pipeline.CreateEmitter(this, key.ToString()); } + /// + public override string ToString() => this.name; + private void Receive(TIn message, Envelope e) { var key = this.outputSelector(message, e); diff --git a/Sources/Runtime/Microsoft.Psi/Components/Timer.cs b/Sources/Runtime/Microsoft.Psi/Components/Timer.cs index 088180b29..a10399194 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/Timer.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/Timer.cs @@ -13,6 +13,7 @@ namespace Microsoft.Psi.Components public abstract class Timer : ISourceComponent, IDisposable { private readonly Pipeline pipeline; + private readonly string name; /// /// The interval on which to publish messages. @@ -55,9 +56,11 @@ public abstract class Timer : ISourceComponent, IDisposable /// /// The pipeline to add the component to. /// The timer firing interval, in ms. - public Timer(Pipeline pipeline, uint timerInterval) + /// An optional name for the component. + public Timer(Pipeline pipeline, uint timerInterval, string name = nameof(Timer)) { this.pipeline = pipeline; + this.name = name; this.timerInterval = TimeSpan.FromMilliseconds(timerInterval); } @@ -101,6 +104,9 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) notifyCompleted(); } + /// + public override string ToString() => this.name; + /// /// Called by the timer. Override to publish actual messages. /// diff --git a/Sources/Runtime/Microsoft.Psi/Components/Timer{TOut}.cs b/Sources/Runtime/Microsoft.Psi/Components/Timer{TOut}.cs index 8bcd31747..fd2588d85 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/Timer{TOut}.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/Timer{TOut}.cs @@ -11,7 +11,7 @@ namespace Microsoft.Psi.Components /// The type of messages published by the generator. public class Timer : Timer, IProducer { - private Func generator; + private readonly Func generator; /// /// Initializes a new instance of the class. @@ -19,8 +19,9 @@ public class Timer : Timer, IProducer /// The pipeline to add the component to. /// Time interval with which to produce messages. /// Message generation function. - public Timer(Pipeline pipeline, uint timerInterval, Func generator) - : base(pipeline, timerInterval) + /// An optional name for the component. + public Timer(Pipeline pipeline, uint timerInterval, Func generator, string name = nameof(Timer)) + : base(pipeline, timerInterval, name) { this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); this.generator = generator; diff --git a/Sources/Runtime/Microsoft.Psi/Components/Zip.cs b/Sources/Runtime/Microsoft.Psi/Components/Zip.cs index 0a36bd243..1dcb2b0ef 100644 --- a/Sources/Runtime/Microsoft.Psi/Components/Zip.cs +++ b/Sources/Runtime/Microsoft.Psi/Components/Zip.cs @@ -16,6 +16,7 @@ namespace Microsoft.Psi.Components public class Zip : IProducer { private readonly Pipeline pipeline; + private readonly string name; private readonly IList> inputs = new List>(); private readonly IList<(T data, Envelope envelope, IRecyclingPool recycler)> buffer = new List<(T, Envelope, IRecyclingPool)>(); @@ -23,9 +24,11 @@ public class Zip : IProducer /// Initializes a new instance of the class. /// /// The pipeline to add the component to. - public Zip(Pipeline pipeline) + /// An optional name for this component. + public Zip(Pipeline pipeline, string name = nameof(Zip)) { this.pipeline = pipeline; + this.name = name; this.Out = pipeline.CreateEmitter(this, nameof(this.Out)); } @@ -58,6 +61,9 @@ public Receiver AddInput(string name) } } + /// + public override string ToString() => this.name; + private void Receive(T clonedData, Envelope envelope, IRecyclingPool recycler) { this.buffer.Add((data: clonedData, envelope, recycler)); diff --git a/Sources/Runtime/Microsoft.Psi/Connectors/MessageConnector.cs b/Sources/Runtime/Microsoft.Psi/Connectors/MessageConnector.cs index 0a8cb44d7..be970d1a2 100644 --- a/Sources/Runtime/Microsoft.Psi/Connectors/MessageConnector.cs +++ b/Sources/Runtime/Microsoft.Psi/Connectors/MessageConnector.cs @@ -51,9 +51,6 @@ internal MessageConnector(Pipeline pipeline, string name = null) public Emitter> Out { get; } /// - public override string ToString() - { - return this.name; - } + public override string ToString() => this.name; } } \ No newline at end of file diff --git a/Sources/Runtime/Microsoft.Psi/Connectors/MessageEnvelopeConnector.cs b/Sources/Runtime/Microsoft.Psi/Connectors/MessageEnvelopeConnector.cs index 5b6bacaf0..1897108e5 100644 --- a/Sources/Runtime/Microsoft.Psi/Connectors/MessageEnvelopeConnector.cs +++ b/Sources/Runtime/Microsoft.Psi/Connectors/MessageEnvelopeConnector.cs @@ -51,9 +51,6 @@ internal MessageEnvelopeConnector(Pipeline pipeline, string name = null) public Emitter> Out { get; } /// - public override string ToString() - { - return this.name; - } + public override string ToString() => this.name; } } \ No newline at end of file diff --git a/Sources/Runtime/Microsoft.Psi/Data/Exporter.cs b/Sources/Runtime/Microsoft.Psi/Data/Exporter.cs index 9ec75a5bc..e17ef56bf 100644 --- a/Sources/Runtime/Microsoft.Psi/Data/Exporter.cs +++ b/Sources/Runtime/Microsoft.Psi/Data/Exporter.cs @@ -134,7 +134,7 @@ public void Write(IProducer source, string name, bool largeM public void Write(IProducer source, TSupplementalMetadata supplementalMetadataValue, string name, bool largeMessages = false, DeliveryPolicy deliveryPolicy = null) { var meta = this.WriteToStorage(source.Out, name, largeMessages, deliveryPolicy); - meta.SetSupplementalMetadata(supplementalMetadataValue, this.serializers); + this.WriteSupplementalMetadataToCatalog(meta, supplementalMetadataValue); } /// @@ -160,7 +160,7 @@ public void Write(IProducer source, string name, bool largeM if (!set.Contains(key)) { set.UnionWith(dict.Keys); - meta.SetSupplementalMetadata(new DistinctKeysSupplementalMetadata(set), this.serializers); + this.WriteSupplementalMetadataToCatalog(meta, new DistinctKeysSupplementalMetadata(set)); return set; } } @@ -194,8 +194,12 @@ public void WriteEnvelopes(IProducer source, string name, DeliveryPolicy this.writer.CloseStream(source.Out.Id, closeTime); - // tell the writer to write the serialized stream - var meta = this.writer.OpenStream(source.Out.Id, name, false, handler.Name); + // tell the writer to write the serialized stream. If the pipeline is running + // when the write command arrives, the stream is opened at the current pipeline time + // o/w if the pipeline is not yet running, the opened time is set to minvalue, and + // will be updated on pipeline start-up. + var openedTime = this.pipeline.IsRunning ? this.pipeline.GetCurrentTime() : DateTime.MinValue; + var meta = this.writer.OpenStream(source.Out.Id, name, false, handler.Name, openedTime); // register this stream with the store catalog this.pipeline.ConfigurationStore.Set(Exporter.StreamMetadataNamespace, name, meta); @@ -265,8 +269,12 @@ private PsiStreamMetadata WriteToStorage(Emitter source, str source.Name = connector.Out.Name = name; source.Closed += closeTime => this.writer.CloseStream(source.Id, closeTime); - // tell the writer to write the serialized stream - var meta = this.writer.OpenStream(source.Id, name, largeMessages, handler.Name); + // tell the writer to write the serialized stream. If the pipeline is running + // when the write command arrives, the stream is opened at the current pipeline time + // o/w if the pipeline is not yet running, the opened time is set to minvalue, and + // will be updated on pipeline start-up. + var openedTime = this.pipeline.IsRunning ? this.pipeline.GetCurrentTime() : DateTime.MinValue; + var meta = this.writer.OpenStream(source.Id, name, largeMessages, handler.Name, openedTime); // register this stream with the store catalog this.pipeline.ConfigurationStore.Set(Exporter.StreamMetadataNamespace, name, meta); @@ -284,6 +292,21 @@ private PsiStreamMetadata WriteToStorage(Emitter source, str return meta; } + /// + /// Writes the supplemental metadata for a specified stream to the catalog. + /// + /// The type of the supplemental metadata. + /// The psi stream metadata. + /// The supplemental metadata. + private void WriteSupplementalMetadataToCatalog(PsiStreamMetadata meta, TSupplementalMetadata supplementalMetadata) + { + var handler = this.serializers.GetHandler(); + var writer = new BufferWriter(default(byte[])); + handler.Serialize(writer, supplementalMetadata, new SerializationContext(this.serializers)); + meta.SetSupplementalMetadata(typeof(TSupplementalMetadata).AssemblyQualifiedName, writer.Buffer); + this.writer.WriteToCatalog(meta); + } + /// /// Represents distinct keys on a dictionary stream. /// diff --git a/Sources/Runtime/Microsoft.Psi/Data/PsiStore.cs b/Sources/Runtime/Microsoft.Psi/Data/PsiStore.cs index d76599dac..259e0fa20 100644 --- a/Sources/Runtime/Microsoft.Psi/Data/PsiStore.cs +++ b/Sources/Runtime/Microsoft.Psi/Data/PsiStore.cs @@ -496,6 +496,17 @@ public static string GetPathToLatestVersion(string storeName, string rootPath) return path; } + /// + /// Enumerates all stores under a given path. + /// + /// The root path to search. + /// A value indicating whether to search recursively. + /// An enumeration of names and paths to stores found. + public static IEnumerable<(string Name, string Path)> EnumerateStores(string rootPath, bool recursively = true) + { + return PsiStoreCommon.EnumerateStores(rootPath, recursively).Select(store => (store.Name, store.Path)); + } + /// /// Delete a \psi store. /// diff --git a/Sources/Runtime/Microsoft.Psi/Diagnostics/DiagnosticsCollector.cs b/Sources/Runtime/Microsoft.Psi/Diagnostics/DiagnosticsCollector.cs index 288f6b22b..783c12c7a 100644 --- a/Sources/Runtime/Microsoft.Psi/Diagnostics/DiagnosticsCollector.cs +++ b/Sources/Runtime/Microsoft.Psi/Diagnostics/DiagnosticsCollector.cs @@ -123,6 +123,7 @@ public void PipelineElementCreate(Pipeline pipeline, PipelineElement element, ob public void PipelineElementStart(Pipeline pipeline, PipelineElement element) { this.graphs[pipeline.Id].PipelineElements[element.Id].IsRunning = true; + this.graphs[pipeline.Id].PipelineElements[element.Id].DiagnosticState = element.StateObject.ToString(); } /// @@ -134,6 +135,7 @@ public void PipelineElementStart(Pipeline pipeline, PipelineElement element) public void PipelineElementStop(Pipeline pipeline, PipelineElement element) { this.graphs[pipeline.Id].PipelineElements[element.Id].IsRunning = false; + this.graphs[pipeline.Id].PipelineElements[element.Id].DiagnosticState = element.StateObject.ToString(); } /// diff --git a/Sources/Runtime/Microsoft.Psi/Diagnostics/DiagnosticsSampler.cs b/Sources/Runtime/Microsoft.Psi/Diagnostics/DiagnosticsSampler.cs index 40589f12b..2e168953b 100644 --- a/Sources/Runtime/Microsoft.Psi/Diagnostics/DiagnosticsSampler.cs +++ b/Sources/Runtime/Microsoft.Psi/Diagnostics/DiagnosticsSampler.cs @@ -12,6 +12,7 @@ namespace Microsoft.Psi.Diagnostics internal class DiagnosticsSampler : ISourceComponent, IDisposable { private readonly Pipeline pipeline; + private readonly string name; private readonly DiagnosticsCollector collector; private Time.TimerDelegate timerDelegate; private bool running; @@ -22,12 +23,14 @@ internal class DiagnosticsSampler : ISourceComponent, IDisposable /// /// The pipeline to add the component to. /// Diagnostics collector. - /// Diagnostics configuration. - public DiagnosticsSampler(Pipeline pipeline, DiagnosticsCollector collector, DiagnosticsConfiguration config) + /// Diagnostics configuration. + /// An optional name for the component. + public DiagnosticsSampler(Pipeline pipeline, DiagnosticsCollector collector, DiagnosticsConfiguration configuration, string name = nameof(DiagnosticsSampler)) { this.pipeline = pipeline; + this.name = name; this.collector = collector; - this.Config = config; + this.Config = configuration; this.Diagnostics = pipeline.CreateEmitter(this, nameof(this.Diagnostics)); } @@ -79,6 +82,9 @@ public void Stop(DateTime finalOriginatingTime, Action notifyCompleted) notifyCompleted(); } + /// + public override string ToString() => this.name; + private void Stop() { if (this.running) diff --git a/Sources/Runtime/Microsoft.Psi/Executive/Pipeline.cs b/Sources/Runtime/Microsoft.Psi/Executive/Pipeline.cs index 2432512e6..bc64dc8a7 100644 --- a/Sources/Runtime/Microsoft.Psi/Executive/Pipeline.cs +++ b/Sources/Runtime/Microsoft.Psi/Executive/Pipeline.cs @@ -74,6 +74,7 @@ public class Pipeline : IDisposable private Time.TimerDelegate progressDelegate; private Platform.ITimer progressTimer; private bool pipelineRunEventHandled = false; + private int isPipelineDisposed = 0; /// /// Initializes a new instance of the class. @@ -684,9 +685,9 @@ internal Emitter CreateEmitterWithFixedStreamId(object owner, string name, /// Abandons pending work items. internal void Dispose(bool abandonPendingWorkItems) { - if (this.components == null) + if (Interlocked.CompareExchange(ref this.isPipelineDisposed, 1, 0) != 0) { - // we never started or we've been already disposed + // we've already been disposed return; } diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Aggregators.cs b/Sources/Runtime/Microsoft.Psi/Operators/Aggregators.cs index 31d914d47..787367ce4 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Aggregators.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Aggregators.cs @@ -17,82 +17,117 @@ public static partial class Operators /// Type of source stream. /// Type of output stream. /// Source stream. - /// Initial seed state. - /// Aggregation function. + /// The initial state. + /// The aggregator function. /// An optional delivery policy. + /// An optional name for this stream operator. /// Output stream. - public static IProducer Aggregate(this IProducer source, TOut seed, Func func, DeliveryPolicy deliveryPolicy = null) - { - return Aggregate( - source.Out, - seed, - (a, d, e, s) => + public static IProducer Aggregate( + this IProducer source, + TOut initialState, + Func aggregator, + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Aggregate)) + => source.Aggregate( + initialState, + (state, input, envelope, emitter) => { - var newState = func(a, d); - s.Post(newState, e.OriginatingTime); + var newState = aggregator(state, input); + emitter.Post(newState, envelope.OriginatingTime); return newState; }, - deliveryPolicy); - } + deliveryPolicy, + name); /// /// Aggregate stream values. /// /// Type of source stream messages. - /// Type of initial seed value. + /// Type of accumulator value. /// Type of output stream messages. /// Source stream. - /// Initial seed state. - /// Aggregation function. - /// Selector function. + /// The initial state of the accumulator. + /// The aggregation function. + /// A selector function. /// An optional delivery policy. + /// An optional name for this stream operator. /// Output stream. - public static IProducer Aggregate(this IProducer source, TAcc seed, Func func, Func selector, DeliveryPolicy deliveryPolicy = null) - { - return Aggregate(source, seed, func, deliveryPolicy).Select(selector, DeliveryPolicy.SynchronousOrThrottle); - } + public static IProducer Aggregate( + this IProducer source, + TAccumulate initialState, + Func aggregator, + Func selector, + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Aggregate)) + => source.Aggregate( + initialState, + (state, input, envelope, emitter) => + { + var newState = aggregator(state, input); + emitter.Post(selector(newState), envelope.OriginatingTime); + return newState; + }, + deliveryPolicy, + name); /// /// Aggregate stream values. /// /// Type of source/output stream messages. /// Source stream. - /// Aggregation function. + /// The aggregator function. /// An optional delivery policy. + /// An optional name for this stream operator. /// Output stream. - public static IProducer Aggregate(this IProducer source, Func func, DeliveryPolicy deliveryPolicy = null) + /// The initial state of the aggregation is the first value passed in. + public static IProducer Aggregate( + this IProducer source, + Func aggregator, + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Aggregate)) { - // `Aggregate` where `TIn` is same type as `TOut`, seed becomes first value - return Aggregate( - source, - Tuple.Create(true, default(T)), - (s, x) => + var first = true; + return source.Aggregate( + default, + (state, input, envelope, emitter) => { - var first = s.Item1; - var val = s.Item2; - return Tuple.Create(false, first ? x : func(val, x)); + if (first) + { + state = input; + first = false; + } + else + { + state = aggregator(state, input); + } + + emitter.Post(state, envelope.OriginatingTime); + return state; }, - deliveryPolicy).Select(x => x.Item2, DeliveryPolicy.SynchronousOrThrottle); + deliveryPolicy, + name); } /// /// Aggregate stream values. /// - /// Type of initial seed value. + /// Type of accumulator value. /// Type of input stream messages. /// Type of output stream messages. /// Source stream. - /// Initial seed value. - /// Aggregation function. + /// The initial value for the accumulator. + /// The aggregation function. /// An optional delivery policy. + /// An optional name for this stream operator. /// Output stream. public static IProducer Aggregate( this IProducer source, - TAccumulate seed, - Func, TAccumulate> func, - DeliveryPolicy deliveryPolicy = null) + TAccumulate initialState, + Func, TAccumulate> aggregator, + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Aggregate)) { - var aggregate = new Aggregator(source.Out.Pipeline, seed, func); + var aggregate = new Aggregator(source.Out.Pipeline, initialState, aggregator, name); return PipeTo(source, aggregate, deliveryPolicy); } } diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Connectors.cs b/Sources/Runtime/Microsoft.Psi/Operators/Connectors.cs index 6f1c3a1cb..18cd44273 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Connectors.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Connectors.cs @@ -51,7 +51,7 @@ public static IProducer BridgeTo(this IProducer input, Pipeline targetP } else { - var connector = new Connector(input.Out.Pipeline, targetPipeline, name ?? "BridgeConnector"); + var connector = new Connector(input.Out.Pipeline, targetPipeline, name ?? nameof(BridgeTo)); return input.PipeTo(connector, deliveryPolicy); } } diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Enumerable.cs b/Sources/Runtime/Microsoft.Psi/Operators/Enumerable.cs index da01ebfea..ffd02a554 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Enumerable.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Enumerable.cs @@ -25,11 +25,10 @@ public static partial class Operators /// The source stream. /// Predicate condition while which values will be enumerated (otherwise infinite). /// An optional delivery policy. + /// An optional name for this stream operator. /// Enumerable with elements from the source stream. - public static IEnumerable ToEnumerable(this IProducer source, Func condition = null, DeliveryPolicy deliveryPolicy = null) - { - return new StreamEnumerable(source, condition, deliveryPolicy); - } + public static IEnumerable ToEnumerable(this IProducer source, Func condition = null, DeliveryPolicy deliveryPolicy = null, string name = nameof(ToEnumerable)) + => new StreamEnumerable(source, condition, deliveryPolicy, name); /// /// Enumerable stream class. @@ -45,7 +44,8 @@ public class StreamEnumerable : IEnumerable, IEnumerable, IDisposable /// The source stream to enumerate. /// Predicate (filter) function. /// An optional delivery policy. - public StreamEnumerable(IProducer source, Func predicate = null, DeliveryPolicy deliveryPolicy = null) + /// An optional name for this operator. + public StreamEnumerable(IProducer source, Func predicate = null, DeliveryPolicy deliveryPolicy = null, string name = nameof(StreamEnumerable)) { this.enumerator = new StreamEnumerator(predicate ?? (_ => true)); @@ -55,7 +55,8 @@ public StreamEnumerable(IProducer source, Func predicate = null, Del { this.enumerator.Queue.Enqueue(d.DeepClone()); this.enumerator.Enqueued.Set(); - }); + }, + name: name); source.PipeTo(processor, deliveryPolicy); processor.In.Unsubscribed += _ => this.enumerator.Closed.Set(); @@ -82,8 +83,8 @@ IEnumerator IEnumerable.GetEnumerator() private class StreamEnumerator : IEnumerator, IEnumerator { private readonly Func predicate; + private readonly WaitHandle[] queueUpdated; private T current; - private WaitHandle[] queueUpdated; public StreamEnumerator(Func predicate) { diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Fuses.cs b/Sources/Runtime/Microsoft.Psi/Operators/Fuses.cs index 303898caf..eb8c9ddd3 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Fuses.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Fuses.cs @@ -28,6 +28,7 @@ public static partial class Operators /// Function mapping the primary and secondary messages to an output message type. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of fused values. public static IProducer Fuse( this IProducer primary, @@ -35,7 +36,8 @@ public static partial class Operators Interpolator interpolator, Func outputCreator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Fuse( primary, @@ -43,7 +45,8 @@ public static partial class Operators interpolator, (m, secondaryArray) => outputCreator(m, secondaryArray[0]), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); } /// @@ -57,16 +60,23 @@ public static partial class Operators /// Interpolator to use when fusing the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of fused values. public static IProducer<(TPrimary, TInterpolation)> Fuse( this IProducer primary, IProducer secondary, Interpolator interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) - { - return Fuse(primary, secondary, interpolator, ValueTuple.Create, primaryDeliveryPolicy, secondaryDeliveryPolicy); - } + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) + => Fuse( + primary, + secondary, + interpolator, + ValueTuple.Create, + primaryDeliveryPolicy, + secondaryDeliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); #endregion Scalar fuse operators @@ -84,22 +94,23 @@ public static partial class Operators /// Interpolator to use when fusing the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of fused tuple values flattened to arity 3. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TInterpolation)> Fuse( this IProducer<(TPrimaryItem1, TPrimaryItem2)> primary, IProducer secondary, Interpolator interpolator, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) - { - return Fuse( + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) + => Fuse( primary, secondary, interpolator, (p, s) => (p.Item1, p.Item2, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); - } + secondaryDeliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); /// /// Fuse with values from a secondary stream based on a specified interpolator. @@ -114,22 +125,23 @@ public static partial class Operators /// Interpolator to use when fusing the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of fused tuple values flattened to arity 4. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TInterpolation)> Fuse( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3)> primary, IProducer secondary, Interpolator interpolator, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) - { - return Fuse( + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) + => Fuse( primary, secondary, interpolator, (p, s) => (p.Item1, p.Item2, p.Item3, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); - } + secondaryDeliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); /// /// Fuse with values from a secondary stream based on a specified interpolator. @@ -145,22 +157,23 @@ public static partial class Operators /// Interpolator to use when fusing the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of fused tuple values flattened to arity 5. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TInterpolation)> Fuse( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4)> primary, IProducer secondary, Interpolator interpolator, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) - { - return Fuse( + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) + => Fuse( primary, secondary, interpolator, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); - } + secondaryDeliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); /// /// Fuse with values from a secondary stream based on a specified interpolator. @@ -177,22 +190,23 @@ public static partial class Operators /// Interpolator to use when fusing the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of fused tuple values flattened to arity 6. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TInterpolation)> Fuse( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5)> primary, IProducer secondary, Interpolator interpolator, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) - { - return Fuse( + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) + => Fuse( primary, secondary, interpolator, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); - } + secondaryDeliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); /// /// Fuse with values from a secondary stream based on a specified interpolator. @@ -210,22 +224,23 @@ public static partial class Operators /// Interpolator to use when fusing the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of fused tuple values flattened to arity 7. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6, TInterpolation)> Fuse( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6)> primary, IProducer secondary, Interpolator interpolator, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) - { - return Fuse( + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) + => Fuse( primary, secondary, interpolator, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, p.Item6, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); - } + secondaryDeliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); #endregion Tuple-flattening scalar fuse operators @@ -242,22 +257,23 @@ public static partial class Operators /// Interpolator to use when fusing the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of fused tuple values flattened to arity 3. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2)> Fuse( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2)> secondary, Interpolator<(TSecondaryItem1, TSecondaryItem2), (TSecondaryItem1, TSecondaryItem2)> interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2)> secondaryDeliveryPolicy = null) - { - return Fuse( + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2)> secondaryDeliveryPolicy = null, + string name = null) + => Fuse( primary, secondary, interpolator, (p, s) => (p, s.Item1, s.Item2), primaryDeliveryPolicy, - secondaryDeliveryPolicy); - } + secondaryDeliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); /// /// Fuse with values from a secondary stream based on a specified interpolator. @@ -271,22 +287,23 @@ public static partial class Operators /// Interpolator to use when fusing the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of fused tuple values flattened to arity 4. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> Fuse( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondary, Interpolator<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3), (TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondaryDeliveryPolicy = null) - { - return Fuse( + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondaryDeliveryPolicy = null, + string name = null) + => Fuse( primary, secondary, interpolator, (p, s) => (p, s.Item1, s.Item2, s.Item3), primaryDeliveryPolicy, - secondaryDeliveryPolicy); - } + secondaryDeliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); /// /// Fuse with values from a secondary stream based on a specified interpolator. @@ -301,22 +318,23 @@ public static partial class Operators /// Interpolator to use when fusing the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of fused tuple values flattened to arity 5. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> Fuse( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondary, Interpolator<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4), (TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondaryDeliveryPolicy = null) - { - return Fuse( + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondaryDeliveryPolicy = null, + string name = null) + => Fuse( primary, secondary, interpolator, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4), primaryDeliveryPolicy, - secondaryDeliveryPolicy); - } + secondaryDeliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); /// /// Fuse with values from a secondary stream based on a specified interpolator. @@ -332,22 +350,23 @@ public static partial class Operators /// Interpolator to use when fusing the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of fused tuple values flattened to arity 6. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> Fuse( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondary, Interpolator<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5), (TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondaryDeliveryPolicy = null) - { - return Fuse( + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondaryDeliveryPolicy = null, + string name = null) + => Fuse( primary, secondary, interpolator, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5), primaryDeliveryPolicy, - secondaryDeliveryPolicy); - } + secondaryDeliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); /// /// Fuse with values from a secondary stream based on a specified interpolator. @@ -364,22 +383,23 @@ public static partial class Operators /// Interpolator to use when fusing the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of fused tuple values flattened to arity 7. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> Fuse( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondary, Interpolator<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6), (TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondaryDeliveryPolicy = null) - { - return Fuse( + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondaryDeliveryPolicy = null, + string name = null) + => Fuse( primary, secondary, interpolator, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5, s.Item6), primaryDeliveryPolicy, - secondaryDeliveryPolicy); - } + secondaryDeliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); #endregion Reverse tuple-flattening scalar fuse operators @@ -397,6 +417,7 @@ public static partial class Operators /// Mapping function from primary and secondary messages to output. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Output stream. public static IProducer Fuse( this IProducer primary, @@ -404,14 +425,16 @@ public static partial class Operators Interpolator interpolator, Func outputCreator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondariesDeliveryPolicy = null) + DeliveryPolicy secondariesDeliveryPolicy = null, + string name = null) { var fuse = new Fuse( primary.Out.Pipeline, interpolator, outputCreator, secondaries.Count(), - null); + null, + name ?? $"{nameof(Fuse)}({interpolator})"); primary.PipeTo(fuse.InPrimary, primaryDeliveryPolicy); @@ -434,16 +457,23 @@ public static partial class Operators /// Interpolator to use when fusing the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Output stream. public static IProducer<(TPrimary, TSecondary[])> Fuse( this IProducer primary, IEnumerable> secondaries, Interpolator interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondariesDeliveryPolicy = null) - { - return primary.Fuse(secondaries, interpolator, ValueTuple.Create, primaryDeliveryPolicy, secondariesDeliveryPolicy); - } + DeliveryPolicy secondariesDeliveryPolicy = null, + string name = null) + => Fuse( + primary, + secondaries, + interpolator, + ValueTuple.Create, + primaryDeliveryPolicy, + secondariesDeliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); /// /// Fuses a primary stream with an enumeration of secondary streams based on a specified interpolator. @@ -458,6 +488,7 @@ public static partial class Operators /// Mapping function from primary and secondary messages to output. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Output stream. public static IProducer Fuse( this IProducer primary, @@ -465,14 +496,16 @@ public static partial class Operators Interpolator interpolator, Func outputCreator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondariesDeliveryPolicy = null) + DeliveryPolicy secondariesDeliveryPolicy = null, + string name = null) { var fuse = new Fuse( primary.Out.Pipeline, interpolator, outputCreator, secondaries.Count(), - null); + null, + name ?? $"{nameof(Fuse)}({interpolator})"); primary.PipeTo(fuse.InPrimary, primaryDeliveryPolicy); @@ -496,16 +529,23 @@ public static partial class Operators /// Interpolator to use when fusing the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Output stream. public static IProducer<(TPrimary, TInterpolation[])> Fuse( this IProducer primary, IEnumerable> secondaries, Interpolator interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondariesDeliveryPolicy = null) - { - return primary.Fuse(secondaries, interpolator, (p, i) => (p, i), primaryDeliveryPolicy, secondariesDeliveryPolicy); - } + DeliveryPolicy secondariesDeliveryPolicy = null, + string name = null) + => Fuse( + primary, + secondaries, + interpolator, + (p, i) => (p, i), + primaryDeliveryPolicy, + secondariesDeliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); /// /// Fuses an enumeration of streams into a vector stream, based on a specified interpolator and output creator function. @@ -516,12 +556,14 @@ public static partial class Operators /// Interpolator to use when fusing the streams. /// Mapping function from input to output messages. /// An optional delivery policy to use for the streams. + /// An optional name for the stream operator. /// Output stream. public static IProducer Fuse( this IEnumerable> inputs, Interpolator interpolator, Func outputCreator, - DeliveryPolicy deliveryPolicy = null) + DeliveryPolicy deliveryPolicy = null, + string name = null) { var count = inputs.Count(); if (count > 1) @@ -542,11 +584,12 @@ public static partial class Operators return buffer; }, deliveryPolicy, - deliveryPolicy); + deliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); } else if (count == 1) { - return inputs.First().Select(x => new[] { outputCreator(x) }, deliveryPolicy); + return inputs.First().Select(x => new[] { outputCreator(x) }, deliveryPolicy, name ?? $"{nameof(Fuse)}({interpolator})"); } else { @@ -561,14 +604,19 @@ public static partial class Operators /// Collection of input streams. /// Interpolator to use when fusing the streams. /// An optional delivery policy to use for the streams. + /// An optional name for the stream operator. /// Output stream. public static IProducer Fuse( this IEnumerable> inputs, Interpolator interpolator, - DeliveryPolicy deliveryPolicy = null) - { - return inputs.Fuse(interpolator, _ => _, deliveryPolicy); - } + DeliveryPolicy deliveryPolicy = null, + string name = null) + => Fuse( + inputs, + interpolator, + _ => _, + deliveryPolicy, + name ?? $"{nameof(Fuse)}({interpolator})"); #endregion Vector fuse operators } diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Generators.cs b/Sources/Runtime/Microsoft.Psi/Operators/Generators.cs index 49d3cbcd7..1d172e2a7 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Generators.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Generators.cs @@ -26,12 +26,11 @@ public static class Generators /// is non-null, the messages will have originating times that align with (i.e., are an integral number of intervals away from) the /// specified alignment time. /// Indicates whether the stream should be kept open after all messages in the sequence have been posted. + /// An optional name for the stream generator. /// A stream of values of type T. /// When the pipeline is in replay mode, the timing of the messages complies with the speed of the pipeline. - public static IProducer Sequence(Pipeline pipeline, T initialValue, Func generateNext, int count, TimeSpan interval, DateTime? alignmentDateTime = null, bool keepOpen = false) - { - return Sequence(pipeline, Enumerate(initialValue, generateNext, count), interval, alignmentDateTime, keepOpen); - } + public static IProducer Sequence(Pipeline pipeline, T initialValue, Func generateNext, int count, TimeSpan interval, DateTime? alignmentDateTime = null, bool keepOpen = false, string name = nameof(Sequence)) + => Sequence(pipeline, Enumerate(initialValue, generateNext, count), interval, alignmentDateTime, keepOpen, name); /// /// Generates an infinite stream of values published at a regular interval from a user-provided function. @@ -44,12 +43,11 @@ public static IProducer Sequence(Pipeline pipeline, T initialValue, FuncIf non-null, this parameter specifies a time to align the generator messages with. If the parameter /// is non-null, the messages will have originating times that align with (i.e., are an integral number of intervals away from) the /// specified alignment time. + /// An optional name for the stream generator. /// A stream of values of type T. /// When the pipeline is in replay mode, the timing of the messages complies with the speed of the pipeline. - public static IProducer Sequence(Pipeline pipeline, T initialValue, Func generateNext, TimeSpan interval, DateTime? alignmentDateTime = null) - { - return Sequence(pipeline, Enumerate(initialValue, generateNext), interval, alignmentDateTime, keepOpen: true); - } + public static IProducer Sequence(Pipeline pipeline, T initialValue, Func generateNext, TimeSpan interval, DateTime? alignmentDateTime = null, string name = nameof(Sequence)) + => Sequence(pipeline, Enumerate(initialValue, generateNext), interval, alignmentDateTime, keepOpen: true, name); /// /// Generates a stream of values published at a regular interval from a specified enumerable. @@ -62,12 +60,11 @@ public static IProducer Sequence(Pipeline pipeline, T initialValue, Func /// Indicates whether the stream should be kept open after all messages in the sequence have been posted. + /// An optional name for the stream generator. /// A stream of values of type T. /// When the pipeline is in replay mode, the timing of the messages complies with the speed of the pipeline. - public static IProducer Sequence(Pipeline pipeline, IEnumerable enumerable, TimeSpan interval, DateTime? alignmentDateTime = null, bool keepOpen = false) - { - return new Generator(pipeline, enumerable.GetEnumerator(), interval, alignmentDateTime, isInfiniteSource: keepOpen); - } + public static IProducer Sequence(Pipeline pipeline, IEnumerable enumerable, TimeSpan interval, DateTime? alignmentDateTime = null, bool keepOpen = false, string name = nameof(Sequence)) + => new Generator(pipeline, enumerable.GetEnumerator(), interval, alignmentDateTime, isInfiniteSource: keepOpen); /// /// Generates a stream of values from a specified enumerable that provides the values and corresponding originating times. @@ -80,12 +77,11 @@ public static IProducer Sequence(Pipeline pipeline, IEnumerable enumera /// time to take this into account. Otherwise, pipeline playback will be determined by the prevailing replay descriptor (taking into /// account any other components in the pipeline which may have proposed replay times). /// Indicates whether the stream should be kept open after all the messages in the enumerable have been posted. + /// An optional name for the stream generator. /// A stream of values of type T. /// When the pipeline is in replay mode, the timing of the messages complies with the speed of the pipeline. - public static IProducer Sequence(Pipeline pipeline, IEnumerable<(T, DateTime)> enumerable, DateTime? startTime = null, bool keepOpen = false) - { - return new Generator(pipeline, enumerable.GetEnumerator(), startTime, isInfiniteSource: keepOpen); - } + public static IProducer Sequence(Pipeline pipeline, IEnumerable<(T, DateTime)> enumerable, DateTime? startTime = null, bool keepOpen = false, string name = nameof(Sequence)) + => new Generator(pipeline, enumerable.GetEnumerator(), startTime, isInfiniteSource: keepOpen, name); /// /// Generates stream containing a single message, and keeps the stream open afterwards. @@ -93,12 +89,11 @@ public static IProducer Sequence(Pipeline pipeline, IEnumerable<(T, DateTi /// The type of value to publish. /// The pipeline to add the component to. /// The value to publish. + /// An optional name for the stream generator. /// A stream of values of type T. /// The generated stream stays open until the pipeline is shut down. - public static IProducer Once(Pipeline pipeline, T value) - { - return Sequence(pipeline, new[] { value }, default, null, keepOpen: true); - } + public static IProducer Once(Pipeline pipeline, T value, string name = nameof(Once)) + => Sequence(pipeline, new[] { value }, default, null, keepOpen: true, name); /// /// Generates stream containing a single message, and closes the stream afterwards. @@ -106,12 +101,11 @@ public static IProducer Once(Pipeline pipeline, T value) /// The type of value to publish. /// The pipeline to add the component to. /// The value to publish. + /// An optional name for the stream generator. /// A stream containing one value of type T. /// The generated stream closes after the message is published. - public static IProducer Return(Pipeline pipeline, T value) - { - return Sequence(pipeline, new[] { value }, default, null, keepOpen: false); - } + public static IProducer Return(Pipeline pipeline, T value, string name = nameof(Return)) + => Sequence(pipeline, new[] { value }, default, null, keepOpen: false, name); /// /// Generates a finite stream of constant values published at a regular interval. @@ -125,12 +119,11 @@ public static IProducer Return(Pipeline pipeline, T value) /// is non-null, the messages will have originating times that align with (i.e., are an integral number of intervals away from) the /// specified alignment time. /// Indicates whether the stream should be kept open after the specified number of messages have been posted. + /// An optional name for the stream generator. /// A stream of values of type T. /// When the pipeline is in replay mode, the timing of the messages complies with the speed of the pipeline. The generated stream closes once the specified number of messages has been published. - public static IProducer Repeat(Pipeline pipeline, T value, int count, TimeSpan interval, DateTime? alignmentDateTime = null, bool keepOpen = false) - { - return Sequence(pipeline, Enumerable.Repeat(value, count), interval, alignmentDateTime, keepOpen); - } + public static IProducer Repeat(Pipeline pipeline, T value, int count, TimeSpan interval, DateTime? alignmentDateTime = null, bool keepOpen = false, string name = nameof(Repeat)) + => Sequence(pipeline, Enumerable.Repeat(value, count), interval, alignmentDateTime, keepOpen, name); /// /// Generates an infinite stream of constant values published at a regular interval. @@ -142,12 +135,11 @@ public static IProducer Repeat(Pipeline pipeline, T value, int count, Time /// If non-null, this parameter specifies a time to align the generator messages with. If the parameter /// is non-null, the messages will have originating times that align with (i.e., are an integral number of intervals away from) the /// specified alignment time. + /// An optional name for the stream generator. /// A stream of values of type T. /// When the pipeline is in replay mode, the timing of the messages complies with the speed of the pipeline. - public static IProducer Repeat(Pipeline pipeline, T value, TimeSpan interval, DateTime? alignmentDateTime = null) - { - return Sequence(pipeline, Enumerate(value, x => x), interval, alignmentDateTime, keepOpen: true); - } + public static IProducer Repeat(Pipeline pipeline, T value, TimeSpan interval, DateTime? alignmentDateTime = null, string name = nameof(Repeat)) + => Sequence(pipeline, Enumerate(value, x => x), interval, alignmentDateTime, keepOpen: true, name); /// /// Generates a stream of a finite range of integer values published at a regular interval. @@ -160,11 +152,10 @@ public static IProducer Repeat(Pipeline pipeline, T value, TimeSpan interv /// is non-null, the messages will have originating times that align with the specified time. /// A stream of consecutive integers. /// Indicates whether the stream should be kept open after the specified number of messages have been posted. + /// An optional name for the stream generator. /// When the pipeline is in replay mode, the timing of the messages complies with the speed of the pipeline. - public static IProducer Range(Pipeline pipeline, int start, int count, TimeSpan interval, DateTime? alignDateTime = null, bool keepOpen = false) - { - return Sequence(pipeline, Enumerable.Range(start, count), interval, alignDateTime, keepOpen); - } + public static IProducer Range(Pipeline pipeline, int start, int count, TimeSpan interval, DateTime? alignDateTime = null, bool keepOpen = false, string name = nameof(Range)) + => Sequence(pipeline, Enumerable.Range(start, count), interval, alignDateTime, keepOpen, name); internal static IEnumerable Enumerate(TResult initialValue, Func generateNext, int count) { diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Interpolators.cs b/Sources/Runtime/Microsoft.Psi/Operators/Interpolators.cs index 6140c1cbf..ec278d143 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Interpolators.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Interpolators.cs @@ -22,17 +22,24 @@ public static partial class Operators /// If non-null, this parameter specifies a time to align the sampling messages with. If the parameter /// is non-null, the messages will have originating times that align with (i.e., are an integral number of intervals away from) the /// specified alignment time. - /// An optional delivery policy. + /// An optional delivery policy for the source stream. + /// An optional name for the stream operator. /// Output stream. public static IProducer Interpolate( this IProducer source, TimeSpan samplingInterval, Interpolator interpolator, DateTime? alignmentDateTime = null, - DeliveryPolicy deliveryPolicy = null) + DeliveryPolicy deliveryPolicy = null, + string name = null) { var clock = Generators.Repeat(source.Out.Pipeline, 0, samplingInterval, alignmentDateTime); - return source.Interpolate(clock, interpolator, deliveryPolicy); + return source.Interpolate( + clock, + interpolator, + deliveryPolicy, + DeliveryPolicy.Unlimited, + name ?? $"{nameof(Interpolate)}({interpolator})"); } /// @@ -47,15 +54,21 @@ public static partial class Operators /// Interpolator to use for generating results. /// An optional delivery policy for the source stream. /// An optional delivery policy for the clock stream. + /// An optional name for the stream operator. /// Output stream. public static IProducer Interpolate( this IProducer source, IProducer clock, Interpolator interpolator, DeliveryPolicy sourceDeliveryPolicy = null, - DeliveryPolicy clockDeliveryPolicy = null) + DeliveryPolicy clockDeliveryPolicy = null, + string name = null) { - var fuse = new Fuse(source.Out.Pipeline, interpolator, (clk, data) => data[0]); + var fuse = new Fuse( + source.Out.Pipeline, + interpolator, + (clk, data) => data[0], + name: name ?? $"{nameof(Interpolate)}({interpolator})"); clock.PipeTo(fuse.InPrimary, clockDeliveryPolicy); source.PipeTo(fuse.InSecondaries[0], sourceDeliveryPolicy); return fuse; @@ -72,20 +85,23 @@ public static partial class Operators /// If non-null, this parameter specifies a time to align the sampling messages with. If the parameter /// is non-null, the messages will have originating times that align with (i.e., are an integral number of intervals away from) the /// specified alignment time. - /// An optional delivery policy. + /// An optional delivery policy. + /// An optional name for the stream operator. /// Sampled stream. public static IProducer Sample( this IProducer source, TimeSpan samplingInterval, TimeSpan tolerance, DateTime? alignmentDateTime = null, - DeliveryPolicy deliveryPolicy = null) + DeliveryPolicy sourceDeliveryPolicy = null, + string name = nameof(Sample)) { return source.Interpolate( samplingInterval, Reproducible.Nearest(new RelativeTimeInterval(-tolerance, tolerance)), alignmentDateTime, - deliveryPolicy); + sourceDeliveryPolicy, + name); } /// @@ -102,20 +118,23 @@ public static partial class Operators /// is non-null, the messages will have originating times that align with (i.e., are an integral number of intervals away from) the /// specified alignment time. /// An optional delivery policy. + /// An optional name for the stream operator. /// Sampled stream. public static IProducer Sample( this IProducer source, TimeSpan samplingInterval, RelativeTimeInterval relativeTimeInterval = null, DateTime? alignmentDateTime = null, - DeliveryPolicy deliveryPolicy = null) + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Sample)) { relativeTimeInterval ??= RelativeTimeInterval.Infinite; return source.Interpolate( samplingInterval, Reproducible.Nearest(relativeTimeInterval), alignmentDateTime, - deliveryPolicy); + deliveryPolicy, + name); } /// @@ -129,15 +148,17 @@ public static partial class Operators /// The tolerance within which to search for the nearest message. /// An optional delivery policy for the source stream. /// An optional delivery policy for the clock stream. + /// An optional name for the stream operator. /// Sampled stream. public static IProducer Sample( this IProducer source, IProducer clock, TimeSpan tolerance, DeliveryPolicy sourceDeliveryPolicy = null, - DeliveryPolicy clockDeliveryPolicy = null) + DeliveryPolicy clockDeliveryPolicy = null, + string name = nameof(Sample)) { - return source.Interpolate(clock, Reproducible.Nearest(new RelativeTimeInterval(-tolerance, tolerance)), sourceDeliveryPolicy, clockDeliveryPolicy); + return source.Interpolate(clock, Reproducible.Nearest(new RelativeTimeInterval(-tolerance, tolerance)), sourceDeliveryPolicy, clockDeliveryPolicy, name); } /// @@ -153,16 +174,18 @@ public static partial class Operators /// used,resulting in sampling the nearest point to the clock signal on the source stream. /// An optional delivery policy for the source stream. /// An optional delivery policy for the clock stream. + /// An optional name for the stream operator. /// Sampled stream. public static IProducer Sample( this IProducer source, IProducer clock, RelativeTimeInterval relativeTimeInterval = null, DeliveryPolicy sourceDeliveryPolicy = null, - DeliveryPolicy clockDeliveryPolicy = null) + DeliveryPolicy clockDeliveryPolicy = null, + string name = nameof(Sample)) { relativeTimeInterval ??= RelativeTimeInterval.Infinite; - return source.Interpolate(clock, Reproducible.Nearest(relativeTimeInterval), sourceDeliveryPolicy, clockDeliveryPolicy); + return source.Interpolate(clock, Reproducible.Nearest(relativeTimeInterval), sourceDeliveryPolicy, clockDeliveryPolicy, name); } } } \ No newline at end of file diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Joins.cs b/Sources/Runtime/Microsoft.Psi/Operators/Joins.cs index 07b562618..10e3da712 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Joins.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Joins.cs @@ -13,7 +13,7 @@ namespace Microsoft.Psi /// public static partial class Operators { - #region Scalar joins + #region Scalar join operators /// /// Join with values from a secondary stream based on a specified reproducible interpolator. @@ -27,6 +27,7 @@ public static partial class Operators /// Function mapping the primary and secondary messages to an output message type. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined values. public static IProducer Join( this IProducer primary, @@ -34,7 +35,8 @@ public static partial class Operators ReproducibleInterpolator interpolator, Func outputCreator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -42,7 +44,8 @@ public static partial class Operators interpolator, (m, secondaryArray) => outputCreator(m, secondaryArray[0]), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name ?? $"{nameof(Join)}({interpolator})"); } /// @@ -55,14 +58,16 @@ public static partial class Operators /// Secondary stream. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined values. public static IProducer<(TPrimary, TSecondary)> Join( this IProducer primary, IProducer secondary, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { - return Join(primary, secondary, Reproducible.Exact(), primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Join(primary, secondary, Reproducible.Exact(), primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -76,15 +81,17 @@ public static partial class Operators /// Time tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined values. public static IProducer<(TPrimary, TSecondary)> Join( this IProducer primary, IProducer secondary, TimeSpan tolerance, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { - return Join(primary, secondary, new RelativeTimeInterval(-tolerance, tolerance), primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Join(primary, secondary, new RelativeTimeInterval(-tolerance, tolerance), primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -98,13 +105,15 @@ public static partial class Operators /// Relative time interval tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined values. public static IProducer<(TPrimary, TSecondary)> Join( this IProducer primary, IProducer secondary, RelativeTimeInterval relativeTimeInterval, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -112,7 +121,8 @@ public static partial class Operators Reproducible.Nearest(relativeTimeInterval), ValueTuple.Create, primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -125,15 +135,17 @@ public static partial class Operators /// Reproducible interpolator to use when joining the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined values. public static IProducer<(TPrimary, TSecondary)> Join( this IProducer primary, IProducer secondary, ReproducibleInterpolator interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { - return Join(primary, secondary, interpolator, ValueTuple.Create, primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Join(primary, secondary, interpolator, ValueTuple.Create, primaryDeliveryPolicy, secondaryDeliveryPolicy, name ?? $"{nameof(Join)}({interpolator})"); } #endregion Scalar joins @@ -151,13 +163,15 @@ public static partial class Operators /// Reproducible interpolator to use when joining the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 3. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2)> primary, IProducer secondary, ReproducibleInterpolator interpolator, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -165,7 +179,8 @@ public static partial class Operators interpolator, (p, s) => (p.Item1, p.Item2, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name ?? $"{nameof(Join)}({interpolator})"); } /// @@ -179,12 +194,14 @@ public static partial class Operators /// Secondary stream. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 3. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2)> primary, IProducer secondary, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -192,7 +209,8 @@ public static partial class Operators Reproducible.Exact(), (p, s) => (p.Item1, p.Item2, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -207,13 +225,15 @@ public static partial class Operators /// Time tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 3. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2)> primary, IProducer secondary, TimeSpan tolerance, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -221,7 +241,8 @@ public static partial class Operators Reproducible.Nearest(tolerance), (p, s) => (p.Item1, p.Item2, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -236,13 +257,15 @@ public static partial class Operators /// Relative time interval tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 3. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2)> primary, IProducer secondary, RelativeTimeInterval relativeTimeInterval, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -250,7 +273,8 @@ public static partial class Operators Reproducible.Nearest(relativeTimeInterval), (p, s) => (p.Item1, p.Item2, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -265,13 +289,15 @@ public static partial class Operators /// Reproducible interpolator to use when joining the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 4. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3)> primary, IProducer secondary, ReproducibleInterpolator interpolator, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -279,7 +305,8 @@ public static partial class Operators interpolator, (p, s) => (p.Item1, p.Item2, p.Item3, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name ?? $"{nameof(Join)}({interpolator})"); } /// @@ -294,12 +321,14 @@ public static partial class Operators /// Secondary stream. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 4. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3)> primary, IProducer secondary, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -307,7 +336,8 @@ public static partial class Operators Reproducible.Exact(), (p, s) => (p.Item1, p.Item2, p.Item3, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -323,13 +353,15 @@ public static partial class Operators /// Time tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 4. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3)> primary, IProducer secondary, TimeSpan tolerance, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -337,7 +369,8 @@ public static partial class Operators Reproducible.Nearest(tolerance), (p, s) => (p.Item1, p.Item2, p.Item3, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -353,13 +386,15 @@ public static partial class Operators /// Relative time interval tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 4. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3)> primary, IProducer secondary, RelativeTimeInterval relativeTimeInterval, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -367,7 +402,8 @@ public static partial class Operators Reproducible.Nearest(relativeTimeInterval), (p, s) => (p.Item1, p.Item2, p.Item3, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -383,13 +419,15 @@ public static partial class Operators /// Reproducible interpolator to use when joining the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 5. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4)> primary, IProducer secondary, ReproducibleInterpolator interpolator, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -397,7 +435,8 @@ public static partial class Operators interpolator, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name ?? $"{nameof(Join)}({interpolator})"); } /// @@ -413,12 +452,14 @@ public static partial class Operators /// Secondary stream. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 5. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4)> primary, IProducer secondary, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -426,7 +467,8 @@ public static partial class Operators Reproducible.Exact(), (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -442,13 +484,15 @@ public static partial class Operators /// Time tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 5. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4)> primary, IProducer secondary, TimeSpan tolerance, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -456,7 +500,8 @@ public static partial class Operators Reproducible.Nearest(tolerance), (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -473,13 +518,15 @@ public static partial class Operators /// Relative time interval tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 5. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4)> primary, IProducer secondary, RelativeTimeInterval relativeTimeInterval, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -487,7 +534,8 @@ public static partial class Operators Reproducible.Nearest(relativeTimeInterval), (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -504,13 +552,15 @@ public static partial class Operators /// Reproducible interpolator to use when joining the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 6. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5)> primary, IProducer secondary, ReproducibleInterpolator interpolator, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -518,7 +568,8 @@ public static partial class Operators interpolator, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name ?? $"{nameof(Join)}({interpolator})"); } /// @@ -535,12 +586,14 @@ public static partial class Operators /// Secondary stream. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 6. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5)> primary, IProducer secondary, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -548,7 +601,8 @@ public static partial class Operators Reproducible.Exact(), (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -566,13 +620,15 @@ public static partial class Operators /// Time tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 6. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5)> primary, IProducer secondary, TimeSpan tolerance, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -580,7 +636,8 @@ public static partial class Operators Reproducible.Nearest(tolerance), (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -598,13 +655,15 @@ public static partial class Operators /// Relative time interval tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 6. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5)> primary, IProducer secondary, RelativeTimeInterval relativeTimeInterval, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -612,7 +671,8 @@ public static partial class Operators Reproducible.Nearest(relativeTimeInterval), (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -630,13 +690,15 @@ public static partial class Operators /// Reproducible interpolator to use when joining the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 7. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6)> primary, IProducer secondary, ReproducibleInterpolator interpolator, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -644,7 +706,8 @@ public static partial class Operators interpolator, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, p.Item6, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name ?? $"{nameof(Join)}({interpolator})"); } /// @@ -662,12 +725,14 @@ public static partial class Operators /// Secondary stream. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 7. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6)> primary, IProducer secondary, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -675,7 +740,8 @@ public static partial class Operators Reproducible.Exact(), (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, p.Item6, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -694,13 +760,15 @@ public static partial class Operators /// Time tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 7. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6)> primary, IProducer secondary, TimeSpan tolerance, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -708,7 +776,8 @@ public static partial class Operators Reproducible.Nearest(tolerance), (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, p.Item6, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -727,13 +796,15 @@ public static partial class Operators /// Relative time interval tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 7. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6, TSecondary)> Join( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6)> primary, IProducer secondary, RelativeTimeInterval relativeTimeInterval, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -741,7 +812,8 @@ public static partial class Operators Reproducible.Nearest(relativeTimeInterval), (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, p.Item6, s), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } #endregion Tuple-flattening scalar joins @@ -759,13 +831,15 @@ public static partial class Operators /// Reproducible interpolator to use when joining the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 3. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2)> secondary, ReproducibleInterpolator<(TSecondaryItem1, TSecondaryItem2)> interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -773,7 +847,8 @@ public static partial class Operators interpolator, (p, s) => (p, s.Item1, s.Item2), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name ?? $"{nameof(Join)}({interpolator})"); } /// @@ -787,12 +862,14 @@ public static partial class Operators /// Secondary stream of tuples (arity 2). /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 3. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2)> secondary, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -800,7 +877,8 @@ public static partial class Operators Reproducible.Exact<(TSecondaryItem1, TSecondaryItem2)>(), (p, s) => (p, s.Item1, s.Item2), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -815,13 +893,15 @@ public static partial class Operators /// Time tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 3. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2)> secondary, TimeSpan tolerance, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -829,7 +909,8 @@ public static partial class Operators Reproducible.Nearest<(TSecondaryItem1, TSecondaryItem2)>(tolerance), (p, s) => (p, s.Item1, s.Item2), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -844,13 +925,15 @@ public static partial class Operators /// Relative time interval tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 3. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2)> secondary, RelativeTimeInterval relativeTimeInterval, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -858,7 +941,8 @@ public static partial class Operators Reproducible.Nearest<(TSecondaryItem1, TSecondaryItem2)>(relativeTimeInterval), (p, s) => (p, s.Item1, s.Item2), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -873,13 +957,15 @@ public static partial class Operators /// Reproducible interpolator to use when joining the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 4. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondary, ReproducibleInterpolator<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -887,7 +973,8 @@ public static partial class Operators interpolator, (p, s) => (p, s.Item1, s.Item2, s.Item3), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name ?? $"{nameof(Join)}({interpolator})"); } /// @@ -902,12 +989,14 @@ public static partial class Operators /// Secondary stream of tuples (arity 3). /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 4. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondary, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -915,7 +1004,8 @@ public static partial class Operators Reproducible.Exact<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)>(), (p, s) => (p, s.Item1, s.Item2, s.Item3), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -931,13 +1021,15 @@ public static partial class Operators /// Time tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 4. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondary, TimeSpan tolerance, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -945,7 +1037,8 @@ public static partial class Operators Reproducible.Nearest<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)>(tolerance), (p, s) => (p, s.Item1, s.Item2, s.Item3), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -961,13 +1054,15 @@ public static partial class Operators /// Relative time interval tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 4. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondary, RelativeTimeInterval relativeTimeInterval, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -975,7 +1070,8 @@ public static partial class Operators Reproducible.Nearest<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)>(relativeTimeInterval), (p, s) => (p, s.Item1, s.Item2, s.Item3), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -991,13 +1087,15 @@ public static partial class Operators /// Reproducible interpolator to use when joining the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 5. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondary, ReproducibleInterpolator<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -1005,7 +1103,8 @@ public static partial class Operators interpolator, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name ?? $"{nameof(Join)}({interpolator})"); } /// @@ -1021,12 +1120,14 @@ public static partial class Operators /// Secondary stream of tuples (arity 4). /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 5. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondary, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -1034,7 +1135,8 @@ public static partial class Operators Reproducible.Exact<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)>(), (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -1051,13 +1153,15 @@ public static partial class Operators /// Time tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 5. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondary, TimeSpan tolerance, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -1065,7 +1169,8 @@ public static partial class Operators Reproducible.Nearest<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)>(tolerance), (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -1082,13 +1187,15 @@ public static partial class Operators /// Relative time interval tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 5. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondary, RelativeTimeInterval relativeTimeInterval, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -1096,7 +1203,8 @@ public static partial class Operators Reproducible.Nearest<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)>(relativeTimeInterval), (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -1113,13 +1221,15 @@ public static partial class Operators /// Reproducible interpolator to use when joining the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 6. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondary, ReproducibleInterpolator<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -1127,7 +1237,8 @@ public static partial class Operators interpolator, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name ?? $"{nameof(Join)}({interpolator})"); } /// @@ -1144,12 +1255,14 @@ public static partial class Operators /// Secondary stream of tuples (arity 5). /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 6. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondary, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -1157,7 +1270,8 @@ public static partial class Operators Reproducible.Exact<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)>(), (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -1175,13 +1289,15 @@ public static partial class Operators /// Time tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 6. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondary, TimeSpan tolerance, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -1189,7 +1305,8 @@ public static partial class Operators Reproducible.Nearest<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)>(tolerance), (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -1207,13 +1324,15 @@ public static partial class Operators /// Relative time interval tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 6. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondary, RelativeTimeInterval relativeTimeInterval, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -1221,7 +1340,8 @@ public static partial class Operators Reproducible.Nearest<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)>(relativeTimeInterval), (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -1239,13 +1359,15 @@ public static partial class Operators /// Reproducible interpolator to use when joining the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 7. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondary, ReproducibleInterpolator<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -1253,7 +1375,8 @@ public static partial class Operators interpolator, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5, s.Item6), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name ?? $"{nameof(Join)}({interpolator})"); } /// @@ -1271,12 +1394,14 @@ public static partial class Operators /// Secondary stream of tuples (arity 6). /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 7. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondary, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -1284,7 +1409,8 @@ public static partial class Operators Reproducible.Exact<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)>(), (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5, s.Item6), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -1303,13 +1429,15 @@ public static partial class Operators /// Time tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 7. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondary, TimeSpan tolerance, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -1317,7 +1445,8 @@ public static partial class Operators Reproducible.Nearest<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)>(tolerance), (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5, s.Item6), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } /// @@ -1336,13 +1465,15 @@ public static partial class Operators /// Relative time interval tolerance. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of joined tuple values flattened to arity 7. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> Join( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondary, RelativeTimeInterval relativeTimeInterval, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondaryDeliveryPolicy = null, + string name = null) { return Join( primary, @@ -1350,7 +1481,8 @@ public static partial class Operators Reproducible.Nearest<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)>(relativeTimeInterval), (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5, s.Item6), primaryDeliveryPolicy, - secondaryDeliveryPolicy); + secondaryDeliveryPolicy, + name); } #endregion Reverse tuple-flattening scalar joins @@ -1369,6 +1501,7 @@ public static partial class Operators /// Mapping function from primary and secondary messages to output. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Output stream. public static IProducer Join( this IProducer primary, @@ -1376,14 +1509,16 @@ public static partial class Operators ReproducibleInterpolator interpolator, Func outputCreator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondariesDeliveryPolicy = null) + DeliveryPolicy secondariesDeliveryPolicy = null, + string name = null) { var join = new Join( primary.Out.Pipeline, interpolator, outputCreator, secondaries.Count(), - null); + null, + name ?? $"{nameof(Join)}({interpolator})"); primary.PipeTo(join.InPrimary, primaryDeliveryPolicy); @@ -1406,15 +1541,17 @@ public static partial class Operators /// Reproducible interpolator to use when joining the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Output stream. public static IProducer<(TPrimary, TSecondary[])> Join( this IProducer primary, IEnumerable> secondaries, ReproducibleInterpolator interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondariesDeliveryPolicy = null) + DeliveryPolicy secondariesDeliveryPolicy = null, + string name = null) { - return primary.Join(secondaries, interpolator, ValueTuple.Create, primaryDeliveryPolicy, secondariesDeliveryPolicy); + return primary.Join(secondaries, interpolator, ValueTuple.Create, primaryDeliveryPolicy, secondariesDeliveryPolicy, name ?? $"{nameof(Join)}({interpolator})"); } /// @@ -1430,6 +1567,7 @@ public static partial class Operators /// Mapping function from primary and secondary messages to output. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Output stream. public static IProducer Join( this IProducer primary, @@ -1437,14 +1575,16 @@ public static partial class Operators ReproducibleInterpolator interpolator, Func outputCreator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondariesDeliveryPolicy = null) + DeliveryPolicy secondariesDeliveryPolicy = null, + string name = null) { var join = new Join( primary.Out.Pipeline, interpolator, outputCreator, secondaries.Count(), - null); + null, + name ?? $"{nameof(Join)}({interpolator})"); primary.PipeTo(join.InPrimary, primaryDeliveryPolicy); @@ -1468,15 +1608,17 @@ public static partial class Operators /// Reproducible interpolator to use when joining the streams. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Output stream. public static IProducer<(TPrimary, TInterpolation[])> Join( this IProducer primary, IEnumerable> secondaries, ReproducibleInterpolator interpolator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondariesDeliveryPolicy = null) + DeliveryPolicy secondariesDeliveryPolicy = null, + string name = null) { - return primary.Join(secondaries, interpolator, (p, i) => (p, i), primaryDeliveryPolicy, secondariesDeliveryPolicy); + return primary.Join(secondaries, interpolator, (p, i) => (p, i), primaryDeliveryPolicy, secondariesDeliveryPolicy, name ?? $"{nameof(Join)}({interpolator})"); } /// @@ -1488,12 +1630,14 @@ public static partial class Operators /// Reproducible interpolator to use when joining the streams. /// Mapping function from input to output messages. /// An optional delivery policy to use for the streams. + /// An optional name for the stream operator. /// Output stream. public static IProducer Join( this IEnumerable> inputs, ReproducibleInterpolator interpolator, Func outputCreator, - DeliveryPolicy deliveryPolicy = null) + DeliveryPolicy deliveryPolicy = null, + string name = null) { var count = inputs.Count(); if (count > 1) @@ -1514,11 +1658,12 @@ public static partial class Operators return buffer; }, deliveryPolicy, - deliveryPolicy); + deliveryPolicy, + name ?? $"{nameof(Join)}({interpolator})"); } else if (count == 1) { - return inputs.First().Select(x => new[] { outputCreator(x) }, deliveryPolicy); + return inputs.First().Select(x => new[] { outputCreator(x) }, deliveryPolicy, name ?? $"{nameof(Join)}({interpolator})"); } else { @@ -1533,13 +1678,15 @@ public static partial class Operators /// Collection of input streams. /// Reproducible interpolator to use when joining the streams. /// An optional delivery policy to use for the streams. + /// An optional name for the stream operator. /// Output stream. public static IProducer Join( this IEnumerable> inputs, ReproducibleInterpolator interpolator, - DeliveryPolicy deliveryPolicy = null) + DeliveryPolicy deliveryPolicy = null, + string name = null) { - return inputs.Join(interpolator, _ => _, deliveryPolicy); + return inputs.Join(interpolator, _ => _, deliveryPolicy, name ?? $"{nameof(Join)}({interpolator})"); } #endregion Vector joins diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Merges.cs b/Sources/Runtime/Microsoft.Psi/Operators/Merges.cs index a10b0aee2..e76bf8894 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Merges.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Merges.cs @@ -20,15 +20,16 @@ public static partial class Operators /// Type of messages. /// Collection of homogeneous inputs. /// An optional delivery policy. + /// An optional name for this stream operator. /// Stream of merged messages. - public static IProducer> Merge(IEnumerable> inputs, DeliveryPolicy deliveryPolicy = null) + public static IProducer> Merge(IEnumerable> inputs, DeliveryPolicy deliveryPolicy = null, string name = nameof(Merge)) { if (inputs.Count() == 0) { throw new ArgumentException("Merge requires one or more inputs."); } - var merge = new Merge(inputs.First().Out.Pipeline); + var merge = new Merge(inputs.First().Out.Pipeline, name); foreach (var i in inputs) { i.PipeTo(merge.AddInput($"Receiver{i.Out.Id}"), deliveryPolicy); @@ -45,10 +46,9 @@ public static IProducer> Merge(IEnumerable> inputs, D /// First input stream. /// Second input stream with same message type. /// An optional delivery policy. + /// An optional name for this stream operator. /// Stream of merged messages. - public static IProducer> Merge(this IProducer input1, IProducer input2, DeliveryPolicy deliveryPolicy = null) - { - return Merge(new List>() { input1, input2 }, deliveryPolicy); - } + public static IProducer> Merge(this IProducer input1, IProducer input2, DeliveryPolicy deliveryPolicy = null, string name = nameof(Merge)) + => Merge(new List>() { input1, input2 }, deliveryPolicy, name); } } \ No newline at end of file diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Observable.cs b/Sources/Runtime/Microsoft.Psi/Operators/Observable.cs index 1fb32c9e8..aedab47d7 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Observable.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Observable.cs @@ -18,11 +18,10 @@ public static partial class Operators /// Type of messages for the source stream. /// The source stream. /// An optional delivery policy. + /// An optional name for this stream operator. /// Observable with elements from the source stream. - public static IObservable ToObservable(this IProducer stream, DeliveryPolicy deliveryPolicy = null) - { - return new StreamObservable(stream, deliveryPolicy); - } + public static IObservable ToObservable(this IProducer stream, DeliveryPolicy deliveryPolicy = null, string name = nameof(ToObservable)) + => new StreamObservable(stream, deliveryPolicy, name); /// /// Observable stream class. @@ -30,14 +29,15 @@ public static IObservable ToObservable(this IProducer stream, DeliveryP /// Type of stream messages. public class StreamObservable : IObservable { - private ConcurrentDictionary, IObserver> observers = new ConcurrentDictionary, IObserver>(); + private readonly ConcurrentDictionary, IObserver> observers = new (); /// /// Initializes a new instance of the class. /// /// The source stream to observe. /// An optional delivery policy. - public StreamObservable(IProducer stream, DeliveryPolicy deliveryPolicy = null) + /// An optional name for this stream operator. + public StreamObservable(IProducer stream, DeliveryPolicy deliveryPolicy = null, string name = nameof(StreamObservable)) { var processor = new Processor( stream.Out.Pipeline, @@ -49,7 +49,8 @@ public StreamObservable(IProducer stream, DeliveryPolicy deliveryPolicy = } s.Post(d, e.OriginatingTime); - }); + }, + name: name); stream.Out.PipeTo(processor, deliveryPolicy); @@ -76,8 +77,8 @@ public IDisposable Subscribe(IObserver observer) private class Unsubscriber : IDisposable { - private StreamObservable observable; - private IObserver observer; + private readonly StreamObservable observable; + private readonly IObserver observer; public Unsubscriber(StreamObservable observable, IObserver observer) { diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Pairs.cs b/Sources/Runtime/Microsoft.Psi/Operators/Pairs.cs index f57224a6e..c339b1b80 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Pairs.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Pairs.cs @@ -25,6 +25,7 @@ public static partial class Operators /// An initial value to be used until the first secondary message is received. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output values. public static IProducer Pair( this IProducer primary, @@ -32,10 +33,11 @@ public static partial class Operators Func outputCreator, TSecondary initialValue, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { return Pair( - new Pair(primary.Out.Pipeline, outputCreator, initialValue), + new Pair(primary.Out.Pipeline, outputCreator, initialValue, name), primary, secondary, primaryDeliveryPolicy, @@ -53,6 +55,7 @@ public static partial class Operators /// Mapping function from primary/secondary pairs to output type. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Primary messages will be dropped until the first secondary message is received (no `initialValue` provided). /// Stream of output values. public static IProducer Pair( @@ -60,10 +63,11 @@ public static partial class Operators IProducer secondary, Func outputCreator, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { return Pair( - new Pair(primary.Out.Pipeline, outputCreator), + new Pair(primary.Out.Pipeline, outputCreator, name), primary, secondary, primaryDeliveryPolicy, @@ -80,16 +84,18 @@ public static partial class Operators /// An initial value to be used until the first secondary message is received. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples. public static IProducer<(TPrimary, TSecondary)> Pair( this IProducer primary, IProducer secondary, TSecondary initialValue, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { return Pair( - new Pair(primary.Out.Pipeline, ValueTuple.Create, initialValue), + new Pair(primary.Out.Pipeline, ValueTuple.Create, initialValue, name), primary, secondary, primaryDeliveryPolicy, @@ -105,16 +111,18 @@ public static partial class Operators /// Secondary stream. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Primary messages will be dropped until the first secondary message is received (no `initialValue` provided). /// Stream of output tuples. public static IProducer<(TPrimary, TSecondary)> Pair( this IProducer primary, IProducer secondary, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { return Pair( - new Pair(primary.Out.Pipeline, ValueTuple.Create), + new Pair(primary.Out.Pipeline, ValueTuple.Create, name), primary, secondary, primaryDeliveryPolicy, @@ -136,15 +144,17 @@ public static partial class Operators /// An initial value to be used until the first secondary message is received. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 3. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TSecondary)> Pair( this IProducer<(TPrimaryItem1, TPrimaryItem2)> primary, IProducer secondary, TSecondary initialValue, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, s), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, s), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -157,14 +167,16 @@ public static partial class Operators /// Secondary stream. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 3. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TSecondary)> Pair( this IProducer<(TPrimaryItem1, TPrimaryItem2)> primary, IProducer secondary, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, s), primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, s), primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -179,15 +191,17 @@ public static partial class Operators /// An initial value to be used until the first secondary message is received. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 4. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TSecondary)> Pair( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3)> primary, IProducer secondary, TSecondary initialValue, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, s), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, s), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -201,14 +215,16 @@ public static partial class Operators /// Secondary stream. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 4. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TSecondary)> Pair( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3)> primary, IProducer secondary, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, s), primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, s), primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -224,15 +240,17 @@ public static partial class Operators /// An initial value to be used until the first secondary message is received. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 5. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TSecondary)> Pair( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4)> primary, IProducer secondary, TSecondary initialValue, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, s), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, s), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -247,14 +265,16 @@ public static partial class Operators /// Secondary stream. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 5. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TSecondary)> Pair( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4)> primary, IProducer secondary, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, s), primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, s), primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -271,15 +291,17 @@ public static partial class Operators /// An initial value to be used until the first secondary message is received. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 6. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TSecondary)> Pair( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5)> primary, IProducer secondary, TSecondary initialValue, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, s), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, s), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -295,14 +317,16 @@ public static partial class Operators /// Secondary stream. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 6. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TSecondary)> Pair( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5)> primary, IProducer secondary, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, s), primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, s), primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -320,15 +344,17 @@ public static partial class Operators /// An initial value to be used until the first secondary message is received. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 7. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6, TSecondary)> Pair( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6)> primary, IProducer secondary, TSecondary initialValue, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, p.Item6, s), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, p.Item6, s), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -345,14 +371,16 @@ public static partial class Operators /// Secondary stream. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 7. public static IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6, TSecondary)> Pair( this IProducer<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6)> primary, IProducer secondary, DeliveryPolicy<(TPrimaryItem1, TPrimaryItem2, TPrimaryItem3, TPrimaryItem4, TPrimaryItem5, TPrimaryItem6)> primaryDeliveryPolicy = null, - DeliveryPolicy secondaryDeliveryPolicy = null) + DeliveryPolicy secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, p.Item6, s), primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p.Item1, p.Item2, p.Item3, p.Item4, p.Item5, p.Item6, s), primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } #endregion Tuple-flattening scalar pairs @@ -370,15 +398,17 @@ public static partial class Operators /// An initial value to be used until the first secondary message is received. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 3. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2)> Pair( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2)> secondary, (TSecondaryItem1, TSecondaryItem2) initialValue, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2)> secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -391,14 +421,16 @@ public static partial class Operators /// Secondary stream of tuples (arity 2). /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 3. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2)> Pair( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2)> secondary, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2)> secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2), primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2), primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -413,15 +445,17 @@ public static partial class Operators /// An initial value to be used until the first secondary message is received. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 4. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> Pair( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondary, (TSecondaryItem1, TSecondaryItem2, TSecondaryItem3) initialValue, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -435,14 +469,16 @@ public static partial class Operators /// Secondary stream of tuples (arity 3). /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 4. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> Pair( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondary, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3)> secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3), primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3), primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -458,15 +494,17 @@ public static partial class Operators /// An initial value to be used until the first secondary message is received. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 5. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> Pair( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondary, (TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4) initialValue, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -481,14 +519,16 @@ public static partial class Operators /// Secondary stream of tuples (arity 4). /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 5. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> Pair( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondary, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4)> secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4), primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4), primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -505,15 +545,17 @@ public static partial class Operators /// An initial value to be used until the first secondary message is received. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 6. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> Pair( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondary, (TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5) initialValue, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -529,14 +571,16 @@ public static partial class Operators /// Secondary stream of tuples (arity 5). /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 6. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> Pair( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondary, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5)> secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5), primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5), primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -554,15 +598,17 @@ public static partial class Operators /// An initial value to be used until the first secondary message is received. /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 7. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> Pair( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondary, (TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6) initialValue, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5, s.Item6), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5, s.Item6), initialValue, primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } /// @@ -579,14 +625,16 @@ public static partial class Operators /// Secondary stream of tuples (arity 6). /// An optional delivery policy for the primary stream. /// An optional delivery policy for the secondary stream(s). + /// An optional name for the stream operator. /// Stream of output tuples flattened to arity 7. public static IProducer<(TPrimary, TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> Pair( this IProducer primary, IProducer<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondary, DeliveryPolicy primaryDeliveryPolicy = null, - DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondaryDeliveryPolicy = null) + DeliveryPolicy<(TSecondaryItem1, TSecondaryItem2, TSecondaryItem3, TSecondaryItem4, TSecondaryItem5, TSecondaryItem6)> secondaryDeliveryPolicy = null, + string name = nameof(Pair)) { - return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5, s.Item6), primaryDeliveryPolicy, secondaryDeliveryPolicy); + return Pair(primary, secondary, (p, s) => (p, s.Item1, s.Item2, s.Item3, s.Item4, s.Item5, s.Item6), primaryDeliveryPolicy, secondaryDeliveryPolicy, name); } #endregion Reverse tuple-flattening scalar pairs diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Pickers.cs b/Sources/Runtime/Microsoft.Psi/Operators/Pickers.cs index 46fd58a1e..9b13ca433 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Pickers.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Pickers.cs @@ -19,9 +19,10 @@ public static partial class Operators /// Source stream. /// Predicate function by which to filter messages. /// An optional delivery policy. + /// An optional name for this stream operator. /// Output stream. - public static IProducer Where(this IProducer source, Func condition, DeliveryPolicy deliveryPolicy = null) => - Process( + public static IProducer Where(this IProducer source, Func condition, DeliveryPolicy deliveryPolicy = null, string name = nameof(Where)) + => Process( source, (d, e, s) => { @@ -30,7 +31,8 @@ public static partial class Operators s.Post(d, e.OriginatingTime); } }, - deliveryPolicy); + deliveryPolicy, + name); /// /// Filter messages to those where a given condition is met. @@ -39,9 +41,10 @@ public static partial class Operators /// Source stream. /// Predicate function by which to filter messages. /// An optional delivery policy. + /// An optional name for this stream operator. /// Output stream. - public static IProducer Where(this IProducer source, Predicate condition, DeliveryPolicy deliveryPolicy = null) => - Where(source, (d, e) => condition(d), deliveryPolicy); + public static IProducer Where(this IProducer source, Predicate condition, DeliveryPolicy deliveryPolicy = null, string name = nameof(Where)) + => Where(source, (d, e) => condition(d), deliveryPolicy, name); /// /// Filter stream to the first n messages. @@ -50,9 +53,10 @@ public static partial class Operators /// Source stream. /// Number of messages. /// An optional delivery policy. + /// An optional name for this stream operator. /// Output stream. - public static IProducer First(this IProducer source, int number, DeliveryPolicy deliveryPolicy = null) => - source.Where(v => number-- > 0, deliveryPolicy); + public static IProducer First(this IProducer source, int number, DeliveryPolicy deliveryPolicy = null, string name = nameof(First)) + => source.Where(v => number-- > 0, deliveryPolicy, name); /// /// Filter stream to the first message (single-message stream). @@ -60,9 +64,10 @@ public static partial class Operators /// Type of source/output messages. /// Source stream. /// An optional delivery policy. + /// An optional name for this stream operator. /// An output stream containing only the first message. - public static IProducer First(this IProducer source, DeliveryPolicy deliveryPolicy = null) => - First(source, 1, deliveryPolicy); + public static IProducer First(this IProducer source, DeliveryPolicy deliveryPolicy = null, string name = nameof(First)) + => First(source, 1, deliveryPolicy, name); /// /// Filter stream to the last n messages. @@ -71,8 +76,9 @@ public static partial class Operators /// Source stream. /// The number of messages to filter. /// An optional delivery policy. + /// An optional name for this stream operator. /// An output stream containing only the last message. - public static IProducer Last(this IProducer source, int count, DeliveryPolicy deliveryPolicy = null) + public static IProducer Last(this IProducer source, int count, DeliveryPolicy deliveryPolicy = null, string name = nameof(Last)) { var lastValues = new List<(T, DateTime)>(); var processor = new Processor( @@ -91,7 +97,8 @@ public static IProducer Last(this IProducer source, int count, Delivery { emitter.Post(t, originatingTime); } - }); + }, + name); return source.PipeTo(processor, deliveryPolicy); } @@ -102,8 +109,9 @@ public static IProducer Last(this IProducer source, int count, Delivery /// Type of source/output messages. /// Source stream. /// An optional delivery policy. + /// An optional name for this stream operator. /// An output stream containing only the last message. - public static IProducer Last(this IProducer source, DeliveryPolicy deliveryPolicy = null) + public static IProducer Last(this IProducer source, DeliveryPolicy deliveryPolicy = null, string name = nameof(Last)) { var captured = false; T last = default; @@ -122,7 +130,8 @@ public static IProducer Last(this IProducer source, DeliveryPolicy d { emitter.Post(last, lastOriginatingTime); } - }); + }, + name); return source.PipeTo(processor, deliveryPolicy); } diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Processors.cs b/Sources/Runtime/Microsoft.Psi/Operators/Processors.cs index 9004e17c6..4f164f44e 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Processors.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Processors.cs @@ -22,10 +22,11 @@ public static partial class Operators /// The action to perform on every message in the source stream. /// The action parameters are the message, the envelope and an emitter to post results to. /// An optional delivery policy. + /// An optional name for this stream operator. /// A stream of type . - public static IProducer Process(this IProducer source, Action> transform, DeliveryPolicy deliveryPolicy = null) + public static IProducer Process(this IProducer source, Action> transform, DeliveryPolicy deliveryPolicy = null, string name = nameof(Process)) { - var select = new Processor(source.Out.Pipeline, transform); + var select = new Processor(source.Out.Pipeline, transform, name: name); return PipeTo(source, select, deliveryPolicy); } @@ -38,11 +39,10 @@ public static partial class Operators /// The source stream to subscribe to. /// The function to perform on every message in the source stream. The function takes two parameters, the input message and its envelope. /// An optional delivery policy. + /// An optional name for this stream operator. /// A stream of type . - public static IProducer Select(this IProducer source, Func selector, DeliveryPolicy deliveryPolicy = null) - { - return Process(source, (d, e, s) => s.Post(selector(d, e), e.OriginatingTime), deliveryPolicy); - } + public static IProducer Select(this IProducer source, Func selector, DeliveryPolicy deliveryPolicy = null, string name = nameof(Select)) + => Process(source, (d, e, s) => s.Post(selector(d, e), e.OriginatingTime), deliveryPolicy, name); /// /// Executes a transform function for each item in the input stream, generating a new stream with the values returned by the function. @@ -52,11 +52,10 @@ public static partial class Operators /// The source stream to subscribe to. /// The function to perform on every message in the source stream. /// An optional delivery policy. + /// An optional name for this stream operator. /// A stream of type . - public static IProducer Select(this IProducer source, Func selector, DeliveryPolicy deliveryPolicy = null) - { - return Select(source, (d, e) => selector(d), deliveryPolicy); - } + public static IProducer Select(this IProducer source, Func selector, DeliveryPolicy deliveryPolicy = null, string name = nameof(Select)) + => Select(source, (d, e) => selector(d), deliveryPolicy, name); /// /// Executes a transform function for each non-null item in the input stream, generating a new stream with the values returned by the function, or null if the input was null. @@ -67,13 +66,12 @@ public static partial class Operators /// The source stream to subscribe to. /// The function to perform on every message in the source stream. The function takes two parameters, the input message and its envelope. /// An optional delivery policy. + /// An optional name for this stream operator. /// A stream of type . - public static IProducer NullableSelect(this IProducer source, Func selector, DeliveryPolicy deliveryPolicy = null) + public static IProducer NullableSelect(this IProducer source, Func selector, DeliveryPolicy deliveryPolicy = null, string name = nameof(NullableSelect)) where TIn : struct where TOut : struct - { - return source.Select((v, e) => v.HasValue ? new TOut?(selector(v.Value, e)) : null, deliveryPolicy); - } + => source.Select((v, e) => v.HasValue ? new TOut?(selector(v.Value, e)) : null, deliveryPolicy, name); /// /// Executes a transform function for each non-null item in the input stream, generating a new stream with the values returned by the function, or null if the input was null. @@ -83,13 +81,12 @@ public static partial class Operators /// The source stream to subscribe to. /// The function to perform on every message in the source stream. /// An optional delivery policy. + /// An optional name for this stream operator. /// A stream of type . - public static IProducer NullableSelect(this IProducer source, Func selector, DeliveryPolicy deliveryPolicy = null) + public static IProducer NullableSelect(this IProducer source, Func selector, DeliveryPolicy deliveryPolicy = null, string name = nameof(NullableSelect)) where TIn : struct where TOut : struct - { - return source.Select(v => v.HasValue ? new TOut?(selector(v.Value)) : null, deliveryPolicy); - } + => source.Select(v => v.HasValue ? new TOut?(selector(v.Value)) : null, deliveryPolicy, name); /// /// Decomposes a stream of tuples into a stream containing just the first item of each tuple. @@ -98,11 +95,10 @@ public static partial class Operators /// The type of the second item in the tuple. /// The source stream of tuples. /// An optional delivery policy. + /// An optional name for this stream operator. /// A stream containing the first item of each tuple. - public static IProducer Item1(this IProducer<(T1, T2)> source, DeliveryPolicy<(T1, T2)> deliveryPolicy = null) - { - return source.Select(t => t.Item1, deliveryPolicy); - } + public static IProducer Item1(this IProducer<(T1, T2)> source, DeliveryPolicy<(T1, T2)> deliveryPolicy = null, string name = nameof(Item1)) + => source.Select(t => t.Item1, deliveryPolicy, name); /// /// Decomposes a stream of tuples into a stream containing just the second item of each tuple. @@ -111,11 +107,10 @@ public static partial class Operators /// The type of the second item in the tuple. /// The source stream of tuples. /// An optional delivery policy. + /// An optional name for this stream operator. /// A stream containing the second item of each tuple. - public static IProducer Item2(this IProducer<(T1, T2)> source, DeliveryPolicy<(T1, T2)> deliveryPolicy = null) - { - return source.Select(t => t.Item2, deliveryPolicy); - } + public static IProducer Item2(this IProducer<(T1, T2)> source, DeliveryPolicy<(T1, T2)> deliveryPolicy = null, string name = nameof(Item2)) + => source.Select(t => t.Item2, deliveryPolicy, name); /// /// Flip takes a tuple of 2 elements and flips their order. @@ -124,17 +119,17 @@ public static partial class Operators /// Type of second element. /// Source to read tuples from. /// An optional delivery policy. + /// An optional name for this stream operator. /// Returns a new producer with flipped tuples. - public static IProducer<(T2, T1)> Flip(this IProducer<(T1, T2)> source, DeliveryPolicy<(T1, T2)> deliveryPolicy = null) - { - return Process<(T1, T2), (T2, T1)>( + public static IProducer<(T2, T1)> Flip(this IProducer<(T1, T2)> source, DeliveryPolicy<(T1, T2)> deliveryPolicy = null, string name = nameof(Flip)) + => Process<(T1, T2), (T2, T1)>( source, (d, e, s) => { s.Post((d.Item2, d.Item1), e.OriginatingTime); }, - deliveryPolicy); - } + deliveryPolicy, + name); /// /// Executes an action for each item in the input stream and then outputs the item. If the action modifies the item, the resulting stream reflects the change. @@ -143,18 +138,18 @@ public static IProducer<(T2, T1)> Flip(this IProducer<(T1, T2)> source, /// The source stream to subscribe to. /// The action to perform on every message in the source stream. The action has access to the message envelope. /// An optional delivery policy. + /// An optional name for this stream operator. /// A stream of the same type as the source stream, containing one item for each input item, possibly modified by the action delegate. - public static IProducer Do(this IProducer source, Action action, DeliveryPolicy deliveryPolicy = null) - { - return Process( + public static IProducer Do(this IProducer source, Action action, DeliveryPolicy deliveryPolicy = null, string name = nameof(Do)) + => Process( source, (d, e, s) => { action(d, e); s.Post(d, e.OriginatingTime); }, - deliveryPolicy); - } + deliveryPolicy, + name); /// /// Executes an action for each item in the input stream and then outputs the item. If the action modifies the item, the resulting stream reflects the change. @@ -163,11 +158,10 @@ public static IProducer Do(this IProducer source, Action a /// The source stream to subscribe to. /// The action to perform on every message in the source stream. The action has access to the message envelope. /// An optional delivery policy. + /// An optional name for this stream operator. /// A stream of the same type as the source stream, containing one item for each input item, possibly modified by the action delegate. - public static IProducer Do(this IProducer source, Action action, DeliveryPolicy deliveryPolicy = null) - { - return Do(source, (d, e) => action(d), deliveryPolicy); - } + public static IProducer Do(this IProducer source, Action action, DeliveryPolicy deliveryPolicy = null, string name = nameof(Do)) + => Do(source, (d, e) => action(d), deliveryPolicy, name); /// /// Edit messages in a stream; applying updates/inserts and deletes. diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Time.cs b/Sources/Runtime/Microsoft.Psi/Operators/Time.cs index 8a6a564ea..b3603879b 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Time.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Time.cs @@ -17,11 +17,10 @@ public static partial class Operators /// Type of source stream messages. /// Source stream. /// An optional delivery policy. + /// An optional name for this stream operator. /// Stream of originating times. - public static IProducer TimeOf(this IProducer source, DeliveryPolicy deliveryPolicy = null) - { - return source.Select((_, e) => e.OriginatingTime, deliveryPolicy); - } + public static IProducer TimeOf(this IProducer source, DeliveryPolicy deliveryPolicy = null, string name = nameof(TimeOf)) + => source.Select((_, e) => e.OriginatingTime, deliveryPolicy, name); /// /// Map messages to their current latency (time since origination). @@ -29,11 +28,10 @@ public static IProducer TimeOf(this IProducer source, DeliveryPo /// Type of source stream messages. /// Source stream. /// An optional delivery policy. + /// An optional name for this stream operator. /// Stream of latency (time span) values. - public static IProducer Latency(this IProducer source, DeliveryPolicy deliveryPolicy = null) - { - return source.Select((_, e) => e.CreationTime - e.OriginatingTime, deliveryPolicy); - } + public static IProducer Latency(this IProducer source, DeliveryPolicy deliveryPolicy = null, string name = nameof(Latency)) + => source.Select((_, e) => e.CreationTime - e.OriginatingTime, deliveryPolicy, name); /// /// Delays the delivery of messages by a given time span. diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Timers.cs b/Sources/Runtime/Microsoft.Psi/Operators/Timers.cs index 033d0f1b8..cf0e63bfc 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Timers.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Timers.cs @@ -22,11 +22,10 @@ public static class Timers /// The pipeline to add the component to. /// The interval at which to generate messages. /// The function generating the messages. + /// An optional name for the component. /// A stream of messages of type T. - public static IProducer Timer(Pipeline pipeline, TimeSpan interval, Func generatorFn) - { - return new Timer(pipeline, (uint)interval.TotalMilliseconds, generatorFn); - } + public static IProducer Timer(Pipeline pipeline, TimeSpan interval, Func generatorFn, string name = nameof(Timer)) + => new Timer(pipeline, (uint)interval.TotalMilliseconds, generatorFn, name); /// /// Generates a stream of messages indicating the time elapsed from the start of the pipeline. @@ -37,10 +36,9 @@ public static IProducer Timer(Pipeline pipeline, TimeSpan interval, Func /// The pipeline to add the component to. /// The interval at which to generate messages. + /// An optional name for the component. /// A stream of messages representing time elapsed since the start of the pipeline. - public static IProducer Timer(Pipeline pipeline, TimeSpan interval) - { - return Timer(pipeline, interval, (_, t) => t); - } + public static IProducer Timer(Pipeline pipeline, TimeSpan interval, string name = nameof(Timer)) + => Timer(pipeline, interval, (_, t) => t, name); } } \ No newline at end of file diff --git a/Sources/Runtime/Microsoft.Psi/Operators/VectorProcessors.cs b/Sources/Runtime/Microsoft.Psi/Operators/VectorProcessors.cs index 5659ba3df..642343f4a 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/VectorProcessors.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/VectorProcessors.cs @@ -146,7 +146,7 @@ public static partial class Operators bool outputDefaultIfDropped = false, TOut defaultValue = default, DeliveryPolicy deliveryPolicy = null, - string name = null, + string name = nameof(Parallel), DeliveryPolicy defaultParallelDeliveryPolicy = null) { var p = new ParallelVariableLength(source.Out.Pipeline, streamTransform, outputDefaultIfDropped, defaultValue, name, defaultParallelDeliveryPolicy); @@ -168,7 +168,7 @@ public static partial class Operators this IProducer source, Action> streamAction, DeliveryPolicy deliveryPolicy = null, - string name = null, + string name = nameof(Parallel), DeliveryPolicy defaultParallelDeliveryPolicy = null) { var p = new ParallelVariableLength(source.Out.Pipeline, streamAction, name, defaultParallelDeliveryPolicy); @@ -258,7 +258,7 @@ public static partial class Operators TBranchOut defaultValue = default, DeliveryPolicy deliveryPolicy = null, Func, DateTime, (bool, DateTime)> branchTerminationPolicy = null, - string name = null, + string name = nameof(Parallel), DeliveryPolicy defaultParallelDeliveryPolicy = null) { var p = new ParallelSparseSelect( @@ -298,7 +298,7 @@ public static partial class Operators TBranchOut defaultValue = default, DeliveryPolicy> deliveryPolicy = null, Func, DateTime, (bool, DateTime)> branchTerminationPolicy = null, - string name = null, + string name = nameof(Parallel), DeliveryPolicy defaultParallelDeliveryPolicy = null) { var p = new ParallelSparseSelect, TBranchKey, TBranchIn, TBranchOut, Dictionary>( @@ -335,7 +335,7 @@ public static partial class Operators Action> streamAction, DeliveryPolicy deliveryPolicy = null, Func, DateTime, (bool, DateTime)> branchTerminationPolicy = null, - string name = null, + string name = nameof(Parallel), DeliveryPolicy defaultParallelDeliveryPolicy = null) { var p = new ParallelSparseDo( @@ -367,7 +367,7 @@ public static partial class Operators Action> streamAction, DeliveryPolicy> deliveryPolicy = null, Func, DateTime, (bool, DateTime)> branchTerminationPolicy = null, - string name = null, + string name = nameof(Parallel), DeliveryPolicy defaultParallelDeliveryPolicy = null) { var p = new ParallelSparseDo, TBranchKey, TBranchIn>( @@ -411,7 +411,7 @@ public static partial class Operators TBranchOut defaultValue = default, DeliveryPolicy deliveryPolicy = null, Func, DateTime, (bool, DateTime)> branchTerminationPolicy = null, - string name = null, + string name = nameof(Parallel), DeliveryPolicy defaultParallelDeliveryPolicy = null) { var p = new ParallelSparseSelect( @@ -454,7 +454,7 @@ public static partial class Operators TBranchOut defaultValue = default, DeliveryPolicy deliveryPolicy = null, Func, DateTime, (bool, DateTime)> branchTerminationPolicy = null, - string name = null, + string name = nameof(Parallel), DeliveryPolicy defaultParallelDeliveryPolicy = null) { var p = new ParallelSparseSelect>( @@ -494,7 +494,7 @@ public static partial class Operators TBranchOut defaultValue = default, DeliveryPolicy> deliveryPolicy = null, Func, DateTime, (bool, DateTime)> branchTerminationPolicy = null, - string name = null, + string name = nameof(Parallel), DeliveryPolicy defaultParallelDeliveryPolicy = null) { var p = new ParallelSparseSelect, TBranchKey, TBranchIn, TBranchOut, Dictionary>( @@ -531,7 +531,7 @@ public static partial class Operators Action> streamAction, DeliveryPolicy deliveryPolicy = null, Func, DateTime, (bool, DateTime)> branchTerminationPolicy = null, - string name = null, + string name = nameof(Parallel), DeliveryPolicy defaultParallelDeliveryPolicy = null) { var p = new ParallelSparseDo( @@ -563,7 +563,7 @@ public static partial class Operators Action> streamAction, DeliveryPolicy> deliveryPolicy = null, Func, DateTime, (bool, DateTime)> branchTerminationPolicy = null, - string name = null, + string name = nameof(Parallel), DeliveryPolicy defaultParallelDeliveryPolicy = null) { var p = new ParallelSparseDo, TBranchKey, TBranchIn>( diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Windows.cs b/Sources/Runtime/Microsoft.Psi/Operators/Windows.cs index 7cffbc398..ba2cc7a93 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Windows.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Windows.cs @@ -22,10 +22,16 @@ public static partial class Operators /// The relative index interval over which to gather messages. /// Selector function. /// An optional delivery policy. + /// An optional name for the stream operator. /// Output stream. - public static IProducer Window(this IProducer source, IntInterval indexInterval, Func>, TOutput> selector, DeliveryPolicy deliveryPolicy = null) + public static IProducer Window( + this IProducer source, + IntInterval indexInterval, + Func>, TOutput> selector, + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Window)) { - var window = new RelativeIndexWindow(source.Out.Pipeline, indexInterval, selector); + var window = new RelativeIndexWindow(source.Out.Pipeline, indexInterval, selector, name); return PipeTo(source, window, deliveryPolicy); } @@ -39,11 +45,16 @@ public static partial class Operators /// The relative index to which to gather messages. /// Selector function. /// An optional delivery policy. + /// An optional name for the stream operator. /// Output stream. - public static IProducer Window(this IProducer source, int fromIndex, int toIndex, Func>, TOutput> selector, DeliveryPolicy deliveryPolicy = null) - { - return Window(source, new IntInterval(fromIndex, toIndex), selector, deliveryPolicy); - } + public static IProducer Window( + this IProducer source, + int fromIndex, + int toIndex, + Func>, TOutput> selector, + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Window)) + => Window(source, new IntInterval(fromIndex, toIndex), selector, deliveryPolicy, name); /// /// Get windows of messages by relative index interval. @@ -52,11 +63,10 @@ public static partial class Operators /// Source stream. /// The relative index interval over which to gather messages. /// An optional delivery policy. + /// An optional name for the stream operator. /// Output stream. - public static IProducer Window(this IProducer source, IntInterval indexInterval, DeliveryPolicy deliveryPolicy = null) - { - return Window(source, indexInterval, GetMessageData, deliveryPolicy); - } + public static IProducer Window(this IProducer source, IntInterval indexInterval, DeliveryPolicy deliveryPolicy = null, string name = nameof(Window)) + => Window(source, indexInterval, GetMessageData, deliveryPolicy, name); /// /// Get windows of messages by relative index interval. @@ -66,11 +76,10 @@ public static IProducer Window(this IProducer sourc /// The relative index from which to gather messages. /// The relative index to which to gather messages. /// An optional delivery policy. + /// An optional name for the stream operator. /// Output stream. - public static IProducer Window(this IProducer source, int fromIndex, int toIndex, DeliveryPolicy deliveryPolicy = null) - { - return Window(source, fromIndex, toIndex, GetMessageData, deliveryPolicy); - } + public static IProducer Window(this IProducer source, int fromIndex, int toIndex, DeliveryPolicy deliveryPolicy = null, string name = nameof(Window)) + => Window(source, fromIndex, toIndex, GetMessageData, deliveryPolicy, name); /// /// Process windows of messages by relative time interval. @@ -81,10 +90,16 @@ public static IProducer Window(this IProducer sourc /// The relative time interval over which to gather messages. /// Selector function. /// An optional delivery policy. + /// An optional name for the stream operator. /// Output stream. - public static IProducer Window(this IProducer source, RelativeTimeInterval relativeTimeInterval, Func>, TOutput> selector, DeliveryPolicy deliveryPolicy = null) + public static IProducer Window( + this IProducer source, + RelativeTimeInterval relativeTimeInterval, + Func>, TOutput> selector, + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Window)) { - var window = new RelativeTimeWindow(source.Out.Pipeline, relativeTimeInterval, selector); + var window = new RelativeTimeWindow(source.Out.Pipeline, relativeTimeInterval, selector, name); return PipeTo(source, window, deliveryPolicy); } @@ -98,11 +113,16 @@ public static IProducer Window(this IProducer sourc /// The relative timespan to which to gather messages. /// Selector function. /// An optional delivery policy. + /// An optional name for the stream operator. /// Output stream. - public static IProducer Window(this IProducer source, TimeSpan fromTime, TimeSpan toTime, Func>, TOutput> selector, DeliveryPolicy deliveryPolicy = null) - { - return Window(source, new RelativeTimeInterval(fromTime, toTime), selector, deliveryPolicy); - } + public static IProducer Window( + this IProducer source, + TimeSpan fromTime, + TimeSpan toTime, + Func>, TOutput> selector, + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Window)) + => Window(source, new RelativeTimeInterval(fromTime, toTime), selector, deliveryPolicy, name); /// /// Get windows of messages by relative time interval. @@ -111,11 +131,14 @@ public static IProducer Window(this IProducer sourc /// Source stream of messages. /// The relative time interval over which to gather messages. /// An optional delivery policy. + /// An optional name for the stream operator. /// Output stream. - public static IProducer Window(this IProducer source, RelativeTimeInterval relativeTimeInterval, DeliveryPolicy deliveryPolicy = null) - { - return Window(source, relativeTimeInterval, GetMessageData, deliveryPolicy); - } + public static IProducer Window( + this IProducer source, + RelativeTimeInterval relativeTimeInterval, + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Window)) + => Window(source, relativeTimeInterval, GetMessageData, deliveryPolicy, name); /// /// Get windows of messages by relative time interval. @@ -125,11 +148,15 @@ public static IProducer Window(this IProducer sourc /// The relative timespan from which to gather messages. /// The relative timespan to which to gather messages. /// An optional delivery policy. + /// An optional name for the stream operator. /// Output stream. - public static IProducer Window(this IProducer source, TimeSpan fromTime, TimeSpan toTime, DeliveryPolicy deliveryPolicy = null) - { - return Window(source, new RelativeTimeInterval(fromTime, toTime), deliveryPolicy); - } + public static IProducer Window( + this IProducer source, + TimeSpan fromTime, + TimeSpan toTime, + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Window)) + => Window(source, new RelativeTimeInterval(fromTime, toTime), deliveryPolicy, name); /// /// Get windows of messages specified via data from an additional window-defining stream. @@ -148,6 +175,7 @@ public static IProducer Window(this IProducer sourc /// A function that creates output messages given a message on the window-defining stream and a buffer of messages on the source stream. /// An optional delivery policy for the source stream. /// An optional delivery policy for the window-defining stream. + /// An optional name for this operator. /// A stream of computed outputs. public static IProducer Window( this IProducer source, @@ -155,9 +183,10 @@ public static IProducer Window(this IProducer sourc Func, (TimeInterval, DateTime)> windowCreator, Func, IEnumerable>, TOutput> outputCreator, DeliveryPolicy sourceDeliveryPolicy = null, - DeliveryPolicy windowDeliveryPolicy = null) + DeliveryPolicy windowDeliveryPolicy = null, + string name = nameof(Window)) { - var dynamicWindow = new DynamicWindow(source.Out.Pipeline, windowCreator, outputCreator); + var dynamicWindow = new DynamicWindow(source.Out.Pipeline, windowCreator, outputCreator, name); window.PipeTo(dynamicWindow.WindowIn, windowDeliveryPolicy); source.PipeTo(dynamicWindow.In, sourceDeliveryPolicy); return dynamicWindow; @@ -178,25 +207,24 @@ public static IProducer Window(this IProducer sourc /// The function that creates the actual window to use at every point. /// An optional delivery policy for the source stream. /// An optional delivery policy for the window-defining stream. + /// An optional name for this operator. /// A stream of computed outputs. public static IProducer[]> Window( this IProducer source, IProducer window, Func, (TimeInterval, DateTime)> windowCreator, DeliveryPolicy sourceDeliveryPolicy = null, - DeliveryPolicy windowDeliveryPolicy = null) - { - return source.Window( + DeliveryPolicy windowDeliveryPolicy = null, + string name = nameof(Window)) + => source.Window( window, windowCreator, (_, messages) => messages.ToArray(), sourceDeliveryPolicy, - windowDeliveryPolicy); - } + windowDeliveryPolicy, + name); private static T[] GetMessageData(IEnumerable> messages) - { - return messages.Select(m => m.Data).ToArray(); - } + => messages.Select(m => m.Data).ToArray(); } } diff --git a/Sources/Runtime/Microsoft.Psi/Operators/Zips.cs b/Sources/Runtime/Microsoft.Psi/Operators/Zips.cs index 21592184e..4c6074817 100644 --- a/Sources/Runtime/Microsoft.Psi/Operators/Zips.cs +++ b/Sources/Runtime/Microsoft.Psi/Operators/Zips.cs @@ -22,15 +22,16 @@ public static partial class Operators /// Type of messages. /// Collection of input streams to zip. /// An optional delivery policy. + /// An optional name for this stream operator. /// Stream of zipped messages. - public static IProducer Zip(IEnumerable> inputs, DeliveryPolicy deliveryPolicy = null) + public static IProducer Zip(IEnumerable> inputs, DeliveryPolicy deliveryPolicy = null, string name = nameof(Zip)) { if (inputs.Count() == 0) { throw new ArgumentException($"{nameof(Zip)} requires one or more inputs."); } - var zip = new Zip(inputs.First().Out.Pipeline); + var zip = new Zip(inputs.First().Out.Pipeline, name); foreach (var i in inputs) { i.PipeTo(zip.AddInput($"Receiver{i.Out.Id}"), deliveryPolicy); @@ -49,10 +50,9 @@ public static IProducer Zip(IEnumerable> inputs, DeliveryPo /// First input stream. /// Second input stream with same message type. /// An optional delivery policy. + /// An optional name for this stream operator. /// Stream of zipped messages. - public static IProducer Zip(this IProducer input1, IProducer input2, DeliveryPolicy deliveryPolicy = null) - { - return Zip(new List>() { input1, input2 }, deliveryPolicy); - } + public static IProducer Zip(this IProducer input1, IProducer input2, DeliveryPolicy deliveryPolicy = null, string name = nameof(Zip)) + => Zip(new List>() { input1, input2 }, deliveryPolicy, name); } } \ No newline at end of file diff --git a/Sources/Runtime/Microsoft.Psi/Persistence/MessageReader.cs b/Sources/Runtime/Microsoft.Psi/Persistence/MessageReader.cs index b06c3c3e0..2dc0aee52 100644 --- a/Sources/Runtime/Microsoft.Psi/Persistence/MessageReader.cs +++ b/Sources/Runtime/Microsoft.Psi/Persistence/MessageReader.cs @@ -86,7 +86,7 @@ public unsafe int Read(byte* buffer, int size) public void Dispose() { - this.fileReader.Dispose(); + this.fileReader?.Dispose(); this.fileReader = null; } } diff --git a/Sources/Runtime/Microsoft.Psi/Persistence/PsiStoreCommon.cs b/Sources/Runtime/Microsoft.Psi/Persistence/PsiStoreCommon.cs index a9fc4d00c..f4e324cee 100644 --- a/Sources/Runtime/Microsoft.Psi/Persistence/PsiStoreCommon.cs +++ b/Sources/Runtime/Microsoft.Psi/Persistence/PsiStoreCommon.cs @@ -54,6 +54,12 @@ internal static bool TryGetPathToLatestVersion(string appName, string rootPath, var fileName = GetDataFileName(appName); fullPath = Path.GetFullPath(rootPath); + + if (!Directory.Exists(fullPath)) + { + return false; + } + if (!Directory.EnumerateFiles(fullPath, fileName + "*").Any()) { fullPath = Directory.EnumerateDirectories(fullPath, appName + ".*").OrderByDescending(d => Directory.GetCreationTimeUtc(d)).FirstOrDefault(); @@ -126,11 +132,13 @@ internal static int GetDeterministicHashCode(this string str) private static IEnumerable<(string Name, string Session, string Path)> EnumerateStores(string rootPath, string currentPath, bool recursively) { // scan for any psi catalog files - var catalogFile = $"*.{CatalogFileName}_000000.psi"; - foreach (var filename in Directory.EnumerateFiles(currentPath, catalogFile)) + var catalogFileSuffix = $".{CatalogFileName}_000000.psi"; + var catalogFileSearchPattern = $"*{catalogFileSuffix}"; + + foreach (var filename in Directory.EnumerateFiles(currentPath, catalogFileSearchPattern)) { var fi = new FileInfo(filename); - var storeName = fi.Name.Substring(0, fi.Name.Length - catalogFile.Length); + var storeName = fi.Name.Substring(0, fi.Name.Length - catalogFileSuffix.Length); var sessionName = (currentPath == rootPath) ? filename : Path.Combine(currentPath, filename).Substring(rootPath.Length); sessionName = sessionName.Substring(0, sessionName.Length - fi.Name.Length); sessionName = sessionName.Trim('\\'); diff --git a/Sources/Runtime/Microsoft.Psi/Persistence/PsiStoreWriter.cs b/Sources/Runtime/Microsoft.Psi/Persistence/PsiStoreWriter.cs index 85ea25c2e..d059310de 100644 --- a/Sources/Runtime/Microsoft.Psi/Persistence/PsiStoreWriter.cs +++ b/Sources/Runtime/Microsoft.Psi/Persistence/PsiStoreWriter.cs @@ -41,16 +41,17 @@ public sealed class PsiStoreWriter : IDisposable private readonly Dictionary metadata = new (); private readonly BufferWriter metadataBuffer = new (128); private readonly BufferWriter indexBuffer = new (24); - private MessageWriter largeMessageWriter; - private int unindexedBytes = IndexPageSize; - private IndexEntry nextIndexEntry; /// /// This file is opened in exclusive share mode when the exporter is constructed, and is /// deleted when it gets disposed. Other processes can check the live status of the store /// by attempting to also open this file. If that fails, then the store is still live. /// - private FileStream liveMarkerFile; + private readonly FileStream liveMarkerFile; + + private MessageWriter largeMessageWriter; + private int unindexedBytes = IndexPageSize; + private IndexEntry nextIndexEntry; /// /// Initializes a new instance of the class. @@ -146,34 +147,33 @@ public void Dispose() /// The metadata describing the stream to open. /// The complete metadata for the stream just created. public PsiStreamMetadata OpenStream(PsiStreamMetadata metadata) - { - return this.OpenStream(metadata.Id, metadata.Name, metadata.IsIndexed, metadata.TypeName).UpdateSupplementalMetadataFrom(metadata); - } + => this.OpenStream(metadata.Id, metadata.Name, metadata.IsIndexed, metadata.TypeName, metadata.OpenedTime).UpdateSupplementalMetadataFrom(metadata); /// /// Creates a stream to write messages to. /// - /// The id of the stream, unique for this store. All messages with this stream id will be written to this stream. - /// The name of the stream. This name can be later used to open the stream for reading. + /// The id of the stream, unique for this store. All messages with this stream id will be written to this stream. + /// The name of the stream. This name can be later used to open the stream for reading. /// Indicates whether the stream is indexed or not. Indexed streams have a small index entry in the main data file and the actual message body in a large data file. /// A name identifying the type of the messages in this stream. This is usually a fully-qualified type name or a data contract name, but can be anything that the caller wants. + /// The opened time for the stream. /// The complete metadata for the stream just created. - public PsiStreamMetadata OpenStream(int streamId, string streamName, bool indexed, string typeName) + public PsiStreamMetadata OpenStream(int id, string name, bool indexed, string typeName, DateTime streamOpenedTime) { - if (this.metadata.ContainsKey(streamId)) + if (this.metadata.ContainsKey(id)) { - throw new InvalidOperationException($"The stream id {streamId} has already been registered with this writer."); + throw new InvalidOperationException($"The stream id {id} has already been registered with this writer."); } - var psiStreamMetadata = new PsiStreamMetadata(streamName, streamId, typeName) + var psiStreamMetadata = new PsiStreamMetadata(name, id, typeName) { - OpenedTime = Time.GetCurrentTime(), + OpenedTime = streamOpenedTime, IsPersisted = true, IsIndexed = indexed, StoreName = this.name, StorePath = this.path, }; - this.metadata[streamId] = psiStreamMetadata; + this.metadata[id] = psiStreamMetadata; this.WriteToCatalog(psiStreamMetadata); // make sure we have a large file if needed @@ -204,7 +204,18 @@ public void CloseStream(int streamId, DateTime originatingTime) var meta = this.metadata[streamId]; if (!meta.IsClosed) { - meta.ClosedTime = meta.OpenedTime <= originatingTime ? originatingTime : meta.OpenedTime; + // When a store is being rewritten (e.g. to crop, repair, etc.) the OpenedTime can + // be after the closing originatingTime originally stored. Originating times remain + // in the timeframe of the original store, while Opened/ClosedTimes in the rewritten + // store reflect the wall-clock time at which each stream was rewritten. + // Technically, the rewritten streams are opened, written, and closed in quick + // succession, but we record an interval at least large enough to envelope the + // originating time interval. However, a stream with zero or one message will still + // show an empty interval (ClosedTime = OpenedTime). + meta.ClosedTime = + meta.OpenedTime <= originatingTime ? // opened before/at closing time? + originatingTime : // using closing time + meta.OpenedTime + meta.MessageOriginatingTimeInterval.Span; // o/w assume closed after span of messages meta.IsClosed = true; this.WriteToCatalog(meta); } @@ -231,6 +242,7 @@ public void InitializeStreamOpenedTimes(DateTime originatingTime) foreach (var meta in this.metadata.Values) { meta.OpenedTime = originatingTime; + this.WriteToCatalog(meta); } } @@ -284,6 +296,12 @@ public void Write(BufferReader buffer, Envelope envelope) /// The type schema. internal void WriteToCatalog(TypeSchema typeSchema) => this.WriteToCatalog((Metadata)typeSchema); + /// + /// Writes the psi stream metadata to the catalog. + /// + /// The psi stream metadata schema. + internal void WriteToCatalog(PsiStreamMetadata typeSchema) => this.WriteToCatalog((Metadata)typeSchema); + /// /// Writes details about a stream to the stream catalog. /// diff --git a/Sources/Runtime/Microsoft.Psi/Remoting/RemoteClockExporter.cs b/Sources/Runtime/Microsoft.Psi/Remoting/RemoteClockExporter.cs index fb599819c..9c38d0cbd 100644 --- a/Sources/Runtime/Microsoft.Psi/Remoting/RemoteClockExporter.cs +++ b/Sources/Runtime/Microsoft.Psi/Remoting/RemoteClockExporter.cs @@ -15,8 +15,12 @@ namespace Microsoft.Psi.Remoting /// public class RemoteClockExporter : IDisposable { + /// + /// Default TCP port used to communicate with . + /// + public const int DefaultPort = 11511; + internal const short ProtocolVersion = 0; - internal const int DefaultPort = 11511; private TcpListener listener; private bool isDisposing; diff --git a/Sources/Runtime/Microsoft.Psi/Remoting/RemoteClockImporter.cs b/Sources/Runtime/Microsoft.Psi/Remoting/RemoteClockImporter.cs index 63a206116..31ba18a7c 100644 --- a/Sources/Runtime/Microsoft.Psi/Remoting/RemoteClockImporter.cs +++ b/Sources/Runtime/Microsoft.Psi/Remoting/RemoteClockImporter.cs @@ -16,6 +16,7 @@ namespace Microsoft.Psi.Remoting public class RemoteClockImporter : IDisposable { private readonly Pipeline pipeline; + private readonly string name; private readonly string host; private readonly int port; private readonly TcpClient client; @@ -27,9 +28,11 @@ public class RemoteClockImporter : IDisposable /// The pipeline to add the component to. /// The host name of the remote clock exporter/server. /// The port on which to connect. - public RemoteClockImporter(Pipeline pipeline, string host, int port = RemoteClockExporter.DefaultPort) + /// An optional name for the component. + public RemoteClockImporter(Pipeline pipeline, string host, int port = RemoteClockExporter.DefaultPort, string name = nameof(RemoteClockImporter)) { this.pipeline = pipeline; + this.name = name; this.client = new TcpClient(); this.host = host; this.port = port; @@ -58,6 +61,9 @@ public void Dispose() this.connected.Dispose(); } + /// + public override string ToString() => this.name; + private void SynchronizeLocalPipelineClock() { var completed = false; diff --git a/Sources/Runtime/Microsoft.Psi/Remoting/RemoteImporter.cs b/Sources/Runtime/Microsoft.Psi/Remoting/RemoteImporter.cs index 802b8d24c..b452d693f 100644 --- a/Sources/Runtime/Microsoft.Psi/Remoting/RemoteImporter.cs +++ b/Sources/Runtime/Microsoft.Psi/Remoting/RemoteImporter.cs @@ -112,7 +112,15 @@ public RemoteImporter(Importer importer, string host, int port = RemoteExporter. { } - private RemoteImporter(Func importerThunk, TimeInterval replay, bool replayRemoteLatestStart, string host, int port, string name, string path, bool allowSequenceRestart) + private RemoteImporter( + Func importerThunk, + TimeInterval replay, + bool replayRemoteLatestStart, + string host, + int port, + string storeName, + string storePath, + bool allowSequenceRestart) { this.importerThunk = importerThunk; this.replayStart = replay.Left.Ticks; @@ -121,7 +129,7 @@ private RemoteImporter(Func importerThunk, TimeInterval replay this.host = host; this.port = port; this.allowSequenceRestart = allowSequenceRestart; - this.storeWriter = new PsiStoreWriter(name, path); + this.storeWriter = new PsiStoreWriter(storeName, storePath); this.StartMetaClient(); } diff --git a/Sources/Runtime/Microsoft.Psi/Scheduling/FutureWorkItemQueue.cs b/Sources/Runtime/Microsoft.Psi/Scheduling/FutureWorkItemQueue.cs index 943ce48e5..501f71349 100644 --- a/Sources/Runtime/Microsoft.Psi/Scheduling/FutureWorkItemQueue.cs +++ b/Sources/Runtime/Microsoft.Psi/Scheduling/FutureWorkItemQueue.cs @@ -18,7 +18,10 @@ public FutureWorkItemQueue(string name, Scheduler scheduler) protected override bool DequeueCondition(WorkItem item) { - return item.StartTime <= this.scheduler.Clock.GetCurrentTime(); + // Dequeue work item if it is due for execution, or will never be executed + // due to the scheduler context being finalized before its execution time. + return item.StartTime <= this.scheduler.Clock.GetCurrentTime() + || item.StartTime > item.SchedulerContext.FinalizeTime; } } } diff --git a/Sources/Runtime/Microsoft.Psi/Scheduling/Scheduler.cs b/Sources/Runtime/Microsoft.Psi/Scheduling/Scheduler.cs index ef8892e24..c601ea031 100644 --- a/Sources/Runtime/Microsoft.Psi/Scheduling/Scheduler.cs +++ b/Sources/Runtime/Microsoft.Psi/Scheduling/Scheduler.cs @@ -16,7 +16,7 @@ public sealed class Scheduler : IDisposable private readonly Func errorHandler; private readonly bool allowSchedulingOnExternalThreads; private readonly ManualResetEvent stopped = new ManualResetEvent(true); - private readonly AutoResetEvent futureAdded = new AutoResetEvent(false); + private readonly AutoResetEvent futureQueuePulse = new AutoResetEvent(false); private readonly ThreadLocal nextWorkitem = new ThreadLocal(); private readonly ThreadLocal isSchedulerThread = new ThreadLocal(() => false); private readonly ThreadLocal currentWorkitemTime = new ThreadLocal(() => DateTime.MaxValue); @@ -82,7 +82,7 @@ public void Dispose() { this.threadSemaphore.Dispose(); this.stopped.Dispose(); - this.futureAdded.Dispose(); + this.futureQueuePulse.Dispose(); this.globalWorkitems.Dispose(); this.futureWorkitems.Dispose(); this.nextWorkitem.Dispose(); @@ -364,6 +364,10 @@ internal void PauseForQuiescence(SchedulerContext context) return; } + // Pulse the future queue to process any work items that are either due or + // non-reachable due to having a start time past the finalization time. + this.futureQueuePulse.Set(); + if (!this.isSchedulerThread.Value && !this.allowSchedulingOnExternalThreads) { // this is not a scheduler thread, so just wait for the context to empty and return @@ -417,17 +421,21 @@ internal void PauseForQuiescence(SchedulerContext context) /// Flag whether to execute once current operation completes. private void Schedule(WorkItem wi, bool asContinuation = true) { + // Exit the context without executing the work item if: + // (1) a forced shutdown has been initiated + // (2) the scheduler has already been stopped (completed) if (this.forcedShutdownRequested || this.completed) { wi.SchedulerContext.Exit(); return; } - // if the workitem not yet due, add it to the future workitem queue - if ((wi.StartTime > wi.SchedulerContext.Clock.GetCurrentTime() && this.delayFutureWorkitemsUntilDue) || !this.started || !wi.SchedulerContext.Started) + // if the work item is not yet due, add it to the future work item queue, as long as it is not after the finalize time + if ((wi.StartTime > wi.SchedulerContext.Clock.GetCurrentTime() && wi.StartTime <= wi.SchedulerContext.FinalizeTime && this.delayFutureWorkitemsUntilDue) || + !this.started || !wi.SchedulerContext.Started) { this.futureWorkitems.Enqueue(wi); - this.futureAdded.Set(); + this.futureQueuePulse.Set(); return; } @@ -582,7 +590,7 @@ private void ProcessFutureQueue() { int waitTimeout = -1; - var workReadyHandles = new EventWaitHandle[] { this.stopped, this.futureAdded }; + var workReadyHandles = new EventWaitHandle[] { this.stopped, this.futureQueuePulse }; var allHandles = new[] { this.threadSemaphore.Empty, this.globalWorkitems.Empty, this.futureWorkitems.Empty }; var spinWait = default(SpinWait); while (true) diff --git a/Sources/Runtime/Microsoft.Psi/Serialization/DictionarySerializer.cs b/Sources/Runtime/Microsoft.Psi/Serialization/DictionarySerializer.cs new file mode 100644 index 000000000..42d2c3b89 --- /dev/null +++ b/Sources/Runtime/Microsoft.Psi/Serialization/DictionarySerializer.cs @@ -0,0 +1,227 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Serialization +{ + using System.Collections.Generic; + using System.Linq; + using System.Reflection.Emit; + using Microsoft.Psi.Common; + + /// + /// Provides serialization and cloning methods for objects. + /// + internal sealed class DictionarySerializer : ISerializer> + { + // Bumping the schema version number from the auto-generated version which defaults to RuntimeInfo.CurrentRuntimeVersion (2) + private const int SchemaVersion = 3; + + private ISerializer> innerSerializer; + + /// + public bool? IsClearRequired => true; + + /// + public TypeSchema Initialize(KnownSerializers serializers, TypeSchema targetSchema) + { + if (targetSchema?.Version <= 2) + { + // maintain backward compatibility with older serialized data + this.innerSerializer = new ClassSerializer>(); + } + else + { + // otherwise default to the new implementation + this.innerSerializer = new DictionarySerializerImpl(); + } + + return this.innerSerializer.Initialize(serializers, targetSchema); + } + + /// + public void Serialize(BufferWriter writer, Dictionary instance, SerializationContext context) + { + this.innerSerializer.Serialize(writer, instance, context); + } + + /// + public void PrepareDeserializationTarget(BufferReader reader, ref Dictionary target, SerializationContext context) + { + this.innerSerializer.PrepareDeserializationTarget(reader, ref target, context); + } + + /// + public void Deserialize(BufferReader reader, ref Dictionary target, SerializationContext context) + { + this.innerSerializer.Deserialize(reader, ref target, context); + } + + /// + public void PrepareCloningTarget(Dictionary instance, ref Dictionary target, SerializationContext context) + { + this.innerSerializer.PrepareCloningTarget(instance, ref target, context); + } + + /// + public void Clone(Dictionary instance, ref Dictionary target, SerializationContext context) + { + this.innerSerializer.Clone(instance, ref target, context); + } + + /// + public void Clear(ref Dictionary target, SerializationContext context) + { + this.innerSerializer.Clear(ref target, context); + } + + /// + /// Provides serialization and cloning methods for objects. + /// + private class DictionarySerializerImpl : ISerializer> + { + private SerializationHandler> comparerHandler; + private SerializationHandler[]> entriesHandler; + private SetComparerDelegate setComparerImpl; + + private delegate void SetComparerDelegate(Dictionary target, IEqualityComparer value); + + /// + public bool? IsClearRequired => true; + + /// + public TypeSchema Initialize(KnownSerializers serializers, TypeSchema targetSchema) + { + // Add a comparerHandler of type IEqualityComparer. This will take care of serializing, + // deserializing and cloning the Comparer member of Dictionary. Because the comparer field + // is private, we will also need to generate a dynamic method so that we can set the + // comparer field upon deserialization or cloning. This method should be invoked right after + // clearing the target Dictionary, before adding the deserialized or cloned entries to it. + this.comparerHandler = serializers.GetHandler>(); + this.setComparerImpl = this.GenerateSetComparerMethod(); + + // Use an array serializer to serialize the dictionary elements as an array of key-value pairs + this.entriesHandler = serializers.GetHandler[]>(); + + var type = typeof(Dictionary); + var name = TypeSchema.GetContractName(type, serializers.RuntimeVersion); + + // Treat the Dictionary as a class with 2 members - a comparer and an array of key-value pairs + var comparerMember = new TypeMemberSchema("Comparer", typeof(IEqualityComparer).AssemblyQualifiedName, true); + var entriesMember = new TypeMemberSchema("KeyValuePairs", typeof(KeyValuePair[]).AssemblyQualifiedName, true); + var schema = new TypeSchema(name, TypeSchema.GetId(name), type.AssemblyQualifiedName, TypeFlags.IsClass, new[] { comparerMember, entriesMember }, SchemaVersion); + + return targetSchema ?? schema; + } + + /// + public void Serialize(BufferWriter writer, Dictionary instance, SerializationContext context) + { + this.comparerHandler.Serialize(writer, instance.Comparer, context); + this.entriesHandler.Serialize(writer, instance.ToArray(), context); + } + + /// + public void PrepareDeserializationTarget(BufferReader reader, ref Dictionary target, SerializationContext context) + { + if (target == null) + { + target = new Dictionary(); + } + + // Note that we don't clear an existing target dictionary as we want to preserve its elements until the + // call to Deserialize which will attempt to re-use any existing target elements for deserializing into. + } + + /// + public void Deserialize(BufferReader reader, ref Dictionary target, SerializationContext context) + { + var targetComparer = target.Comparer; + this.comparerHandler.Deserialize(reader, ref targetComparer, context); + + // Deserialize into an array of existing target elements, then add them back to the dictionary + var targetElements = target.ToArray(); + + // Following this call, targetElements will be an array containing the deserialized items + this.entriesHandler.Deserialize(reader, ref targetElements, context); + + // Clear and add the deserialized items into the target dictionary + target.Clear(); + + // Call dynamic method to set the private comparer field + this.setComparerImpl(target, targetComparer); + + foreach (var element in targetElements) + { + target.Add(element.Key, element.Value); + } + } + + /// + public void PrepareCloningTarget(Dictionary instance, ref Dictionary target, SerializationContext context) + { + if (target == null) + { + target = new Dictionary(instance.Count); + } + + // Note that we don't clear an existing target dictionary as we want to preserve its elements until the + // call to Clone which will attempt to re-use any existing target elements for cloning into. + } + + /// + public void Clone(Dictionary instance, ref Dictionary target, SerializationContext context) + { + var targetComparer = target.Comparer; + this.comparerHandler.Clone(instance.Comparer, ref targetComparer, context); + + // Clone into an array of existing target elements, then add them back to the dictionary + var targetElements = target.ToArray(); + + // Following this call, targetElements will be an array containing the cloned items + this.entriesHandler.Clone(instance.ToArray(), ref targetElements, context); + + // Clear and add the cloned items into the target dictionary + target.Clear(); + + // Call dynamic method to set the private comparer field + this.setComparerImpl(target, targetComparer); + + foreach (var element in targetElements) + { + target.Add(element.Key, element.Value); + } + } + + /// + public void Clear(ref Dictionary target, SerializationContext context) + { + // Note that this clears the items in the dictionary, but does not actually remove them + // i.e. the dictionary will contain the cleared items upon returning from this method. + var items = target.ToArray(); + this.entriesHandler.Clear(ref items, context); + + var comparer = target.Comparer; + this.comparerHandler.Clear(ref comparer, context); + } + + private SetComparerDelegate GenerateSetComparerMethod() + { + // Create DynamicMethod using delegate's Invoke method as prototype + var prototype = typeof(SetComparerDelegate).GetMethod(nameof(SetComparerDelegate.Invoke)); + var method = new DynamicMethod(prototype.Name, prototype.ReturnType, prototype.GetParameters().Select(p => p.ParameterType).ToArray(), this.GetType(), true); + var il = method.GetILGenerator(); + + // Get the comparer field (uses the first field of type IEquialityComparer) + var field = Generator.GetAllFields(typeof(Dictionary)).Where(fi => fi.FieldType == typeof(IEqualityComparer)).First(); + + // Generate code to set the comparer field directly (in place of field.SetValue(target, value)) + il.Emit(OpCodes.Ldarg_0); // target + il.Emit(OpCodes.Ldarg_1); // value + il.Emit(OpCodes.Stfld, field); // target.comparer = value + il.Emit(OpCodes.Ret); + + return (SetComparerDelegate)method.CreateDelegate(typeof(SetComparerDelegate)); + } + } + } +} diff --git a/Sources/Runtime/Microsoft.Psi/Serialization/KnownSerializers.cs b/Sources/Runtime/Microsoft.Psi/Serialization/KnownSerializers.cs index debf7def9..c218930a5 100644 --- a/Sources/Runtime/Microsoft.Psi/Serialization/KnownSerializers.cs +++ b/Sources/Runtime/Microsoft.Psi/Serialization/KnownSerializers.cs @@ -169,6 +169,7 @@ private KnownSerializers(bool isDefault, RuntimeInfo runtimeVersion) this.Register(); this.Register(); this.RegisterGenericSerializer(typeof(EnumerableSerializer<>)); + this.RegisterGenericSerializer(typeof(DictionarySerializer<,>)); } else { diff --git a/Sources/Runtime/Microsoft.Psi/Serialization/RecyclingPool.cs b/Sources/Runtime/Microsoft.Psi/Serialization/RecyclingPool.cs index 3105ea2e4..52efe8bf5 100644 --- a/Sources/Runtime/Microsoft.Psi/Serialization/RecyclingPool.cs +++ b/Sources/Runtime/Microsoft.Psi/Serialization/RecyclingPool.cs @@ -37,14 +37,15 @@ public static IRecyclingPool Create(StackTrace debugTrace = null) private class Cloner : IRecyclingPool { private const int MaxAllocationsWithoutRecycling = 100; - private readonly SerializationContext serializationContext = new SerializationContext(); - private readonly Stack free = new Stack(); // not ConcurrentStack because ConcurrentStack performs an allocation for each Push. We want to be allocation free. - private int outstandingAllocationCount; + private readonly SerializationContext serializationContext = new (); + private readonly Stack free = new (); // not ConcurrentStack because ConcurrentStack performs an allocation for each Push. We want to be allocation free. #if TRACKLEAKS - private StackTrace debugTrace; + private readonly StackTrace debugTrace; private bool recycledOnce; #endif + private int outstandingAllocationCount; + public Cloner(StackTrace debugTrace = null) { #if TRACKLEAKS @@ -77,7 +78,7 @@ public T Get() } else { - clone = default(T); + clone = default; } this.outstandingAllocationCount++; @@ -86,8 +87,8 @@ public T Get() // alert if the component is not recycling messages if (!this.recycledOnce && this.outstandingAllocationCount == MaxAllocationsWithoutRecycling && this.debugTrace != null) { - StringBuilder sb = new StringBuilder("\\psi output **********************************************"); - sb.AppendLine($"This component is not recycling messages {this.GetType()}. Constructor stack trace below:"); + var sb = new StringBuilder("\\psi output **********************************************"); + sb.AppendLine($"This component is not recycling messages {typeof(T)} (no recycling after {this.outstandingAllocationCount} allocations). Constructor stack trace below:"); foreach (var frame in this.debugTrace.GetFrames()) { sb.AppendLine($"{frame.GetFileName()}({frame.GetFileLineNumber()}): {frame.GetMethod().DeclaringType}.{frame.GetMethod().Name}"); @@ -133,10 +134,7 @@ private class FakeCloner : IRecyclingPool public int AvailableAllocationCount => 0; - public T Get() - { - return default(T); - } + public T Get() => default; public void Recycle(T freeInstance) { diff --git a/Sources/Runtime/Microsoft.Psi/Streams/Receiver{T}.cs b/Sources/Runtime/Microsoft.Psi/Streams/Receiver{T}.cs index b9f32ced8..85be9f10c 100644 --- a/Sources/Runtime/Microsoft.Psi/Streams/Receiver{T}.cs +++ b/Sources/Runtime/Microsoft.Psi/Streams/Receiver{T}.cs @@ -271,14 +271,7 @@ internal void Receive(Message message) this.pipeline.DiagnosticsConfiguration.TrackMessageSize ? this.ComputeDataSize(message.Data) : 0, diagnosticsTime); - var ownerToString = this.Owner.ToString(); - - // If the owner to-string rendering is different from the default one (i.e., the type name) - if (ownerToString != this.Owner.GetType().ToString()) - { - // Then add this as diagnostic state information - this.receiverDiagnosticsCollector.Value.UpdateDiagnosticState(ownerToString); - } + this.receiverDiagnosticsCollector.Value.UpdateDiagnosticState(this.Owner.ToString()); } return; @@ -341,14 +334,7 @@ internal void DeliverNext() this.pipeline.DiagnosticsConfiguration.TrackMessageSize ? this.ComputeDataSize(message.Data) : 0, currentTime); - var ownerToString = this.Owner.ToString(); - - // If the owner to-string rendering is different from the default one (i.e., the type name) - if (ownerToString != this.Owner.GetType().ToString()) - { - // Then add this as diagnostic state information - this.receiverDiagnosticsCollector.Value.UpdateDiagnosticState(ownerToString); - } + this.receiverDiagnosticsCollector.Value.UpdateDiagnosticState(this.Owner.ToString()); } if (this.counters != null) diff --git a/Sources/Runtime/Test.Psi/CrossFrameworkSerializationTester.cs b/Sources/Runtime/Test.Psi/CrossFrameworkSerializationTester.cs index 74ed62e3f..57739c28b 100644 --- a/Sources/Runtime/Test.Psi/CrossFrameworkSerializationTester.cs +++ b/Sources/Runtime/Test.Psi/CrossFrameworkSerializationTester.cs @@ -89,6 +89,7 @@ public void CrossFrameworkSerialize() Array intArray = new[] { 0, 3 }; ICollection stringArray = new[] { "three", "four" }; IEqualityComparer enumComparer = EqualityComparer.Default; + Dictionary dictionary = new Dictionary { { "one", 1 }, { "two", 2 } }; using (var p = Pipeline.Create()) { @@ -112,6 +113,7 @@ public void CrossFrameworkSerialize() Generators.Return(p, intArray).Write("intArray", store); Generators.Return(p, stringArray).Write("stringArray", store); Generators.Return(p, enumComparer).Write("enumComparer", store); + Generators.Return(p, dictionary).Write("dictionary", store); p.Run(); } @@ -150,6 +152,7 @@ public void CrossFrameworkDeserialize() Array intArray = null; ICollection stringArray = null; IEqualityComparer enumComparer = null; + Dictionary dictionary = null; using (var p = Pipeline.Create()) { @@ -174,6 +177,7 @@ public void CrossFrameworkDeserialize() store.OpenStream("intArray").Do(x => intArray = x.DeepClone()); store.OpenStream("stringArray").Do(x => stringArray = x.DeepClone()); store.OpenStream("enumComparer").Do(x => enumComparer = x.DeepClone()); + store.OpenStream>("dictionary").Do(x => dictionary = x.DeepClone()); p.Run(); } @@ -198,6 +202,9 @@ public void CrossFrameworkDeserialize() CollectionAssert.AreEqual(new[] { 0, 3 }, intArray); CollectionAssert.AreEqual(new[] { "three", "four" }, stringArray); Assert.IsTrue(enumComparer.Equals(DayOfWeek.Friday, DayOfWeek.Friday)); + Assert.AreEqual(2, dictionary.Count); + Assert.AreEqual(1, dictionary["one"]); + Assert.AreEqual(2, dictionary["two"]); } public class TypeMembers @@ -221,6 +228,7 @@ public class TypeMembers public Array IntArray; public ICollection StringArray; public IEqualityComparer EnumComparer; + public Dictionary Dictionary; } /// @@ -255,6 +263,7 @@ public void CrossFrameworkSerializeMembers() IntArray = new[] { 0, 3 }, StringArray = new[] { "three", "four" }, EnumComparer = EqualityComparer.Default, + Dictionary = new Dictionary { { "one", 1 }, { "two", 2 } }, }; var store = PsiStore.Create(p, "Store2", this.testPath); @@ -305,6 +314,9 @@ public void CrossFrameworkDeserializeMembers() CollectionAssert.AreEqual(new[] { 782, 33 }, result.ValueTuple.Item2); CollectionAssert.AreEqual(new[] { "three", "four" }, result.StringArray); Assert.IsTrue(result.EnumComparer.Equals(DayOfWeek.Friday, DayOfWeek.Friday)); + Assert.AreEqual(2, result.Dictionary.Count); + Assert.AreEqual(1, result.Dictionary["one"]); + Assert.AreEqual(2, result.Dictionary["two"]); } private void ExecuteTest(string testPath, string testMethod) diff --git a/Sources/Runtime/Test.Psi/PipelineTest.cs b/Sources/Runtime/Test.Psi/PipelineTest.cs index 2d8d62482..49398d0ab 100644 --- a/Sources/Runtime/Test.Psi/PipelineTest.cs +++ b/Sources/Runtime/Test.Psi/PipelineTest.cs @@ -9,6 +9,7 @@ namespace Test.Psi using System.Reactive; using System.Reactive.Linq; using System.Threading; + using System.Threading.Tasks; using Microsoft.Psi; using Microsoft.Psi.Components; using Microsoft.Psi.Diagnostics; @@ -572,6 +573,27 @@ public void SubpipelineShutdownWithLatency() } } + [TestMethod] + [Timeout(60000)] + public void PipelineShutdownWithPendingMessage() + { + var mre = new ManualResetEvent(false); + var p = Pipeline.Create("root"); + + // Post two messages with an interval of 10 seconds + Generators.Repeat(p, 0, 2, TimeSpan.FromSeconds(10000)).Do(_ => mre.Set()); + + // Run the pipeline, and stop it as soon as the first message is seen + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + p.RunAsync(); + mre.WaitOne(); + p.Dispose(); + stopwatch.Stop(); + + // The pipeline should shutdown without waiting for the Generator's loopback message + Assert.IsTrue(stopwatch.ElapsedMilliseconds < 5000); + } + [TestMethod] [Timeout(60000)] [ExpectedException(typeof(InvalidOperationException))] @@ -2238,6 +2260,36 @@ public void NestedOnPipelineRun() } } + [TestMethod] + [Timeout(60000)] + public void PipelineDispose() + { + var log = new List(); + var p = Pipeline.Create(); + var c = new FinalizationTestComponent(p, "C", log); + var b = new FinalizationTestComponent(p, "B", log); + var a = new FinalizationTestComponent(p, "A", log); + Assert.IsTrue(p.IsInitial); + + // Test dispose before starting the pipeline + p.Dispose(); + Assert.IsTrue(p.IsCompleted); + Assert.IsTrue(log.Count == 0); + + log.Clear(); + p = Pipeline.Create(); + c = new FinalizationTestComponent(p, "C", log); + b = new FinalizationTestComponent(p, "B", log); + a = new FinalizationTestComponent(p, "A", log); + p.RunAsync(); + Assert.IsTrue(p.IsRunning); + + // Tests for resilience to double-dispose + Parallel.For(0, 10, _ => p.Dispose()); + Assert.IsTrue(p.IsCompleted); + Assert.IsTrue(log.Count == 15); + } + private class FinalizationTestComponent : ISourceComponent { private readonly Pipeline pipeline; diff --git a/Sources/Runtime/Test.Psi/SerializationTester.cs b/Sources/Runtime/Test.Psi/SerializationTester.cs index 62a57a52d..8ebc78baa 100644 --- a/Sources/Runtime/Test.Psi/SerializationTester.cs +++ b/Sources/Runtime/Test.Psi/SerializationTester.cs @@ -357,6 +357,76 @@ public void SerializeDictionary() Serializer.Clear(ref clonedDict, new SerializationContext()); } + private class IntegerEqualityComparer : IEqualityComparer + { + public bool Equals(double x, double y) => (int)x == (int)y; + + public int GetHashCode(double obj) => ((int)obj).GetHashCode(); + } + + [TestMethod] + [Timeout(60000)] + public void SerializeDictionaryWithComparer() + { + var dict = new Dictionary(new IntegerEqualityComparer()); + dict.Add(0, "zero"); + dict.Add(1, "one"); + + var clonedDict = dict.DeepClone(); + dict.DeepClone(ref clonedDict); + foreach (var key in dict.Keys) + { + Assert.AreEqual(dict[key], clonedDict[key]); + } + + clonedDict[0.9] = "0"; + clonedDict[1.1] = "1"; + Assert.AreEqual("0", clonedDict[0]); + Assert.AreEqual("1", clonedDict[1]); + + var buf = new byte[256]; + clonedDict = this.SerializationClone(dict, buf); + foreach (var key in dict.Keys) + { + Assert.AreEqual(dict[key], clonedDict[key]); + } + + clonedDict[0.9] = "0"; + clonedDict[1.1] = "1"; + Assert.AreEqual("0", clonedDict[0]); + Assert.AreEqual("1", clonedDict[1]); + + Serializer.Clear(ref clonedDict, new SerializationContext()); + } + + [TestMethod] + [Timeout(60000)] + public void DictionaryBackCompat() + { + // Represents a Dictionary { { 0, "zero" }, { 1, "one" } } serialized using the previous scheme (auto-generated ClassSerializer) + var buf = new byte[] + { + 0, 0, 0, 128, 0, 0, 0, 128, 3, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 3, 0, 0, 0, + 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 4, 0, 0, 0, 122, 101, 114, 111, 255, 255, 255, 255, + 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 128, 3, 0, 0, 0, 111, 110, 101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, 0, 0, 0, 255, 255, 255, 255, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, + }; + + // Create the known serializers and register the old version of the Dictionary schema. This simulates what would be read from an older store. + var serializers = new KnownSerializers(); + var oldSchema = TypeSchema.FromType(typeof(Dictionary), new RuntimeInfo(serializationSystemVersion: 2), typeof(ClassSerializer>), serializerVersion: 1); + serializers.RegisterSchema(oldSchema); + + // Deserialize the buffer using a SerializationContext initialized with the old schema + var br = new BufferReader(buf); + var dict = default(Dictionary); + Serializer.Deserialize(br, ref dict, new SerializationContext(serializers)); + + Assert.AreEqual(2, dict.Count); + Assert.AreEqual("zero", dict[0]); + Assert.AreEqual("one", dict[1]); + } + [TestMethod] [Timeout(60000)] public void SerializeObjectDictionary() @@ -384,12 +454,18 @@ public void SerializeObjectDictionary() [Timeout(60000)] public void SerializeDictionaryTree() { - var innerDict = new Dictionary(); - innerDict.Add(0, new STClass(FooEnum.One, 1, "one", new[] { 1.0 })); - innerDict.Add(1, new STClass(FooEnum.Two, 2, "two", new[] { 2.0 })); + var innerDict1 = new Dictionary(); + innerDict1.Add(0, new STClass(FooEnum.One, 1, "one", new[] { 1.0 })); + innerDict1.Add(1, new STClass(FooEnum.Two, 2, "two", new[] { 2.0 })); + + var innerDict2 = new Dictionary(); + innerDict2.Add(0, new STClass(FooEnum.Two, 3, "two", new[] { 2.0 })); + innerDict2.Add(1, new STClass(FooEnum.One, 4, "one", new[] { 1.0 })); var dict = new Dictionary, Dictionary>(); - dict.Add(Tuple.Create(0), innerDict); + dict.Add(Tuple.Create(0), innerDict1); + dict.Add(Tuple.Create(1), innerDict1); // testing duplicate reference + dict.Add(Tuple.Create(2), innerDict2); var clonedDict = dict.DeepClone(); dict.DeepClone(ref clonedDict); @@ -401,6 +477,10 @@ public void SerializeDictionaryTree() Assert.AreNotEqual(dict[key][1], clonedDict[key][1]); } + // verify duplicate reference + Assert.AreSame(dict[Tuple.Create(0)], dict[Tuple.Create(1)]); + Assert.AreNotSame(dict[Tuple.Create(1)], dict[Tuple.Create(2)]); + var buf = new byte[256]; clonedDict = this.SerializationClone(dict, buf); foreach (var key in dict.Keys) @@ -410,6 +490,10 @@ public void SerializeDictionaryTree() Assert.AreNotEqual(dict[key][0], clonedDict[key][0]); Assert.AreNotEqual(dict[key][1], clonedDict[key][1]); } + + // verify duplicate reference + Assert.AreSame(dict[Tuple.Create(0)], dict[Tuple.Create(1)]); + Assert.AreNotSame(dict[Tuple.Create(1)], dict[Tuple.Create(2)]); } [TestMethod] diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Adapters.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Adapters.cs index 919a8dd7d..c0e7d4ee0 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Adapters.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Adapters.cs @@ -8,162 +8,65 @@ namespace Microsoft.Psi.Spatial.Euclidean.Visualization { using System.Collections.Generic; using System.Linq; + using MathNet.Spatial.Euclidean; using Microsoft.Psi.Spatial.Euclidean; using Microsoft.Psi.Visualization.Adapters; using Microsoft.Psi.Visualization.Data; /// - /// Adapter for encoded image rectangles. + /// Adapter for to nullable . /// [StreamAdapter] - public class EncodedImageRectangle3DAdapter : StreamAdapter + public class Rectangle3DToNullableRectangle3DAdapter : StreamAdapter { - private readonly EncodedImageToImageAdapter imageAdapter = new (); - - /// - public override ImageRectangle3D GetAdaptedValue(EncodedImageRectangle3D source, Envelope envelope) - { - if (source != null) - { - var encodedImage = this.imageAdapter.GetAdaptedValue(source.Image, envelope); - if (encodedImage != null) - { - return new ImageRectangle3D(source.Rectangle3D, encodedImage); - } - } - - return null; - } - /// - public override void Dispose(ImageRectangle3D destination) - { - if (destination != null) - { - this.imageAdapter.Dispose(destination.Image); - } - } + public override Rectangle3D? GetAdaptedValue(Rectangle3D source, Envelope envelope) + => source; } /// - /// Adapter for encoded depth image rectangles. + /// Adapter for list of to list of nullable . /// [StreamAdapter] - public class EncodedDepthImageRectangle3DAdapter : StreamAdapter + public class Rectangle3DListToNullableRectangle3DListAdapter : StreamAdapter, List> { - private readonly EncodedDepthImageToDepthImageAdapter imageAdapter = new (); - /// - public override DepthImageRectangle3D GetAdaptedValue(EncodedDepthImageRectangle3D source, Envelope envelope) - { - if (source != null) - { - var encodedDepthImage = this.imageAdapter.GetAdaptedValue(source.DepthImage, envelope); - if (encodedDepthImage != null) - { - return new DepthImageRectangle3D(source.Rectangle3D, encodedDepthImage); - } - } - - return null; - } - - /// - public override void Dispose(DepthImageRectangle3D destination) - { - if (destination != null) - { - this.imageAdapter.Dispose(destination.DepthImage); - } - } + public override List GetAdaptedValue(List source, Envelope envelope) + => source?.Select(p => p as Rectangle3D?).ToList(); } /// - /// Adapter for list of nullable to list of nullable . + /// Adapter from list of to . /// [StreamAdapter] - public class NullableEncodedDepthImageRectangle3DListAdapter : StreamAdapter, List> + public class Mesh3DListToPointCloud3DAdapter : StreamAdapter, PointCloud3D> { - private readonly EncodedDepthImageToDepthImageAdapter imageAdapter = new (); - /// - public override List GetAdaptedValue(List source, Envelope envelope) + public override PointCloud3D GetAdaptedValue(List source, Envelope envelope) { - if (source != null) + if (source == null) { - List outputList = new (); - foreach (var inputRectangle in source) - { - if (inputRectangle != null) - { - var encodedDepthImage = this.imageAdapter.GetAdaptedValue(inputRectangle.DepthImage, envelope); - if (encodedDepthImage != null) - { - outputList.Add(new DepthImageRectangle3D(inputRectangle.Rectangle3D, encodedDepthImage)); - } - } - } - - return outputList; + return PointCloud3D.Empty; } - return null; - } - - /// - public override void Dispose(List destination) - { - foreach (var imgRect in destination) + var points = new List(); + foreach (var mesh in source) { - if (imgRect != null) - { - this.imageAdapter.Dispose(imgRect.DepthImage); - } + points.AddRange(mesh.Vertices); } - } - - /// - /// Adapter for to nullable . - /// - [StreamAdapter] - public class Rectangle3DToNullableAdapter : StreamAdapter - { - /// - public override Rectangle3D? GetAdaptedValue(Rectangle3D source, Envelope envelope) - => source; - } - /// - /// Adapter for list of to list of nullable . - /// - [StreamAdapter] - public class Rectangle3DListToNullableAdapter : StreamAdapter, List> - { - /// - public override List GetAdaptedValue(List source, Envelope envelope) - => source?.Select(p => p as Rectangle3D?).ToList(); - } - - /// - /// Adapter for to nullable . - /// - [StreamAdapter] - public class Box3DToNullableAdapter : StreamAdapter - { - /// - public override Box3D? GetAdaptedValue(Box3D source, Envelope envelope) - => source; + return new PointCloud3D(points); } + } - /// - /// Adapter for list of to list of nullable . - /// - [StreamAdapter] - public class Box3DListToNullableAdapter : StreamAdapter, List> - { - /// - public override List GetAdaptedValue(List source, Envelope envelope) - => source?.Select(p => p as Box3D?).ToList(); - } + /// + /// Adapter from to . + /// + [StreamAdapter] + public class Mesh3DToPointCloud3DAdapter : StreamAdapter + { + /// + public override PointCloud3D GetAdaptedValue(Mesh3D source, Envelope envelope) + => source?.ToPointCloud3D(); } } \ No newline at end of file diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Box3DListVisualizationObject.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Box3DListVisualizationObject.cs index 8e0172269..a13fc1aac 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Box3DListVisualizationObject.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Box3DListVisualizationObject.cs @@ -10,7 +10,7 @@ namespace Microsoft.Psi.Spatial.Euclidean.Visualization /// Implements a visualization object that can display lists of 3D boxes. /// [VisualizationObject("3D boxes")] - public class Box3DListVisualizationObject : ModelVisual3DListVisualizationObject + public class Box3DListVisualizationObject : ModelVisual3DListVisualizationObject { } } diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Box3DVisualizationObject.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Box3DVisualizationObject.cs index 3e06c3168..80d10b738 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Box3DVisualizationObject.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Box3DVisualizationObject.cs @@ -17,7 +17,7 @@ namespace Microsoft.Psi.Spatial.Euclidean.Visualization /// Implements a 3D box visualization object. /// [VisualizationObject("3D box")] - public class Box3DVisualizationObject : ModelVisual3DVisualizationObject + public class Box3DVisualizationObject : ModelVisual3DVisualizationObject { // Edge properties private Color edgeColor = Colors.White; @@ -146,10 +146,10 @@ public bool FacetVisible /// public override void UpdateData() { - if (this.CurrentData.HasValue) + if (this.CurrentData != null) { // Update the edge locations. - var box = this.CurrentData.Value; + var box = this.CurrentData; var p0 = (box.Origin + box.XAxis.ScaleBy(box.Bounds.Min.X) + box.YAxis.ScaleBy(box.Bounds.Min.Y) + box.ZAxis.ScaleBy(box.Bounds.Min.Z)).ToPoint3D(); var p1 = (box.Origin + box.XAxis.ScaleBy(box.Bounds.Max.X) + box.YAxis.ScaleBy(box.Bounds.Min.Y) + box.ZAxis.ScaleBy(box.Bounds.Min.Z)).ToPoint3D(); var p2 = (box.Origin + box.XAxis.ScaleBy(box.Bounds.Max.X) + box.YAxis.ScaleBy(box.Bounds.Max.Y) + box.ZAxis.ScaleBy(box.Bounds.Min.Z)).ToPoint3D(); @@ -175,7 +175,7 @@ public override void UpdateData() // Update the facets. for (int i = 0; i < this.facets.Length; i++) { - var rectangle = this.CurrentData.Value.GetFacet((Box3DFacet)Enum.GetValues(typeof(Box3DFacet)).GetValue(i)); + var rectangle = box.GetFacet((Box3DFacet)Enum.GetValues(typeof(Box3DFacet)).GetValue(i)); this.facets[i].MeshGeometry.Positions[0] = rectangle.TopLeft.ToPoint3D(); this.facets[i].MeshGeometry.Positions[1] = rectangle.TopRight.ToPoint3D(); this.facets[i].MeshGeometry.Positions[2] = rectangle.BottomRight.ToPoint3D(); @@ -288,12 +288,12 @@ private void UpdateVisibility() { foreach (var line in this.edges) { - this.UpdateChildVisibility(line, this.Visible && this.EdgeVisible && this.CurrentData.HasValue); + this.UpdateChildVisibility(line, this.Visible && this.EdgeVisible && this.CurrentData != null); } foreach (var facet in this.facets) { - this.UpdateChildVisibility(facet, this.Visible && this.FacetVisible && this.CurrentData.HasValue); + this.UpdateChildVisibility(facet, this.Visible && this.FacetVisible && this.CurrentData != null); } } diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/DepthImageIntrinsicsPoseToDepthImageCameraViewAdapter.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/DepthImageIntrinsicsPoseToDepthImageCameraViewAdapter.cs new file mode 100644 index 000000000..39364c16e --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/DepthImageIntrinsicsPoseToDepthImageCameraViewAdapter.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean.Visualization +{ + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; + using Microsoft.Psi.Visualization.Adapters; + using Microsoft.Psi.Visualization.Data; + + /// + /// Implements a stream adapter from a depth image with intrinsics and pose to a depth image camera view. + /// + [StreamAdapter] + public class DepthImageIntrinsicsPoseToDepthImageCameraViewAdapter : StreamAdapter<(Shared, ICameraIntrinsics, CoordinateSystem), DepthImageCameraView> + { + /// + public override DepthImageCameraView GetAdaptedValue((Shared, ICameraIntrinsics, CoordinateSystem) source, Envelope envelope) + { + if (source.Item1 == null || source.Item1.Resource == null) + { + return default; + } + + return new DepthImageCameraView(source.Item1, source.Item2, source.Item3); + } + + /// + public override void Dispose(DepthImageCameraView destination) + => destination?.ViewedObject?.Dispose(); + } +} \ No newline at end of file diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/DepthImageIntrinsicsToDepthImageCameraViewAdapter.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/DepthImageIntrinsicsToDepthImageCameraViewAdapter.cs new file mode 100644 index 000000000..37be16496 --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/DepthImageIntrinsicsToDepthImageCameraViewAdapter.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean.Visualization +{ + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; + using Microsoft.Psi.Visualization.Adapters; + using Microsoft.Psi.Visualization.Data; + + /// + /// Implements a stream adapter from depth image and intrinsics to a depth image camera view. + /// + [StreamAdapter] + public class DepthImageIntrinsicsToDepthImageCameraViewAdapter : StreamAdapter<(Shared, ICameraIntrinsics), DepthImageCameraView> + { + /// + public override DepthImageCameraView GetAdaptedValue((Shared, ICameraIntrinsics) source, Envelope envelope) + => new (source.Item1, source.Item2, new CoordinateSystem()); + } +} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/DepthImageToSpatialDepthImageAdapter.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/DepthImageToDepthImageWithDefaultPoseAdapter.cs similarity index 70% rename from Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/DepthImageToSpatialDepthImageAdapter.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/DepthImageToDepthImageWithDefaultPoseAdapter.cs index 4345e3e7f..bff8cb636 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/DepthImageToSpatialDepthImageAdapter.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/DepthImageToDepthImageWithDefaultPoseAdapter.cs @@ -1,17 +1,18 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -namespace Microsoft.Psi.Visualization.Adapters +namespace Microsoft.Psi.Spatial.Euclidean.Visualization { using MathNet.Spatial.Euclidean; using Microsoft.Psi.Imaging; + using Microsoft.Psi.Visualization.Adapters; using Microsoft.Psi.Visualization.Data; /// /// Implements a stream adapter from depth image to spatial depth image with default position. /// [StreamAdapter] - public class DepthImageToSpatialDepthImageAdapter : StreamAdapter, (Shared, CoordinateSystem)> + public class DepthImageToDepthImageWithDefaultPoseAdapter : StreamAdapter, (Shared, CoordinateSystem)> { /// public override (Shared, CoordinateSystem) GetAdaptedValue(Shared source, Envelope envelope) diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedDepthImageIntrinsicsPoseToDepthImageCameraViewAdapter.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedDepthImageIntrinsicsPoseToDepthImageCameraViewAdapter.cs new file mode 100644 index 000000000..a69c253d5 --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedDepthImageIntrinsicsPoseToDepthImageCameraViewAdapter.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean.Visualization +{ + using System; + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; + using Microsoft.Psi.Visualization.Adapters; + using Microsoft.Psi.Visualization.Data; + + /// + /// Implements a stream adapter from an encoded depth image with intrinsics and pose to a depth image camera view. + /// + [StreamAdapter] + public class EncodedDepthImageIntrinsicsPoseToDepthImageCameraViewAdapter : StreamAdapter<(Shared, ICameraIntrinsics, CoordinateSystem), DepthImageCameraView> + { + private readonly DepthImageFromStreamDecoder depthImageDecoder = new (); + + /// + public override DepthImageCameraView GetAdaptedValue((Shared, ICameraIntrinsics, CoordinateSystem) source, Envelope envelope) + { + if (source.Item1 == null || source.Item1.Resource == null) + { + return default; + } + + var encodedDepthImage = source.Item1.Resource; + var depthImage = DepthImagePool.GetOrCreate( + encodedDepthImage.Width, + encodedDepthImage.Height, + encodedDepthImage.DepthValueSemantics, + encodedDepthImage.DepthValueToMetersScaleFactor); + depthImage.Resource.DecodeFrom(encodedDepthImage, this.depthImageDecoder); + return new DepthImageCameraView(depthImage, source.Item2, source.Item3); + } + + /// + public override void Dispose(DepthImageCameraView destination) + => destination?.ViewedObject?.Dispose(); + } +} \ No newline at end of file diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedDepthImageIntrinsicsToDepthImageCameraViewAdapter.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedDepthImageIntrinsicsToDepthImageCameraViewAdapter.cs new file mode 100644 index 000000000..c9a5631dd --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedDepthImageIntrinsicsToDepthImageCameraViewAdapter.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean.Visualization +{ + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; + using Microsoft.Psi.Visualization.Adapters; + using Microsoft.Psi.Visualization.Data; + + /// + /// Implements a stream adapter from an encoded depth image with intrinsics to a depth image camera view. + /// + [StreamAdapter] + public class EncodedDepthImageIntrinsicsToDepthImageCameraViewAdapter : StreamAdapter<(Shared, ICameraIntrinsics), DepthImageCameraView> + { + private readonly EncodedDepthImageToDepthImageAdapter depthImageAdapter = new (); + + /// + public override DepthImageCameraView GetAdaptedValue((Shared, ICameraIntrinsics) source, Envelope envelope) + => new (this.depthImageAdapter.GetAdaptedValue(source.Item1, envelope), source.Item2, new CoordinateSystem()); + + /// + public override void Dispose(DepthImageCameraView destination) + => this.depthImageAdapter.Dispose(destination.ViewedObject); + } +} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthImageToSpatialDepthCameraViewManualFocalLengthAdapter.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedDepthImageToDepthImageWithDefaultPoseAdapter.cs similarity index 67% rename from Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthImageToSpatialDepthCameraViewManualFocalLengthAdapter.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedDepthImageToDepthImageWithDefaultPoseAdapter.cs index 58d651e4f..5e32bcfce 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthImageToSpatialDepthCameraViewManualFocalLengthAdapter.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedDepthImageToDepthImageWithDefaultPoseAdapter.cs @@ -1,19 +1,20 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -namespace Microsoft.Psi.Visualization.Adapters +namespace Microsoft.Psi.Spatial.Euclidean.Visualization { using MathNet.Spatial.Euclidean; using Microsoft.Psi.Imaging; + using Microsoft.Psi.Visualization.Adapters; using Microsoft.Psi.Visualization.Data; /// - /// Implements a stream adapter from shared encoded depth image to shared depth image with default position. + /// Implements a stream adapter from an encoded depth image to a depth image with default position. /// [StreamAdapter] - public class EncodedDepthImageToSpatialDepthCameraViewManualFocalLengthAdapter : StreamAdapter, (Shared, CoordinateSystem)> + public class EncodedDepthImageToDepthImageWithDefaultPoseAdapter : StreamAdapter, (Shared, CoordinateSystem)> { - private readonly EncodedDepthImageToDepthImageAdapter encodedDepthImageAdapter = new EncodedDepthImageToDepthImageAdapter(); + private readonly EncodedDepthImageToDepthImageAdapter encodedDepthImageAdapter = new (); /// public override (Shared, CoordinateSystem) GetAdaptedValue(Shared source, Envelope envelope) diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageCameraViewToImageCameraViewAdapter.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageCameraViewToImageCameraViewAdapter.cs new file mode 100644 index 000000000..48bf3183a --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageCameraViewToImageCameraViewAdapter.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean.Visualization +{ + using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; + using Microsoft.Psi.Visualization.Adapters; + using Microsoft.Psi.Visualization.Data; + + /// + /// Implements a stream adapter from to . + /// + [StreamAdapter] + public class EncodedImageCameraViewToImageCameraViewAdapter : StreamAdapter + { + private readonly ImageFromStreamDecoder imageDecoder = new (); + + /// + public override ImageCameraView GetAdaptedValue(EncodedImageCameraView source, Envelope envelope) + { + if (source == null) + { + return default; + } + + var encodedImage = source.ViewedObject.Resource; + var image = ImagePool.GetOrCreate(encodedImage.Width, encodedImage.Height, encodedImage.PixelFormat); + image.Resource.DecodeFrom(encodedImage, this.imageDecoder); + return new ImageCameraView(image, source.CameraIntrinsics, source.CameraPose); + } + + /// + public override void Dispose(ImageCameraView destination) + => destination?.ViewedObject?.Dispose(); + } +} \ No newline at end of file diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageIntrinsicsPoseToImageCameraViewAdapter.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageIntrinsicsPoseToImageCameraViewAdapter.cs new file mode 100644 index 000000000..d6a53beca --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageIntrinsicsPoseToImageCameraViewAdapter.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean.Visualization +{ + using System; + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; + using Microsoft.Psi.Visualization.Adapters; + using Microsoft.Psi.Visualization.Data; + + /// + /// Implements a stream adapter from an encoded image with intrinsics and pose to an image camera view. + /// + [StreamAdapter] + public class EncodedImageIntrinsicsPoseToImageCameraViewAdapter : StreamAdapter<(Shared, ICameraIntrinsics, CoordinateSystem), ImageCameraView> + { + private readonly ImageFromStreamDecoder imageDecoder = new (); + + /// + public override ImageCameraView GetAdaptedValue((Shared, ICameraIntrinsics, CoordinateSystem) source, Envelope envelope) + { + if (source.Item1 == null || source.Item1.Resource == null) + { + return default; + } + + var encodedImage = source.Item1.Resource; + var image = ImagePool.GetOrCreate(encodedImage.Width, encodedImage.Height, encodedImage.PixelFormat); + image.Resource.DecodeFrom(encodedImage, this.imageDecoder); + return new ImageCameraView(image, source.Item2, source.Item3); + } + + /// + public override void Dispose(ImageCameraView destination) + => destination?.ViewedObject?.Dispose(); + } +} \ No newline at end of file diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageIntrinsicsToImageCameraViewAdapter.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageIntrinsicsToImageCameraViewAdapter.cs new file mode 100644 index 000000000..e8471ff6d --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageIntrinsicsToImageCameraViewAdapter.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean.Visualization +{ + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; + using Microsoft.Psi.Visualization.Adapters; + using Microsoft.Psi.Visualization.Data; + + /// + /// Implements a stream adapter from an encoded image with intrinsics to an image camera view. + /// + [StreamAdapter] + public class EncodedImageIntrinsicsToImageCameraViewAdapter : StreamAdapter<(Shared, ICameraIntrinsics), ImageCameraView> + { + private readonly EncodedImageToImageAdapter imageAdapter = new (); + + /// + public override ImageCameraView GetAdaptedValue((Shared, ICameraIntrinsics) source, Envelope envelope) + => new (this.imageAdapter.GetAdaptedValue(source.Item1, envelope), source.Item2, new CoordinateSystem()); + + /// + public override void Dispose(ImageCameraView destination) + => this.imageAdapter.Dispose(destination.ViewedObject); + } +} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialImageToSpatialImageAdapter.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageWithPoseToImageWithPoseAdapter.cs similarity index 67% rename from Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialImageToSpatialImageAdapter.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageWithPoseToImageWithPoseAdapter.cs index b76fb2aaa..46ef351eb 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialImageToSpatialImageAdapter.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/EncodedImageWithPoseToImageWithPoseAdapter.cs @@ -1,19 +1,20 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -namespace Microsoft.Psi.Visualization.Adapters +namespace Microsoft.Psi.Spatial.Euclidean.Visualization { using MathNet.Spatial.Euclidean; using Microsoft.Psi.Imaging; + using Microsoft.Psi.Visualization.Adapters; using Microsoft.Psi.Visualization.Data; /// - /// Implements a stream adapter from encoded spatial image (shared encoded image with position) to spatial image (shared image with position). + /// Implements a stream adapter from encoded image with position to an image with position. /// [StreamAdapter] - public class EncodedSpatialImageToSpatialImageAdapter : StreamAdapter<(Shared, CoordinateSystem), (Shared, CoordinateSystem)> + public class EncodedImageWithPoseToImageWithPoseAdapter : StreamAdapter<(Shared, CoordinateSystem), (Shared, CoordinateSystem)> { - private readonly EncodedImageToImageAdapter imageAdapter = new EncodedImageToImageAdapter(); + private readonly EncodedImageToImageAdapter imageAdapter = new (); /// public override (Shared, CoordinateSystem) GetAdaptedValue((Shared, CoordinateSystem) source, Envelope envelope) diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/ImageIntrinsicsPoseToImageCameraViewAdapter.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/ImageIntrinsicsPoseToImageCameraViewAdapter.cs new file mode 100644 index 000000000..cf33fd03e --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/ImageIntrinsicsPoseToImageCameraViewAdapter.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean.Visualization +{ + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; + using Microsoft.Psi.Visualization.Adapters; + using Microsoft.Psi.Visualization.Data; + + /// + /// Implements a stream adapter from an image with intrinsics and pose to an image camera view. + /// + [StreamAdapter] + public class ImageIntrinsicsPoseToImageCameraViewAdapter : StreamAdapter<(Shared, ICameraIntrinsics, CoordinateSystem), ImageCameraView> + { + /// + public override ImageCameraView GetAdaptedValue((Shared, ICameraIntrinsics, CoordinateSystem) source, Envelope envelope) + { + if (source.Item1 == null || source.Item1.Resource == null) + { + return default; + } + + return new ImageCameraView(source.Item1, source.Item2, source.Item3); + } + + /// + public override void Dispose(ImageCameraView destination) + => destination?.ViewedObject?.Dispose(); + } +} \ No newline at end of file diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/ImageIntrisicsToImageCameraViewAdapter.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/ImageIntrisicsToImageCameraViewAdapter.cs new file mode 100644 index 000000000..a62bd1d94 --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/ImageIntrisicsToImageCameraViewAdapter.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean.Visualization +{ + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; + using Microsoft.Psi.Visualization.Adapters; + using Microsoft.Psi.Visualization.Data; + + /// + /// Implements a stream adapter from image and camera intrinsics to an image camera view. + /// + [StreamAdapter] + public class ImageIntrisicsToImageCameraViewAdapter : StreamAdapter<(Shared, ICameraIntrinsics), ImageCameraView> + { + /// + public override ImageCameraView GetAdaptedValue((Shared, ICameraIntrinsics) source, Envelope envelope) + => new (source.Item1, source.Item2, new CoordinateSystem()); + } +} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/ImageToSpatialImageAdapter.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/ImageToImageWithDefaultPoseAdapter.cs similarity index 61% rename from Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/ImageToSpatialImageAdapter.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/ImageToImageWithDefaultPoseAdapter.cs index c1e0dcfe6..a60bb436d 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/ImageToSpatialImageAdapter.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/Adapters/ImageToImageWithDefaultPoseAdapter.cs @@ -1,17 +1,18 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -namespace Microsoft.Psi.Visualization.Adapters +namespace Microsoft.Psi.Spatial.Euclidean.Visualization { using MathNet.Spatial.Euclidean; using Microsoft.Psi.Imaging; + using Microsoft.Psi.Visualization.Adapters; using Microsoft.Psi.Visualization.Data; /// - /// Implements a stream adapter from shared image to spatial image (shared image with default position). + /// Implements a stream adapter from an image to an image with a default pose. /// [StreamAdapter] - public class ImageToSpatialImageAdapter : StreamAdapter, (Shared, CoordinateSystem)> + public class ImageToImageWithDefaultPoseAdapter : StreamAdapter, (Shared, CoordinateSystem)> { /// public override (Shared, CoordinateSystem) GetAdaptedValue(Shared source, Envelope envelope) diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialCameraVisualizationObject.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/CameraIntrinsicsWithPoseVisualizationObject.cs similarity index 72% rename from Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialCameraVisualizationObject.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/CameraIntrinsicsWithPoseVisualizationObject.cs index 2db31937b..a218944be 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialCameraVisualizationObject.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/CameraIntrinsicsWithPoseVisualizationObject.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -namespace Microsoft.Psi.Visualization.VisualizationObjects +namespace Microsoft.Psi.Spatial.Euclidean.Visualization { using System.Collections.Generic; using System.ComponentModel; @@ -11,23 +11,24 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects using MathNet.Spatial.Euclidean; using Microsoft.Psi.Calibration; using Microsoft.Psi.Visualization.Extensions; + using Microsoft.Psi.Visualization.VisualizationObjects; /// - /// Implements a visualization object for a spatial camera (camera intrinsics + position) as a frustum. + /// Implements a visualization object for camera frustrum described via camera intrinsics and position. /// - [VisualizationObject("Spatial Camera")] - public class SpatialCameraVisualizationObject : ModelVisual3DVisualizationObject<(ICameraIntrinsics, CoordinateSystem)> + [VisualizationObject("Camera Intrinsics with Pose")] + public class CameraIntrinsicsWithPoseVisualizationObject : ModelVisual3DVisualizationObject<(ICameraIntrinsics, CoordinateSystem)> { - private readonly List pyramid = new List(); + private readonly List frustum = new (); private CoordinateSystem position = null; private ICameraIntrinsics intrinsics = null; private Color color = Colors.DimGray; private double imagePlaneDistanceCm = 100; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - public SpatialCameraVisualizationObject() + public CameraIntrinsicsWithPoseVisualizationObject() { // Create frustum lines for (int i = 0; i < 8; i++) @@ -42,7 +43,7 @@ public SpatialCameraVisualizationObject() linesVisual3D.Points.Add(default); } - this.pyramid.Add(linesVisual3D); + this.frustum.Add(linesVisual3D); } } @@ -119,35 +120,35 @@ private void UpdateVisuals() var bottomRightPoint3D = (principalPoint + imageRightAxis.ScaleBy(rightWidth) - imageUpAxis.ScaleBy(bottomHeight)).ToPoint3D(); var bottomLeftPoint3D = (principalPoint - imageRightAxis.ScaleBy(leftWidth) - imageUpAxis.ScaleBy(bottomHeight)).ToPoint3D(); - this.pyramid[0].Points[0] = topLeftPoint3D; - this.pyramid[0].Points[1] = topRightPoint3D; + this.frustum[0].Points[0] = topLeftPoint3D; + this.frustum[0].Points[1] = topRightPoint3D; - this.pyramid[1].Points[0] = topRightPoint3D; - this.pyramid[1].Points[1] = bottomRightPoint3D; + this.frustum[1].Points[0] = topRightPoint3D; + this.frustum[1].Points[1] = bottomRightPoint3D; - this.pyramid[2].Points[0] = bottomRightPoint3D; - this.pyramid[2].Points[1] = bottomLeftPoint3D; + this.frustum[2].Points[0] = bottomRightPoint3D; + this.frustum[2].Points[1] = bottomLeftPoint3D; - this.pyramid[3].Points[0] = bottomLeftPoint3D; - this.pyramid[3].Points[1] = topLeftPoint3D; + this.frustum[3].Points[0] = bottomLeftPoint3D; + this.frustum[3].Points[1] = topLeftPoint3D; - this.pyramid[4].Points[0] = cameraPoint3D; - this.pyramid[4].Points[1] = topLeftPoint3D; + this.frustum[4].Points[0] = cameraPoint3D; + this.frustum[4].Points[1] = topLeftPoint3D; - this.pyramid[5].Points[0] = cameraPoint3D; - this.pyramid[5].Points[1] = topRightPoint3D; + this.frustum[5].Points[0] = cameraPoint3D; + this.frustum[5].Points[1] = topRightPoint3D; - this.pyramid[6].Points[0] = cameraPoint3D; - this.pyramid[6].Points[1] = bottomLeftPoint3D; + this.frustum[6].Points[0] = cameraPoint3D; + this.frustum[6].Points[1] = bottomLeftPoint3D; - this.pyramid[7].Points[0] = cameraPoint3D; - this.pyramid[7].Points[1] = bottomRightPoint3D; + this.frustum[7].Points[0] = cameraPoint3D; + this.frustum[7].Points[1] = bottomRightPoint3D; } } private void UpdateVisibility() { - foreach (var edge in this.pyramid) + foreach (var edge in this.frustum) { this.UpdateChildVisibility(edge, this.Visible && this.CurrentData != default && this.intrinsics != null && this.position != null); } @@ -155,7 +156,7 @@ private void UpdateVisibility() private void UpdateColor() { - foreach (var edge in this.pyramid) + foreach (var edge in this.frustum) { edge.Color = this.color; } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsMeshVisualizationObject.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageCameraViewAsMeshVisualizationObject.cs similarity index 82% rename from Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsMeshVisualizationObject.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageCameraViewAsMeshVisualizationObject.cs index 70f9d3c86..8217efc08 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsMeshVisualizationObject.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageCameraViewAsMeshVisualizationObject.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -namespace Microsoft.Psi.Visualization.VisualizationObjects +namespace Microsoft.Psi.Spatial.Euclidean.Visualization { using System; using System.Collections.Generic; @@ -13,17 +13,19 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects using MathNet.Spatial.Euclidean; using Microsoft.Psi.Calibration; using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; using Microsoft.Psi.Visualization.Extensions; + using Microsoft.Psi.Visualization.VisualizationObjects; using Xceed.Wpf.Toolkit.PropertyGrid.Attributes; /// - /// Implements a visualization object for a spatial depth camera view as a 3D mesh, from a depth image, camera intrinsics, and position. + /// Implements a visualization object for a . /// - [VisualizationObject("3D Depth Camera View as Mesh")] - public class SpatialDepthCameraViewAsMeshVisualizationObject : ModelVisual3DVisualizationObject<(Shared, ICameraIntrinsics, CoordinateSystem)> + [VisualizationObject("Depth Camera View (Mesh)")] + public class DepthImageCameraViewAsMeshVisualizationObject : ModelVisual3DVisualizationObject { - private SpatialCameraVisualizationObject spatialCamera; - private ModelVisual3D depthImageMesh; + private readonly ModelVisual3D depthImageMesh; + private CameraIntrinsicsWithPoseVisualizationObject frustum; private Shared depthImage = null; private ICameraIntrinsics intrinsics = null; @@ -35,25 +37,25 @@ public class SpatialDepthCameraViewAsMeshVisualizationObject : ModelVisual3DVisu private int meshTransparency = 50; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - public SpatialDepthCameraViewAsMeshVisualizationObject() + public DepthImageCameraViewAsMeshVisualizationObject() { this.depthImageMesh = new ModelVisual3D(); - this.spatialCamera = new SpatialCameraVisualizationObject(); + this.frustum = new CameraIntrinsicsWithPoseVisualizationObject(); } /// - /// Gets or sets the visualization object for the camera intrinsics as a frustum. + /// Gets or sets the visualization object for the frustum. /// [ExpandableObject] [DataMember] [DisplayName("Frustum")] - [Description("The frustum representing the spatial depth camera.")] - public SpatialCameraVisualizationObject SpatialCamera + [Description("The frustum representing the depth camera.")] + public CameraIntrinsicsWithPoseVisualizationObject Frustum { - get { return this.spatialCamera; } - set { this.Set(nameof(this.SpatialCamera), ref this.spatialCamera, value); } + get { return this.frustum; } + set { this.Set(nameof(this.Frustum), ref this.frustum, value); } } /// @@ -81,7 +83,7 @@ public int MeshTransparency } /// - protected override Action<(Shared, ICameraIntrinsics, CoordinateSystem)> Deallocator => data => data.Item1?.Dispose(); + protected override Action Deallocator => data => data.ViewedObject?.Dispose(); /// public override void UpdateData() @@ -91,9 +93,9 @@ public override void UpdateData() this.depthImage.Dispose(); } - this.depthImage = this.CurrentData.Item1?.AddRef(); - this.intrinsics = this.CurrentData.Item2; - this.position = this.CurrentData.Item3; + this.depthImage = this.CurrentData?.ViewedObject?.AddRef(); + this.intrinsics = this.CurrentData?.CameraIntrinsics; + this.position = this.CurrentData?.CameraPose; this.UpdateVisuals(); this.UpdateVisibility(); @@ -118,7 +120,7 @@ public override void NotifyPropertyChanged(string propertyName) private void UpdateVisuals() { - this.spatialCamera.SetCurrentValue(this.SynthesizeMessage((this.intrinsics, this.position))); + this.frustum.SetCurrentValue(this.SynthesizeMessage((this.intrinsics, this.position))); this.UpdateDepthFramePoints(); @@ -135,7 +137,7 @@ private void UpdateVisuals() private void UpdateVisibility() { this.UpdateChildVisibility(this.depthImageMesh, this.Visible && this.CurrentData != default && this.depthImage != null && this.depthImage.Resource != null && this.intrinsics != null && this.position != null); - this.UpdateChildVisibility(this.spatialCamera.ModelView, this.Visible && this.CurrentData != default && this.intrinsics != null && this.position != null); + this.UpdateChildVisibility(this.frustum.ModelView, this.Visible && this.CurrentData != default && this.intrinsics != null && this.position != null); } private void UpdateMaterial() @@ -232,7 +234,7 @@ private void UpdateDepthFramePoints() int cx = width / 2; int cy = height / 2; - double scale = 0.001; + double scale = this.depthImage.Resource.DepthValueToMetersScaleFactor; unsafe { @@ -252,7 +254,7 @@ private void UpdateDepthFramePoints() } else { - var other = this.intrinsics.GetCameraSpacePosition(new Point2D(ix, iy), this.rawDepth[i] * scale, true); + var other = this.intrinsics.GetCameraSpacePosition(new Point2D(ix, iy), this.rawDepth[i] * scale, this.depthImage.Resource.DepthValueSemantics, true); this.depthFramePoints[i] = other.TransformBy(this.position).ToPoint3D(); } } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsPointCloudVisualizationObject.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageCameraViewAsPointCloudVisualizationObject.cs similarity index 76% rename from Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsPointCloudVisualizationObject.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageCameraViewAsPointCloudVisualizationObject.cs index 8f83d3bac..9d4326726 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsPointCloudVisualizationObject.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageCameraViewAsPointCloudVisualizationObject.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -namespace Microsoft.Psi.Visualization.VisualizationObjects +namespace Microsoft.Psi.Spatial.Euclidean.Visualization { using System; using System.Collections.Generic; @@ -12,17 +12,19 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects using MathNet.Spatial.Euclidean; using Microsoft.Psi.Calibration; using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; + using Microsoft.Psi.Visualization.VisualizationObjects; using Xceed.Wpf.Toolkit.PropertyGrid.Attributes; using Windows = System.Windows.Media.Media3D; /// - /// Implements a visualization object for a spatial depth camera view as a point cloud, from a depth image, camera intrinsics, and position. + /// Implements a visualization object for as a point cloud. /// - [VisualizationObject("3D Depth Camera View as Point Cloud")] - public class SpatialDepthCameraViewAsPointCloudVisualizationObject : ModelVisual3DVisualizationObject<(Shared, ICameraIntrinsics, CoordinateSystem)> + [VisualizationObject("Depth Camera View (Point Cloud)")] + public class DepthImageCameraViewAsPointCloudVisualizationObject : ModelVisual3DVisualizationObject { private Point3DListAsPointCloudVisualizationObject pointCloud; - private SpatialCameraVisualizationObject spatialCamera; + private CameraIntrinsicsWithPoseVisualizationObject frustum; private int sparsity = 3; private Shared depthImage = null; @@ -31,25 +33,25 @@ public class SpatialDepthCameraViewAsPointCloudVisualizationObject : ModelVisual private CoordinateSystem position = null; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - public SpatialDepthCameraViewAsPointCloudVisualizationObject() + public DepthImageCameraViewAsPointCloudVisualizationObject() { this.PointCloud = new Point3DListAsPointCloudVisualizationObject(); - this.SpatialCamera = new SpatialCameraVisualizationObject(); + this.SpatialCamera = new CameraIntrinsicsWithPoseVisualizationObject(); } /// - /// Gets or sets the visualization object for the camera intrinsics as a frustum. + /// Gets or sets the visualization object for the frustum. /// [ExpandableObject] [DataMember] [DisplayName("Frustum")] [Description("The frustum representing the spatial depth camera.")] - public SpatialCameraVisualizationObject SpatialCamera + public CameraIntrinsicsWithPoseVisualizationObject SpatialCamera { - get { return this.spatialCamera; } - set { this.Set(nameof(this.SpatialCamera), ref this.spatialCamera, value); } + get { return this.frustum; } + set { this.Set(nameof(this.SpatialCamera), ref this.frustum, value); } } /// @@ -78,7 +80,7 @@ public int Sparsity } /// - protected override Action<(Shared, ICameraIntrinsics, CoordinateSystem)> Deallocator => data => data.Item1?.Dispose(); + protected override Action Deallocator => data => data.ViewedObject?.Dispose(); /// public override void UpdateData() @@ -88,14 +90,14 @@ public override void UpdateData() this.depthImage.Dispose(); } - this.depthImage = this.CurrentData.Item1?.AddRef(); - if (!Equals(this.intrinsics, this.CurrentData.Item2)) + this.depthImage = this.CurrentData?.ViewedObject?.AddRef(); + if (!Equals(this.intrinsics, this.CurrentData?.CameraIntrinsics)) { - this.intrinsics = this.CurrentData.Item2; - this.cameraSpaceMapping = this.intrinsics?.GetPixelToCameraSpaceMapping(true); + this.intrinsics = this.CurrentData?.CameraIntrinsics; + this.cameraSpaceMapping = this.intrinsics?.GetPixelToCameraSpaceMapping(this.depthImage.Resource.DepthValueSemantics, true); } - this.position = this.CurrentData.Item3; + this.position = this.CurrentData?.CameraPose; this.UpdateVisuals(); this.UpdateVisibility(); @@ -116,7 +118,7 @@ public override void NotifyPropertyChanged(string propertyName) private void UpdateVisuals() { - this.spatialCamera.SetCurrentValue(this.SynthesizeMessage((this.intrinsics, this.position))); + this.frustum.SetCurrentValue(this.SynthesizeMessage((this.intrinsics, this.position))); this.UpdateDepthImagePointCloud(); } @@ -133,7 +135,7 @@ private void UpdateDepthImagePointCloud() var pointsArray = new Windows.Point3D[width, height]; var points = new List(); - double scale = 0.001; + double scale = this.depthImage.Resource.DepthValueToMetersScaleFactor; unsafe { ushort* depthFrame = (ushort*)((byte*)this.depthImage.Resource.ImageData.ToPointer()); @@ -180,7 +182,7 @@ private void UpdateDepthImagePointCloud() private void UpdateVisibility() { this.UpdateChildVisibility(this.pointCloud.ModelView, this.Visible && this.CurrentData != default && this.depthImage != null && this.depthImage.Resource != null && this.intrinsics != null && this.position != null); - this.UpdateChildVisibility(this.spatialCamera.ModelView, this.Visible && this.CurrentData != default && this.intrinsics != null && this.position != null); + this.UpdateChildVisibility(this.frustum.ModelView, this.Visible && this.CurrentData != default && this.intrinsics != null && this.position != null); } } } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsMeshManualFocalLengthVisualizationObject.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageWithPoseAndManualFocalLengthAsMeshVisualizationObject.cs similarity index 67% rename from Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsMeshManualFocalLengthVisualizationObject.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageWithPoseAndManualFocalLengthAsMeshVisualizationObject.cs index a9fbb7b0d..9e39bbca6 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsMeshManualFocalLengthVisualizationObject.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageWithPoseAndManualFocalLengthAsMeshVisualizationObject.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -namespace Microsoft.Psi.Visualization.VisualizationObjects +namespace Microsoft.Psi.Spatial.Euclidean.Visualization { using System; using System.ComponentModel; @@ -9,16 +9,18 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects using MathNet.Spatial.Euclidean; using Microsoft.Psi.Calibration; using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; + using Microsoft.Psi.Visualization.VisualizationObjects; using Xceed.Wpf.Toolkit.PropertyGrid.Attributes; /// - /// Implements a visualization object for a spatial depth camera view as a mesh, from a depth image and position. + /// Implements a visualization object for a depth camera view as a mesh, from a depth image and position. /// Camera intrinsics are determined from manually set focal length properties. /// - [VisualizationObject("3D Depth Camera View as Mesh")] - public class SpatialDepthCameraViewAsMeshManualFocalLengthVisualizationObject : ModelVisual3DVisualizationObject<(Shared, CoordinateSystem)> + [VisualizationObject("Depth Camera View (Mesh) with Manual Focal Length")] + public class DepthImageWithPoseAndManualFocalLengthAsMeshVisualizationObject : ModelVisual3DVisualizationObject<(Shared, CoordinateSystem)> { - private SpatialDepthCameraViewAsMeshVisualizationObject depthCameraViewAsMesh; + private DepthImageCameraViewAsMeshVisualizationObject depthImageCameraViewAsMesh; private Shared depthImage = null; private ICameraIntrinsics intrinsics = null; @@ -27,11 +29,11 @@ public class SpatialDepthCameraViewAsMeshManualFocalLengthVisualizationObject : private double focalLengthY = 500; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - public SpatialDepthCameraViewAsMeshManualFocalLengthVisualizationObject() + public DepthImageWithPoseAndManualFocalLengthAsMeshVisualizationObject() { - this.depthCameraViewAsMesh = new SpatialDepthCameraViewAsMeshVisualizationObject(); + this.depthImageCameraViewAsMesh = new DepthImageCameraViewAsMeshVisualizationObject(); } /// @@ -39,12 +41,12 @@ public SpatialDepthCameraViewAsMeshManualFocalLengthVisualizationObject() /// [ExpandableObject] [DataMember] - [DisplayName("Spatial Depth Camera View")] - [Description("The spatial depth camera view, including mesh and frustum.")] - public SpatialDepthCameraViewAsMeshVisualizationObject DepthCameraViewAsMesh + [DisplayName("Depth Camera View")] + [Description("The depth camera view, including mesh and frustum.")] + public DepthImageCameraViewAsMeshVisualizationObject DepthCameraViewAsMesh { - get { return this.depthCameraViewAsMesh; } - set { this.Set(nameof(this.DepthCameraViewAsMesh), ref this.depthCameraViewAsMesh, value); } + get { return this.depthImageCameraViewAsMesh; } + set { this.Set(nameof(this.DepthCameraViewAsMesh), ref this.depthImageCameraViewAsMesh, value); } } /// @@ -105,12 +107,12 @@ public override void NotifyPropertyChanged(string propertyName) private void UpdateVisuals() { this.intrinsics = this.depthImage?.Resource?.CreateCameraIntrinsics(this.FocalLengthX, this.FocalLengthY); - this.depthCameraViewAsMesh.SetCurrentValue(this.SynthesizeMessage((this.depthImage, this.intrinsics, this.position))); + this.depthImageCameraViewAsMesh.SetCurrentValue(this.SynthesizeMessage(new DepthImageCameraView(this.depthImage, this.intrinsics, this.position))); } private void UpdateVisibility() { - this.UpdateChildVisibility(this.depthCameraViewAsMesh.ModelView, this.Visible && this.CurrentData != default); + this.UpdateChildVisibility(this.depthImageCameraViewAsMesh.ModelView, this.Visible && this.CurrentData != default); } } } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsPointCloudManualFocalLengthVisualizationObject.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageWithPoseAndManualFocalLengthAsPointCloudVisualizationObject.cs similarity index 66% rename from Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsPointCloudManualFocalLengthVisualizationObject.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageWithPoseAndManualFocalLengthAsPointCloudVisualizationObject.cs index 5b435d708..f73c0e692 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialDepthCameraViewAsPointCloudManualFocalLengthVisualizationObject.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/DepthImageWithPoseAndManualFocalLengthAsPointCloudVisualizationObject.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -namespace Microsoft.Psi.Visualization.VisualizationObjects +namespace Microsoft.Psi.Spatial.Euclidean.Visualization { using System; using System.ComponentModel; @@ -9,16 +9,18 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects using MathNet.Spatial.Euclidean; using Microsoft.Psi.Calibration; using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; + using Microsoft.Psi.Visualization.VisualizationObjects; using Xceed.Wpf.Toolkit.PropertyGrid.Attributes; /// - /// Implements a visualization object for a spatial depth camera view as a point cloud, from a depth image and position. + /// Implements a visualization object for a depth image as a point cloud, from a depth image and position. /// Camera intrinsics are determined from manually set focal length properties. /// - [VisualizationObject("3D Depth Camera View as Point Cloud")] - public class SpatialDepthCameraViewAsPointCloudManualFocalLengthVisualizationObject : ModelVisual3DVisualizationObject<(Shared, CoordinateSystem)> + [VisualizationObject("Depth Camera View (Point Cloud) with Manual Focal Length")] + public class DepthImageWithPoseAndManualFocalLengthAsPointCloudVisualizationObject : ModelVisual3DVisualizationObject<(Shared, CoordinateSystem)> { - private SpatialDepthCameraViewAsPointCloudVisualizationObject depthCameraViewAsPointCloud; + private DepthImageCameraViewAsPointCloudVisualizationObject depthImageCameraViewAsPointCloud; private Shared depthImage = null; private ICameraIntrinsics intrinsics = null; @@ -27,11 +29,11 @@ public class SpatialDepthCameraViewAsPointCloudManualFocalLengthVisualizationObj private double focalLengthY = 500; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - public SpatialDepthCameraViewAsPointCloudManualFocalLengthVisualizationObject() + public DepthImageWithPoseAndManualFocalLengthAsPointCloudVisualizationObject() { - this.depthCameraViewAsPointCloud = new SpatialDepthCameraViewAsPointCloudVisualizationObject(); + this.depthImageCameraViewAsPointCloud = new DepthImageCameraViewAsPointCloudVisualizationObject(); } /// @@ -39,12 +41,12 @@ public SpatialDepthCameraViewAsPointCloudManualFocalLengthVisualizationObject() /// [ExpandableObject] [DataMember] - [DisplayName("Spatial Depth Camera View")] - [Description("The spatial depth camera view, including point cloud and frustum.")] - public SpatialDepthCameraViewAsPointCloudVisualizationObject DepthCameraViewAsPointCloud + [DisplayName("Depth Camera View")] + [Description("The depth camera view, including point cloud and frustum.")] + public DepthImageCameraViewAsPointCloudVisualizationObject DepthCameraViewAsPointCloud { - get { return this.depthCameraViewAsPointCloud; } - set { this.Set(nameof(this.DepthCameraViewAsPointCloud), ref this.depthCameraViewAsPointCloud, value); } + get { return this.depthImageCameraViewAsPointCloud; } + set { this.Set(nameof(this.DepthCameraViewAsPointCloud), ref this.depthImageCameraViewAsPointCloud, value); } } /// @@ -105,12 +107,12 @@ public override void NotifyPropertyChanged(string propertyName) private void UpdateVisuals() { this.intrinsics = this.depthImage?.Resource?.CreateCameraIntrinsics(this.FocalLengthX, this.FocalLengthY); - this.depthCameraViewAsPointCloud.SetCurrentValue(this.SynthesizeMessage((this.depthImage, this.intrinsics, this.position))); + this.depthImageCameraViewAsPointCloud.SetCurrentValue(this.SynthesizeMessage(new DepthImageCameraView(this.depthImage, this.intrinsics, this.position))); } private void UpdateVisibility() { - this.UpdateChildVisibility(this.depthCameraViewAsPointCloud.ModelView, this.Visible && this.CurrentData != default); + this.UpdateChildVisibility(this.depthImageCameraViewAsPointCloud.ModelView, this.Visible && this.CurrentData != default); } } } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialCameraViewVisualizationObject.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/ImageCameraViewVisualizationObject.cs similarity index 74% rename from Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialCameraViewVisualizationObject.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/ImageCameraViewVisualizationObject.cs index cb3b734f4..9aeb06585 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialCameraViewVisualizationObject.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/ImageCameraViewVisualizationObject.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -namespace Microsoft.Psi.Visualization.VisualizationObjects +namespace Microsoft.Psi.Spatial.Euclidean.Visualization { using System.ComponentModel; using System.Runtime.Serialization; @@ -10,35 +10,37 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects using MathNet.Spatial.Euclidean; using Microsoft.Psi.Calibration; using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; using Microsoft.Psi.Visualization; using Microsoft.Psi.Visualization.Extensions; + using Microsoft.Psi.Visualization.VisualizationObjects; using Xceed.Wpf.Toolkit.PropertyGrid.Attributes; using Win3D = System.Windows.Media.Media3D; /// - /// Implements a visualization object for a spatial camera view, from an image, camera intrinsics, and spatial position. + /// Implements a visualization object for a image view, from an . /// - [VisualizationObject("3D Camera View")] - public class SpatialCameraViewVisualizationObject : ModelVisual3DVisualizationObject<(Shared, ICameraIntrinsics, CoordinateSystem)> + [VisualizationObject("Image Camera View")] + public class ImageCameraViewVisualizationObject : ModelVisual3DVisualizationObject { private readonly MeshGeometryVisual3D imageModelVisual; private readonly DisplayImage displayImage; - private SpatialCameraVisualizationObject spatialCamera; + private CameraIntrinsicsWithPoseVisualizationObject frustum; private Shared image = null; - private CoordinateSystem position = null; + private CoordinateSystem pose = null; private ICameraIntrinsics intrinsics = null; private int imageOpacity = 50; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - public SpatialCameraViewVisualizationObject() + public ImageCameraViewVisualizationObject() { // Instantiate the child visualizer for visualizing the camera frustum, // and register for its property changed notifications. - this.spatialCamera = new SpatialCameraVisualizationObject(); - this.spatialCamera.RegisterChildPropertyChangedNotifications(this, nameof(this.SpatialCamera)); + this.frustum = new CameraIntrinsicsWithPoseVisualizationObject(); + this.frustum.RegisterChildPropertyChangedNotifications(this, nameof(this.Frustum)); // Create a rectangle mesh for the image this.displayImage = new DisplayImage(); @@ -74,16 +76,16 @@ public SpatialCameraViewVisualizationObject() } /// - /// Gets or sets the visualization object for the camera intrinsics as a frustum. + /// Gets or sets the visualization object for the frustum. /// [ExpandableObject] [DataMember] - [DisplayName("Spatial Camera")] - [Description("The frustum representing the spatial camera.")] - public SpatialCameraVisualizationObject SpatialCamera + [DisplayName("Frustum")] + [Description("The frustum representing the camera.")] + public CameraIntrinsicsWithPoseVisualizationObject Frustum { - get { return this.spatialCamera; } - set { this.Set(nameof(this.SpatialCamera), ref this.spatialCamera, value); } + get { return this.frustum; } + set { this.Set(nameof(this.Frustum), ref this.frustum, value); } } /// @@ -98,20 +100,18 @@ public int ImageOpacity set { this.Set(nameof(this.ImageOpacity), ref this.imageOpacity, value); } } - /// - protected override System.Action<(Shared, ICameraIntrinsics, CoordinateSystem)> Deallocator => data => data.Item1?.Dispose(); - /// public override void UpdateData() { if (this.image != null) { this.image.Dispose(); + this.image = null; } - this.image = this.CurrentData.Item1?.AddRef(); - this.intrinsics = this.CurrentData.Item2; - this.position = this.CurrentData.Item3; + this.image = this.CurrentData?.ViewedObject?.AddRef(); + this.intrinsics = this.CurrentData?.CameraIntrinsics; + this.pose = this.CurrentData?.CameraPose; this.UpdateVisuals(); this.UpdateVisibility(); @@ -133,7 +133,7 @@ public override void NotifyPropertyChanged(string propertyName) /// public override void OnChildPropertyChanged(string path, object value) { - if (path == nameof(this.SpatialCamera) + "." + nameof(this.SpatialCamera.ImagePlaneDistanceCm)) + if (path == nameof(this.Frustum) + "." + nameof(this.Frustum.ImagePlaneDistanceCm)) { this.UpdateVisualPosition(); } @@ -143,7 +143,7 @@ public override void OnChildPropertyChanged(string path, object value) private void UpdateVisuals() { - this.spatialCamera.SetCurrentValue(this.SynthesizeMessage((this.intrinsics, this.position))); + this.frustum.SetCurrentValue(this.SynthesizeMessage((this.intrinsics, this.pose))); this.UpdateImageContents(); this.UpdateVisualPosition(); } @@ -164,19 +164,19 @@ private void UpdateImageContents() private void UpdateVisualPosition() { - if (this.intrinsics != null && this.position != null) + if (this.intrinsics != null && this.pose != null) { - var focalDistance = this.spatialCamera.ImagePlaneDistanceCm * 0.01; + var focalDistance = this.frustum.ImagePlaneDistanceCm * 0.01; var leftWidth = focalDistance * this.intrinsics.PrincipalPoint.X / this.intrinsics.FocalLength; var rightWidth = focalDistance * (this.intrinsics.ImageWidth - this.intrinsics.PrincipalPoint.X) / this.intrinsics.FocalLength; var topHeight = focalDistance * this.intrinsics.PrincipalPoint.Y / this.intrinsics.FocalLength; var bottomHeight = focalDistance * (this.intrinsics.ImageHeight - this.intrinsics.PrincipalPoint.Y) / this.intrinsics.FocalLength; - var pointingAxis = this.position.XAxis; - var imageRightAxis = this.position.YAxis.Negate(); - var imageUpAxis = this.position.ZAxis; + var pointingAxis = this.pose.XAxis; + var imageRightAxis = this.pose.YAxis.Negate(); + var imageUpAxis = this.pose.ZAxis; - var principalPoint = this.position.Origin + pointingAxis.ScaleBy(focalDistance); + var principalPoint = this.pose.Origin + pointingAxis.ScaleBy(focalDistance); var topLeftPoint3D = (principalPoint - imageRightAxis.ScaleBy(leftWidth) + imageUpAxis.ScaleBy(topHeight)).ToPoint3D(); var topRightPoint3D = (principalPoint + imageRightAxis.ScaleBy(rightWidth) + imageUpAxis.ScaleBy(topHeight)).ToPoint3D(); @@ -192,9 +192,9 @@ private void UpdateVisualPosition() private void UpdateVisibility() { - var visible = this.Visible && this.CurrentData != default && this.image != null && this.image.Resource != null && this.intrinsics != null && this.position != null; + var visible = this.Visible && this.CurrentData != default && this.image != null && this.image.Resource != null && this.intrinsics != null && this.pose != null; this.UpdateChildVisibility(this.imageModelVisual, visible); - this.UpdateChildVisibility(this.spatialCamera.ModelView, visible); + this.UpdateChildVisibility(this.frustum.ModelView, visible); } } } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialCameraViewManualFocalLengthVisualizationObject.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/ImageWithPoseAndManualFocalLengthVisualizationObject.cs similarity index 68% rename from Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialCameraViewManualFocalLengthVisualizationObject.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/ImageWithPoseAndManualFocalLengthVisualizationObject.cs index aaf63bf78..10c77e8c9 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/SpatialCameraViewManualFocalLengthVisualizationObject.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/CameraViews/ImageWithPoseAndManualFocalLengthVisualizationObject.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. -namespace Microsoft.Psi.Visualization.VisualizationObjects +namespace Microsoft.Psi.Spatial.Euclidean.Visualization { using System; using System.ComponentModel; @@ -9,16 +9,18 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects using MathNet.Spatial.Euclidean; using Microsoft.Psi.Calibration; using Microsoft.Psi.Imaging; + using Microsoft.Psi.Spatial.Euclidean; + using Microsoft.Psi.Visualization.VisualizationObjects; using Xceed.Wpf.Toolkit.PropertyGrid.Attributes; /// - /// Implements a visualization object for a spatial camera view, from an image and spatial position. + /// Implements a visualization object for an image camera view, from an image and spatial position. /// Camera intrinsics are determined from manually set focal length properties. /// - [VisualizationObject("3D Camera View")] - public class SpatialCameraViewManualFocalLengthVisualizationObject : ModelVisual3DVisualizationObject<(Shared, CoordinateSystem)> + [VisualizationObject("Image Camera View with Manual Focal Length")] + public class ImageWithPoseAndManualFocalLengthVisualizationObject : ModelVisual3DVisualizationObject<(Shared, CoordinateSystem)> { - private SpatialCameraViewVisualizationObject cameraView; + private ImageCameraViewVisualizationObject imageCameraView; private Shared image = null; private ICameraIntrinsics intrinsics = null; @@ -27,11 +29,11 @@ public class SpatialCameraViewManualFocalLengthVisualizationObject : ModelVisual private double focalLengthY = 500; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - public SpatialCameraViewManualFocalLengthVisualizationObject() + public ImageWithPoseAndManualFocalLengthVisualizationObject() { - this.cameraView = new SpatialCameraViewVisualizationObject(); + this.imageCameraView = new ImageCameraViewVisualizationObject(); } /// @@ -39,12 +41,12 @@ public SpatialCameraViewManualFocalLengthVisualizationObject() /// [ExpandableObject] [DataMember] - [DisplayName("Spatial Camera View")] - [Description("The spatial camera view, including image plane and frustum.")] - public SpatialCameraViewVisualizationObject CameraView + [DisplayName("Image Camera View")] + [Description("The image camera view, including image plane and frustum.")] + public ImageCameraViewVisualizationObject ImageCameraView { - get { return this.cameraView; } - set { this.Set(nameof(this.CameraView), ref this.cameraView, value); } + get { return this.imageCameraView; } + set { this.Set(nameof(this.ImageCameraView), ref this.imageCameraView, value); } } /// @@ -105,12 +107,12 @@ public override void NotifyPropertyChanged(string propertyName) private void UpdateVisuals() { this.intrinsics = this.image?.Resource?.CreateCameraIntrinsics(this.FocalLengthX, this.FocalLengthY); - this.cameraView.SetCurrentValue(this.SynthesizeMessage((this.image, this.intrinsics, this.position))); + this.imageCameraView.SetCurrentValue(this.SynthesizeMessage(new ImageCameraView(this.image, this.intrinsics, this.position))); } private void UpdateVisibility() { - this.UpdateChildVisibility(this.cameraView.ModelView, this.Visible && this.CurrentData != default); + this.UpdateChildVisibility(this.imageCameraView.ModelView, this.Visible && this.CurrentData != default); } } } diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/Adapters.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/Adapters.cs new file mode 100644 index 000000000..7751fca61 --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/Adapters.cs @@ -0,0 +1,169 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#pragma warning disable SA1649 // File name should match first type name +#pragma warning disable SA1402 // File may only contain a single type + +namespace Microsoft.Psi.Spatial.Euclidean.Visualization +{ + using System.Collections.Generic; + using Microsoft.Psi.Spatial.Euclidean; + using Microsoft.Psi.Visualization.Adapters; + using Microsoft.Psi.Visualization.Data; + + /// + /// Adapter for encoded image rectangles. + /// + [StreamAdapter] + public class EncodedImageRectangle3DAdapter : StreamAdapter + { + private readonly EncodedImageToImageAdapter imageAdapter = new (); + + /// + public override ImageRectangle3D GetAdaptedValue(EncodedImageRectangle3D source, Envelope envelope) + { + if (source != null) + { + var encodedImage = this.imageAdapter.GetAdaptedValue(source.Image, envelope); + if (encodedImage != null) + { + return new ImageRectangle3D(source.Rectangle3D, encodedImage); + } + } + + return null; + } + + /// + public override void Dispose(ImageRectangle3D destination) + { + if (destination != null) + { + this.imageAdapter.Dispose(destination.Image); + } + } + } + + /// + /// Adapter for encoded depth image rectangles. + /// + [StreamAdapter] + public class EncodedDepthImageRectangle3DAdapter : StreamAdapter + { + private readonly EncodedDepthImageToDepthImageAdapter imageAdapter = new (); + + /// + public override DepthImageRectangle3D GetAdaptedValue(EncodedDepthImageRectangle3D source, Envelope envelope) + { + if (source != null) + { + var encodedDepthImage = this.imageAdapter.GetAdaptedValue(source.Image, envelope); + if (encodedDepthImage != null) + { + return new DepthImageRectangle3D(source.Rectangle3D, encodedDepthImage); + } + } + + return null; + } + + /// + public override void Dispose(DepthImageRectangle3D destination) + { + if (destination != null) + { + this.imageAdapter.Dispose(destination.Image); + } + } + } + + /// + /// Adapter for list of to list of . + /// + [StreamAdapter] + public class EncodedImageRectangle3DListAdapter : StreamAdapter, List> + { + private readonly EncodedImageToImageAdapter imageAdapter = new (); + + /// + public override List GetAdaptedValue(List source, Envelope envelope) + { + if (source != null) + { + List outputList = new (); + foreach (var inputRectangle in source) + { + if (inputRectangle != null) + { + var encodedImage = this.imageAdapter.GetAdaptedValue(inputRectangle.Image, envelope); + if (encodedImage != null) + { + outputList.Add(new ImageRectangle3D(inputRectangle.Rectangle3D, encodedImage)); + } + } + } + + return outputList; + } + + return null; + } + + /// + public override void Dispose(List destination) + { + foreach (var imageRectangle3D in destination) + { + if (imageRectangle3D != null) + { + this.imageAdapter.Dispose(imageRectangle3D.Image); + } + } + } + } + + /// + /// Adapter for list of to list of . + /// + [StreamAdapter] + public class EncodedDepthImageRectangle3DListAdapter : StreamAdapter, List> + { + private readonly EncodedDepthImageToDepthImageAdapter imageAdapter = new (); + + /// + public override List GetAdaptedValue(List source, Envelope envelope) + { + if (source != null) + { + List outputList = new (); + foreach (var inputRectangle in source) + { + if (inputRectangle != null) + { + var encodedDepthImage = this.imageAdapter.GetAdaptedValue(inputRectangle.Image, envelope); + if (encodedDepthImage != null) + { + outputList.Add(new DepthImageRectangle3D(inputRectangle.Rectangle3D, encodedDepthImage)); + } + } + } + + return outputList; + } + + return null; + } + + /// + public override void Dispose(List destination) + { + foreach (var depthImageRectangle3D in destination) + { + if (depthImageRectangle3D != null) + { + this.imageAdapter.Dispose(depthImageRectangle3D.Image); + } + } + } + } +} \ No newline at end of file diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/DepthImageRectangle3DListVisualizationObject.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/DepthImageRectangle3DListVisualizationObject.cs similarity index 71% rename from Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/DepthImageRectangle3DListVisualizationObject.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/DepthImageRectangle3DListVisualizationObject.cs index 4ac64a7a3..725197fe9 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/DepthImageRectangle3DListVisualizationObject.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/DepthImageRectangle3DListVisualizationObject.cs @@ -3,13 +3,14 @@ namespace Microsoft.Psi.Spatial.Euclidean.Visualization { + using System.Collections.Generic; using Microsoft.Psi.Spatial.Euclidean; using Microsoft.Psi.Visualization.VisualizationObjects; /// - /// Implements a 3D depth image rectangle list visualization object. + /// Implements a visualization object for . /// - [VisualizationObject("3D Depth Image Rectangles")] + [VisualizationObject("Depth Images in 3D Rectangles")] public class DepthImageRectangle3DListVisualizationObject : ModelVisual3DListVisualizationObject { } diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/DepthImageRectangle3DVisualizationObject.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/DepthImageRectangle3DVisualizationObject.cs similarity index 98% rename from Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/DepthImageRectangle3DVisualizationObject.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/DepthImageRectangle3DVisualizationObject.cs index 44133accc..2a6eef827 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/DepthImageRectangle3DVisualizationObject.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/DepthImageRectangle3DVisualizationObject.cs @@ -18,7 +18,7 @@ namespace Microsoft.Psi.Spatial.Euclidean.Visualization /// /// Implements a 3D depth image rectangle visualization object. /// - [VisualizationObject("3D Depth Image Rectangle")] + [VisualizationObject("Depth Image in 3D Rectangle")] public class DepthImageRectangle3DVisualizationObject : ModelVisual3DVisualizationObject { private readonly MeshGeometryVisual3D depthImageModelVisual; @@ -260,14 +260,14 @@ public int RangeMax public override void UpdateData() { if (this.CurrentData != null && !this.CurrentData.Rectangle3D.IsDegenerate && - this.CurrentData.DepthImage != null && this.CurrentData.DepthImage.Resource != null) + this.CurrentData.Image != null && this.CurrentData.Image.Resource != null) { if (this.depthImage != null) { this.depthImage.Dispose(); } - this.depthImage = this.CurrentData.DepthImage.AddRef(); + this.depthImage = this.CurrentData.Image.AddRef(); var rectangle = this.CurrentData.Rectangle3D; var topLeftPoint3D = rectangle.TopLeft.ToPoint3D(); diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/ImageRectangle3DListVisualizationObject.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/ImageRectangle3DListVisualizationObject.cs new file mode 100644 index 000000000..545c2d0e8 --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/ImageRectangle3DListVisualizationObject.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean.Visualization +{ + using System.Collections.Generic; + using Microsoft.Psi.Spatial.Euclidean; + using Microsoft.Psi.Visualization.VisualizationObjects; + + /// + /// Implements a visualization object for . + /// + [VisualizationObject("Images in 3D Rectangles")] + public class ImageRectangle3DListVisualizationObject : ModelVisual3DListVisualizationObject + { + } +} \ No newline at end of file diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangle3DVisualizationObject.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/ImageRectangle3DVisualizationObject.cs similarity index 99% rename from Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangle3DVisualizationObject.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/ImageRectangle3DVisualizationObject.cs index f719035d5..772e22fb8 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangle3DVisualizationObject.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/ImageRectangles/ImageRectangle3DVisualizationObject.cs @@ -18,7 +18,7 @@ namespace Microsoft.Psi.Spatial.Euclidean.Visualization /// /// Implements a 3D image rectangle visualization object. /// - [VisualizationObject("3D Image Rectangle")] + [VisualizationObject("Image in 3D Rectangle")] public class ImageRectangle3DVisualizationObject : ModelVisual3DVisualizationObject { private readonly MeshGeometryVisual3D modelVisual; diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows.csproj b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows.csproj index 9f648c23b..b0589316f 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows.csproj +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows/Microsoft.Psi.Spatial.Euclidean.Visualization.Windows.csproj @@ -4,6 +4,7 @@ net472 Microsoft.Psi.Spatial.Euclidean.Visualization Provides visualization objects and adapters for various types defined in Microsoft.Psi.Spatial.Euclidean. + True diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/AngularVelocity3D.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/AngularVelocity3D.cs index 879a9cb6d..0ddf6a6da 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/AngularVelocity3D.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/AngularVelocity3D.cs @@ -58,7 +58,8 @@ public AngularVelocity3D(Matrix originRotation, UnitVector3D axis, doubl /// The origin of rotation. /// A destination rotation. /// The time it took to reach the destination rotation. - public AngularVelocity3D(Matrix originRotation, Matrix destinationRotation, TimeSpan time) + /// An optional angle epsilon parameter used to determine when the specified matrix contains a zero-rotation (by default 0.01 degrees). + public AngularVelocity3D(Matrix originRotation, Matrix destinationRotation, TimeSpan time, double angleEpsilon = 0.01 * Math.PI / 180) { if (originRotation.RowCount != 3 || originRotation.ColumnCount != 3 || @@ -69,7 +70,7 @@ public AngularVelocity3D(Matrix originRotation, Matrix destinati } this.OriginRotation = originRotation; - var axisAngleDistance = Vector3D.OfVector(MatrixToAxisAngle(destinationRotation * originRotation.Inverse())); + var axisAngleDistance = Vector3D.OfVector(MatrixToAxisAngle(destinationRotation * originRotation.Inverse(), angleEpsilon)); var angularSpeed = axisAngleDistance.Length / time.TotalSeconds; this.AxisAngleVector = angularSpeed == 0 ? default : axisAngleDistance.Normalize().ScaleBy(angularSpeed); } @@ -80,8 +81,13 @@ public AngularVelocity3D(Matrix originRotation, Matrix destinati /// The origin coordinate system. /// The destination coordinate system. /// The time it took to reach the destination coordinate system. - public AngularVelocity3D(CoordinateSystem originCoordinateSystem, CoordinateSystem destinationCoordinateSystem, TimeSpan time) - : this(originCoordinateSystem.GetRotationSubMatrix(), destinationCoordinateSystem.GetRotationSubMatrix(), time) + /// An optional angle epsilon parameter used to determine when the specified matrix contains a zero-rotation (by default 0.01 degrees). + public AngularVelocity3D( + CoordinateSystem originCoordinateSystem, + CoordinateSystem destinationCoordinateSystem, + TimeSpan time, + double angleEpsilon = 0.01 * Math.PI / 180) + : this(originCoordinateSystem.GetRotationSubMatrix(), destinationCoordinateSystem.GetRotationSubMatrix(), time, angleEpsilon) { } diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Bounds3D.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Bounds3D.cs index b4b89e203..25a418f00 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Bounds3D.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Bounds3D.cs @@ -126,15 +126,43 @@ public IEnumerable GetCorners() /// /// The point to check. /// True if this contains the point, otherwise false. - public bool ContainsPoint(Point3D point) + /// An optional epsilon parameter to specify a numerical tolerance. + public bool ContainsPoint(Point3D point, double epsilon = 0) { return - point.X >= this.Min.X && - point.X <= this.Max.X && - point.Y >= this.Min.Y && - point.Y <= this.Max.Y && - point.Z >= this.Min.Z && - point.Z <= this.Max.Z; + point.X >= (this.Min.X - epsilon) && + point.X <= (this.Max.X + epsilon) && + point.Y >= (this.Min.Y - epsilon) && + point.Y <= (this.Max.Y + epsilon) && + point.Z >= (this.Min.Z - epsilon) && + point.Z <= (this.Max.Z + epsilon); + } + + /// + /// Inflates the bounds by a specified scale factor. + /// + /// The scale factor. + /// The scaled bounds. + public Bounds3D Scale(double scaleFactor) + => this.Scale(scaleFactor, scaleFactor, scaleFactor); + + /// + /// Inflates the bounds by specified scale factors on different axes. + /// + /// The scale factor on the X axis. + /// The scale factor on the Y axis. + /// The scale factor on the Z axis. + /// The scaled bounds. + public Bounds3D Scale(double scaleFactorX, double scaleFactorY, double scaleFactorZ) + { + var center = this.Center; + var minX = center.X - (center.X - this.Min.X) * scaleFactorX; + var maxX = center.X + (this.Max.X - center.X) * scaleFactorX; + var minY = center.Y - (center.Y - this.Min.Y) * scaleFactorY; + var maxY = center.Y + (this.Max.Y - center.Y) * scaleFactorY; + var minZ = center.Z - (center.Z - this.Min.Z) * scaleFactorZ; + var maxZ = center.Z + (this.Max.Z - center.Z) * scaleFactorZ; + return new Bounds3D(minX, maxX, minY, maxY, minZ, maxZ); } /// diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Box3D.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Box3D.cs index 0afdae7e6..99506e3ed 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Box3D.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Box3D.cs @@ -47,20 +47,22 @@ public enum Box3DFacet /// /// Represents a 3D rectangular box. /// - public readonly struct Box3D : IEquatable + public class Box3D : IEquatable { /// - /// The pose of the box. + /// The private pose (used to determine if the Pose has been mutated and update the inversePose). /// - public readonly CoordinateSystem Pose; + [NonSerialized] + private CoordinateSystem pose = null; /// - /// The bounds for the box. + /// The inverse pose (cached for efficiently resolving ContainsPoint queries). /// - public readonly Bounds3D Bounds; + [NonSerialized] + private CoordinateSystem inversePose = null; /// - /// Initializes a new instance of the struct. + /// Initializes a new instance of the class. /// /// The x-offset of the first diagonal corner of the box relative to its origin. /// The x-offset of the second diagonal corner of the box relative to its origin. @@ -79,7 +81,7 @@ public Box3D(double x1, double x2, double y1, double y2, double z1, double z2, C } /// - /// Initializes a new instance of the struct. + /// Initializes a new instance of the class. /// /// The bounds for the box. /// An optional pose for the box (by default, at origin). @@ -90,7 +92,7 @@ public Box3D(Bounds3D bounds, CoordinateSystem pose = null) } /// - /// Initializes a new instance of the struct. + /// Initializes a new instance of the class. /// /// The origin of the rectangle. /// The x-axis of the rectangle. @@ -112,6 +114,16 @@ public Box3D(Point3D origin, UnitVector3D xAxis, UnitVector3D yAxis, UnitVector3 { } + /// + /// Gets the pose of the box. + /// + public CoordinateSystem Pose { get; } + + /// + /// Gets the bounds for the box. + /// + public Bounds3D Bounds { get; } + /// /// Gets the origin of the box. /// @@ -164,7 +176,22 @@ public Box3D(Point3D origin, UnitVector3D xAxis, UnitVector3D yAxis, UnitVector3 /// The first box. /// The second box. /// True if the boxes are the same; otherwise false. - public static bool operator ==(Box3D left, Box3D right) => left.Equals(right); + public static bool operator ==(Box3D left, Box3D right) + { + if (left is null) + { + if (right is null) + { + return true; + } + + return false; + } + else + { + return left.Equals(right); + } + } /// /// Returns a value indicating whether the specified boxes are different. @@ -172,7 +199,7 @@ public Box3D(Point3D origin, UnitVector3D xAxis, UnitVector3D yAxis, UnitVector3 /// The first box. /// The second box. /// True if the boxes are different; otherwise false. - public static bool operator !=(Box3D left, Box3D right) => !left.Equals(right); + public static bool operator !=(Box3D left, Box3D right) => !(left == right); /// /// Gets the corner points of the box. @@ -187,10 +214,19 @@ public IEnumerable GetCorners() /// /// Determines whether the contains a point. /// - /// The point to check. + /// The point to check. + /// An optional epsilon parameter to specify a numerical tolerance. /// True if this contains the point, otherwise false. - public bool ContainsPoint(Point3D point) => - this.Bounds.ContainsPoint(point.TransformBy(this.Pose.Invert())); + public bool ContainsPoint(Point3D point3D, double epsilon = 0) + { + if (!Equals(this.Pose, this.pose)) + { + this.Pose.DeepClone(ref this.pose); + this.Pose.Invert().DeepClone(ref this.inversePose); + } + + return this.Bounds.ContainsPoint(point3D.TransformBy(this.inversePose), epsilon); + } /// /// Computes the intersection point between a 3D ray and this box. @@ -223,12 +259,32 @@ public IEnumerable GetCorners() } /// - public bool Equals(Box3D other) => - this.Pose == other.Pose && this.Bounds == other.Bounds; + public bool Equals(Box3D other) + { + if (other is null) + { + return false; + } + + // Optimization for a common success case. + if (ReferenceEquals(this, other)) + { + return true; + } + + // If run-time types are not exactly the same, return false. + if (this.GetType() != other.GetType()) + { + return false; + } + + // Return true if the fields match. + return this.Pose == other.Pose && this.Bounds == other.Bounds; + } /// public override bool Equals(object obj) => - obj is Box3D other && this.Equals(other); + obj is not null && obj.GetType().Equals(typeof(Box3D)) && this.Equals((Box3D)obj); /// public override int GetHashCode() => @@ -359,5 +415,23 @@ public Box3D ToCenteredBox3D() var pose = this.Pose.TransformBy(translate); return new Box3D(bounds, pose); } + + /// + /// Inflates the by a specified scale factor. + /// + /// The scale factor. + /// The inflated . + public Box3D Scale(double scaleFactor) + => new (this.Bounds.Scale(scaleFactor), this.Pose); + + /// + /// Inflates the by specified scale factors on different axes. + /// + /// The scale factor on the X axis. + /// The scale factor on the Y axis. + /// The scale factor on the Z axis. + /// The inflated . + public Box3D Scale(double scaleFactorX, double scaleFactorY, double scaleFactorZ) + => new (this.Bounds.Scale(scaleFactorX, scaleFactorY, scaleFactorZ), this.Pose); } } diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/CameraView{T}.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/CameraView{T}.cs new file mode 100644 index 000000000..a8737077b --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/CameraView{T}.cs @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean +{ + using System; + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; + + /// + /// Represents a camera view of an object. + /// + /// Type of view. + public class CameraView + { + /// + /// The private pose (used to determine if the Pose has been mutated and update the inversePose). + /// + [NonSerialized] + private CoordinateSystem cameraPose = null; + + /// + /// The inverse pose (cached for efficiently resolving GetPixelPosition queries). + /// + [NonSerialized] + private CoordinateSystem inverseCameraPose = null; + + /// + /// Initializes a new instance of the class. + /// + /// The viewed object. + /// The camera intrinsics. + /// The camera pose. + public CameraView(T viewedObject, ICameraIntrinsics cameraIntrinsics, CoordinateSystem cameraPose) + { + this.ViewedObject = viewedObject; + this.CameraIntrinsics = cameraIntrinsics; + this.CameraPose = cameraPose; + } + + /// + /// Gets the viewed object. + /// + public T ViewedObject { get; private set; } + + /// + /// Gets the camera intrinsics. + /// + public ICameraIntrinsics CameraIntrinsics { get; private set; } + + /// + /// Gets the camera pose. + /// + public CoordinateSystem CameraPose { get; private set; } + + /// + /// Gets the corresponding pixel position for a point in 3D space. + /// + /// Point in 3D space, assuming MathNet basis (Forward=X, Left=Y, Up=Z), and the coordinate system of the camera. + /// Indicates whether to apply distortion. + /// Optional flag indicating whether to return null if point is outside the field of view (default true). + /// Point containing the pixel position. + public Point2D? GetPixelPosition(Point3D point3D, bool distort, bool nullIfOutsideFieldOfView = true) + { + if (this.CameraIntrinsics == null || this.CameraPose == null) + { + return default; + } + else + { + if (!Equals(this.CameraPose, this.cameraPose)) + { + this.CameraPose.DeepClone(ref this.cameraPose); + this.CameraPose.Invert().DeepClone(ref this.inverseCameraPose); + } + + return this.CameraIntrinsics.GetPixelPosition(this.inverseCameraPose.Transform(point3D), distort, nullIfOutsideFieldOfView); + } + } + + /// + /// Creates a stream of views from a stream of objects, a stream of camera intrinsics, and a stream of poses. + /// + /// Type of viewed object. + /// Type of camera view. + /// Camera view constructor. + /// A stream of viewed objects. + /// A stream of camera intrinsics. + /// A stream of camera poses. + /// The interpolator for the camera intrinsics stream. + /// The interpolator for the camera pose stream. + /// An optional delivery policy for the viewed object stream. + /// An optional delivery policy for the camera intrinsics stream. + /// An optional delivery policy for the camera pose stream. + /// An optional delivery policy for the tuple of viewed object and camera intrinsics stream. + /// Created camera view stream. + protected static IProducer CreateProducer( + Func ctor, + IProducer viewedObject, + IProducer cameraIntrinsics, + IProducer cameraPose, + Interpolator cameraIntrinsicsInterpolator, + Interpolator cameraPoseInterpolator, + DeliveryPolicy viewedObjectDeliveryPolicy = null, + DeliveryPolicy cameraIntrinsicsDeliveryPolicy = null, + DeliveryPolicy cameraPoseDeliveryPolicy = null, + DeliveryPolicy<(TViewedObject, ICameraIntrinsics)> viewedObjectAndCameraIntrinsicsDeliveryPolicy = null) + { + return viewedObject + .Fuse( + cameraIntrinsics, + cameraIntrinsicsInterpolator, + viewedObjectDeliveryPolicy, + cameraIntrinsicsDeliveryPolicy) + .Fuse( + cameraPose, + cameraPoseInterpolator, + viewedObjectAndCameraIntrinsicsDeliveryPolicy, + cameraPoseDeliveryPolicy) + .Select( + tuple => + { + (var image, var intrinsics, var pose) = tuple; + return ctor(image, intrinsics, pose); + }, + DeliveryPolicy.SynchronousOrThrottle); + } + } +} diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/DepthImageCameraView.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/DepthImageCameraView.cs new file mode 100644 index 000000000..d692bf40c --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/DepthImageCameraView.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean +{ + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Imaging; + + /// + /// Represents a camera view of an depth image. + /// + public class DepthImageCameraView : ImageCameraView + { + /// + /// Initializes a new instance of the class. + /// + /// The depth image. + /// The camera intrinsics. + /// The camera pose. + public DepthImageCameraView(Shared depthImage, ICameraIntrinsics cameraIntrinsics, CoordinateSystem cameraPose) + : base(depthImage, cameraIntrinsics, cameraPose) + { + } + + /// + /// Creates an stream from a stream of depth images, a stream of camera intrinsics, and a stream of poses. + /// + /// The stream of depth images. + /// A stream of camera intrinsics. + /// A stream of camera poses. + /// The interpolator for the camera intrinsics stream. + /// The interpolator for the camera pose stream. + /// An optional delivery policy for the image stream. + /// An optional delivery policy for the camera intrinsics stream. + /// An optional delivery policy for the camera pose stream. + /// An optional delivery policy for the tuple of image and camera intrinsics stream. + /// Created stream. + public static IProducer CreateProducer( + IProducer> depthImage, + IProducer cameraIntrinsics, + IProducer cameraPose, + Interpolator cameraIntrinsicsInterpolator, + Interpolator cameraPoseInterpolator, + DeliveryPolicy> depthImageDeliveryPolicy = null, + DeliveryPolicy cameraIntrinsicsDeliveryPolicy = null, + DeliveryPolicy cameraPoseDeliveryPolicy = null, + DeliveryPolicy<(Shared, ICameraIntrinsics)> depthImageAndCameraIntrinsicsDeliveryPolicy = null) + { + return CreateProducer( + (image, intrinsics, pose) => new DepthImageCameraView(image, intrinsics, pose), + depthImage, + cameraIntrinsics, + cameraPose, + cameraIntrinsicsInterpolator, + cameraPoseInterpolator, + depthImageDeliveryPolicy, + cameraIntrinsicsDeliveryPolicy, + cameraPoseDeliveryPolicy, + depthImageAndCameraIntrinsicsDeliveryPolicy); + } + } +} diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/EncodedDepthImageCameraView.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/EncodedDepthImageCameraView.cs new file mode 100644 index 000000000..00aa15cfe --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/EncodedDepthImageCameraView.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean +{ + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Imaging; + + /// + /// Represents a camera view of an encoded depth image. + /// + public class EncodedDepthImageCameraView : ImageCameraView + { + /// + /// Initializes a new instance of the class. + /// + /// The viewed encoded depth image. + /// The camera intrinsics. + /// The camera pose. + public EncodedDepthImageCameraView(Shared encodedDepthImage, ICameraIntrinsics cameraIntrinsics, CoordinateSystem cameraPose) + : base(encodedDepthImage, cameraIntrinsics, cameraPose) + { + } + + /// + /// Creates an stream from a stream of encoded depth images, a stream of camera intrinsics, and a stream of poses. + /// + /// The stream of encoded images. + /// A stream of camera intrinsics. + /// A stream of camera poses. + /// The interpolator for the camera intrinsics stream. + /// The interpolator for the camera pose stream. + /// An optional delivery policy for the encoded image stream. + /// An optional delivery policy for the camera intrinsics stream. + /// An optional delivery policy for the camera pose stream. + /// An optional delivery policy for the tuple of encoded image and camera intrinsics stream. + /// Created stream. + public static IProducer CreateProducer( + IProducer> encodedDepthImage, + IProducer cameraIntrinsics, + IProducer cameraPose, + Interpolator cameraIntrinsicsInterpolator, + Interpolator cameraPoseInterpolator, + DeliveryPolicy> encodedDepthImageDeliveryPolicy = null, + DeliveryPolicy cameraIntrinsicsDeliveryPolicy = null, + DeliveryPolicy cameraPoseDeliveryPolicy = null, + DeliveryPolicy<(Shared, ICameraIntrinsics)> encodedDepthImageAndCameraIntrinsicsDeliveryPolicy = null) + { + return CreateProducer( + (image, intrinsics, pose) => new EncodedDepthImageCameraView(image, intrinsics, pose), + encodedDepthImage, + cameraIntrinsics, + cameraPose, + cameraIntrinsicsInterpolator, + cameraPoseInterpolator, + encodedDepthImageDeliveryPolicy, + cameraIntrinsicsDeliveryPolicy, + cameraPoseDeliveryPolicy, + encodedDepthImageAndCameraIntrinsicsDeliveryPolicy); + } + } +} diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/EncodedImageCameraView.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/EncodedImageCameraView.cs new file mode 100644 index 000000000..bd1e3e2eb --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/EncodedImageCameraView.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean +{ + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Imaging; + + /// + /// Represents a camera view of an encoded image. + /// + public class EncodedImageCameraView : ImageCameraView + { + /// + /// Initializes a new instance of the class. + /// + /// The encoded image. + /// The camera intrinsics. + /// The camera pose. + public EncodedImageCameraView(Shared encodedImage, ICameraIntrinsics cameraIntrinsics, CoordinateSystem cameraPose) + : base(encodedImage, cameraIntrinsics, cameraPose) + { + } + + /// + /// Creates an stream from a stream of encoded images, a stream of camera intrinsics, and a stream of poses. + /// + /// The stream of encoded images. + /// A stream of camera intrinsics. + /// A stream of camera poses. + /// The interpolator for the camera intrinsics stream. + /// The interpolator for the camera pose stream. + /// An optional delivery policy for the encoded image stream. + /// An optional delivery policy for the camera intrinsics stream. + /// An optional delivery policy for the camera pose stream. + /// An optional delivery policy for the tuple of encoded image and camera intrinsics stream. + /// Created stream. + public static IProducer CreateProducer( + IProducer> encodedImage, + IProducer cameraIntrinsics, + IProducer cameraPose, + Interpolator cameraIntrinsicsInterpolator, + Interpolator cameraPoseInterpolator, + DeliveryPolicy> encodedImageDeliveryPolicy = null, + DeliveryPolicy cameraIntrinsicsDeliveryPolicy = null, + DeliveryPolicy cameraPoseDeliveryPolicy = null, + DeliveryPolicy<(Shared, ICameraIntrinsics)> encodedImageAndCameraIntrinsicsDeliveryPolicy = null) + { + return CreateProducer( + (image, intrinsics, pose) => new EncodedImageCameraView(image, intrinsics, pose), + encodedImage, + cameraIntrinsics, + cameraPose, + cameraIntrinsicsInterpolator, + cameraPoseInterpolator, + encodedImageDeliveryPolicy, + cameraIntrinsicsDeliveryPolicy, + cameraPoseDeliveryPolicy, + encodedImageAndCameraIntrinsicsDeliveryPolicy); + } + } +} diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/ImageCameraView.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/ImageCameraView.cs new file mode 100644 index 000000000..4806a1038 --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/ImageCameraView.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean +{ + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Imaging; + + /// + /// Represents a camera view of an image. + /// + public class ImageCameraView : ImageCameraView + { + /// + /// Initializes a new instance of the class. + /// + /// The image viewed by the spatial camera. + /// Intrinsics of the camera. + /// Pose of the camera. + public ImageCameraView(Shared image, ICameraIntrinsics cameraIntrinsics, CoordinateSystem cameraPose) + : base(image, cameraIntrinsics, cameraPose) + { + } + + /// + /// Creates an stream from a stream of images, a stream of camera intrinsics, and a stream of poses. + /// + /// The stream of images. + /// A stream of camera intrinsics. + /// A stream of camera poses. + /// The interpolator for the camera intrinsics stream. + /// The interpolator for the camera pose stream. + /// An optional delivery policy for the image stream. + /// An optional delivery policy for the camera intrinsics stream. + /// An optional delivery policy for the camera pose stream. + /// An optional delivery policy for the tuple of images and camera intrinsics stream. + /// Created stream. + public static IProducer CreateProducer( + IProducer> image, + IProducer cameraIntrinsics, + IProducer cameraPose, + Interpolator cameraIntrinsicsInterpolator, + Interpolator cameraPoseInterpolator, + DeliveryPolicy> imageDeliveryPolicy = null, + DeliveryPolicy cameraIntrinsicsDeliveryPolicy = null, + DeliveryPolicy cameraPoseDeliveryPolicy = null, + DeliveryPolicy<(Shared, ICameraIntrinsics)> imageAndCameraIntrinsicsDeliveryPolicy = null) + { + return CreateProducer( + (image, intrinsics, pose) => new ImageCameraView(image, intrinsics, pose), + image, + cameraIntrinsics, + cameraPose, + cameraIntrinsicsInterpolator, + cameraPoseInterpolator, + imageDeliveryPolicy, + cameraIntrinsicsDeliveryPolicy, + cameraPoseDeliveryPolicy, + imageAndCameraIntrinsicsDeliveryPolicy); + } + } +} diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/ImageCameraView{TImage}.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/ImageCameraView{TImage}.cs new file mode 100644 index 000000000..9fbaeb356 --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/ImageCameraView{TImage}.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean +{ + using System; + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; + using Microsoft.Psi.Imaging; + + /// + /// Base class for implementing image camera views. + /// + /// Type of image. + public class ImageCameraView : CameraView>, IDisposable + where TImage : class, IImage + { + /// + /// Initializes a new instance of the class. + /// + /// The image. + /// The camera intrinsics. + /// The camera pose. + public ImageCameraView(Shared image, ICameraIntrinsics cameraIntrinsics, CoordinateSystem cameraPose) + : base(image?.AddRef(), cameraIntrinsics, cameraPose) + { + } + + /// + public void Dispose() + { + if (this.ViewedObject != null && this.ViewedObject.Resource != null) + { + this.ViewedObject.Dispose(); + } + } + } +} diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/Operators.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/Operators.cs new file mode 100644 index 000000000..86835a107 --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/Operators.cs @@ -0,0 +1,227 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean +{ + using System; + using Microsoft.Psi.Imaging; + + /// + /// Implements a variety of operators and extension methods. + /// + public static partial class Operators + { + #region Transformation operators + + /// + /// Converts the stream of image camera views to a different pixel format. + /// + /// The source stream of image camera views. + /// Scale factor for X. + /// Scale factor for Y. + /// Method for sampling pixels when rescaling. + /// An optional delivery policy. + /// Optional image allocator for creating new shared images. + /// An optional name for the stream operator. + /// The resulting stream. + public static IProducer Scale( + this IProducer source, + float scaleX, + float scaleY, + SamplingMode samplingMode = SamplingMode.Bilinear, + DeliveryPolicy deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(Scale)) + { + sharedImageAllocator ??= (width, height, pixelFormat) => ImagePool.GetOrCreate(width, height, pixelFormat); + return source.Process( + (imageCameraView, envelope, emitter) => + { + // if the image is null, post null + if (imageCameraView == null) + { + emitter.Post(null, envelope.OriginatingTime); + } + else + { + int finalWidth = (int)(imageCameraView.ViewedObject.Resource.Width * scaleX); + int finalHeight = (int)(imageCameraView.ViewedObject.Resource.Height * scaleY); + using var scaledSharedImage = sharedImageAllocator(finalWidth, finalHeight, imageCameraView.ViewedObject.Resource.PixelFormat); + imageCameraView.ViewedObject.Resource.Scale(scaledSharedImage.Resource, scaleX, scaleY, samplingMode); + using var outputImageCameraView = new ImageCameraView(scaledSharedImage, imageCameraView.CameraIntrinsics, imageCameraView.CameraPose); + emitter.Post(outputImageCameraView, envelope.OriginatingTime); + } + }, + deliveryPolicy, + name); + } + + #endregion + + #region Convert image camera views to a different format + + /// + /// Converts the stream of image camera views to a different pixel format. + /// + /// The source stream of image camera views. + /// The pixel format to convert to. + /// An optional delivery policy. + /// Optional image allocator for creating new shared images. + /// An optional name for the stream operator. + /// The resulting stream. + public static IProducer Convert( + this IProducer source, + PixelFormat pixelFormat, + DeliveryPolicy deliveryPolicy = null, + Func> sharedImageAllocator = null, + string name = nameof(Convert)) + { + sharedImageAllocator ??= (width, height, pixelFormat) => ImagePool.GetOrCreate(width, height, pixelFormat); + return source.Process( + (imageCameraView, envelope, emitter) => + { + // if the image is null, post null + if (imageCameraView == null) + { + emitter.Post(null, envelope.OriginatingTime); + } + else if (pixelFormat == imageCameraView.ViewedObject.Resource.PixelFormat) + { + // o/w if image is already in the requested format, shortcut the conversion + emitter.Post(imageCameraView, envelope.OriginatingTime); + } + else + { + using var image = sharedImageAllocator(imageCameraView.ViewedObject.Resource.Width, imageCameraView.ViewedObject.Resource.Height, pixelFormat); + imageCameraView.ViewedObject.Resource.CopyTo(image.Resource); + using var outputImageCameraView = new ImageCameraView(image, imageCameraView.CameraIntrinsics, imageCameraView.CameraPose); + emitter.Post(outputImageCameraView, envelope.OriginatingTime); + } + }, + deliveryPolicy, + name); + } + + #endregion + + #region Encoded/Decode image and depth image camera views + + /// + /// Encodes an image camera view using a specified image encoder. + /// + /// The source stream of image camera views. + /// The image encoder to use. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A stream of encoded image camera views. + public static IProducer Encode( + this IProducer source, + IImageToStreamEncoder encoder, + DeliveryPolicy deliveryPolicy = null, + string name = null) + { + return source.Process( + (imageCameraView, envelope, emitter) => + { + var image = imageCameraView.ViewedObject.Resource; + using var encodedImage = EncodedImagePool.GetOrCreate(image.Width, image.Height, image.PixelFormat); + encodedImage.Resource.EncodeFrom(image, encoder); + using var encodedImageCameraView = new EncodedImageCameraView(encodedImage, imageCameraView.CameraIntrinsics, imageCameraView.CameraPose); + emitter.Post(encodedImageCameraView, envelope.OriginatingTime); + }, + deliveryPolicy, + name ?? $"{nameof(Encode)}({encoder.Description})"); + } + + /// + /// Encodes a depth image camera view using a specified image encoder. + /// + /// The source stream of depth image camera views. + /// The image encoder to use. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A stream of encoded depth image camera views. + public static IProducer Encode( + this IProducer source, + IDepthImageToStreamEncoder encoder, + DeliveryPolicy deliveryPolicy = null, + string name = null) + { + return source.Process( + (depthImageCameraView, envelope, emitter) => + { + var depthImage = depthImageCameraView.ViewedObject.Resource; + using var encodedDepthImage = EncodedDepthImagePool.GetOrCreate( + depthImage.Width, + depthImage.Height, + depthImage.DepthValueSemantics, + depthImage.DepthValueToMetersScaleFactor); + encodedDepthImage.Resource.EncodeFrom(depthImage, encoder); + using var encodedDepthImageCameraView = new EncodedDepthImageCameraView(encodedDepthImage, depthImageCameraView.CameraIntrinsics, depthImageCameraView.CameraPose); + emitter.Post(encodedDepthImageCameraView, envelope.OriginatingTime); + }, + deliveryPolicy, + name ?? $"{nameof(Encode)}({encoder.Description})"); + } + + /// + /// Decodes an encoded image camera view using a specified image decoder. + /// + /// The source stream of encoded image camera views. + /// The image decoder to use. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A stream of decoded image camera views. + public static IProducer Decode( + this IProducer source, + IImageFromStreamDecoder decoder, + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Decode)) + { + return source.Process( + (encodedImageCameraView, envelope, emitter) => + { + var encodedImage = encodedImageCameraView.ViewedObject.Resource; + using var image = ImagePool.GetOrCreate(encodedImage.Width, encodedImage.Height, encodedImage.PixelFormat); + image.Resource.DecodeFrom(encodedImage, decoder); + using var imageCameraView = new ImageCameraView(image, encodedImageCameraView.CameraIntrinsics, encodedImageCameraView.CameraPose); + emitter.Post(imageCameraView, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + /// + /// Decodes an encoded depth image camera view using a specified image decoder. + /// + /// The source stream of encoded depth image camera views. + /// The depth image decoder to use. + /// An optional delivery policy. + /// An optional name for the stream operator. + /// A stream of decoded depth image camera views. + public static IProducer Decode( + this IProducer source, + IDepthImageFromStreamDecoder decoder, + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Decode)) + { + return source.Process( + (encodedDepthImageCameraView, envelope, emitter) => + { + var encodedDepthImage = encodedDepthImageCameraView.ViewedObject.Resource; + using var depthImage = DepthImagePool.GetOrCreate( + encodedDepthImage.Width, + encodedDepthImage.Height, + encodedDepthImage.DepthValueSemantics, + encodedDepthImage.DepthValueToMetersScaleFactor); + depthImage.Resource.DecodeFrom(encodedDepthImage, decoder); + using var depthImageCameraView = new DepthImageCameraView(depthImage, encodedDepthImageCameraView.CameraIntrinsics, encodedDepthImageCameraView.CameraPose); + emitter.Post(depthImageCameraView, envelope.OriginatingTime); + }, + deliveryPolicy, + name); + } + + #endregion + } +} diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/PointCloud3DCameraView.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/PointCloud3DCameraView.cs new file mode 100644 index 000000000..38e15a367 --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CameraViews/PointCloud3DCameraView.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean +{ + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Calibration; + + /// + /// Represents a camera view of a 3D point cloud. + /// + public class PointCloud3DCameraView : CameraView + { + /// + /// Initializes a new instance of the class. + /// + /// The 3D point cloud. + /// Intrinsics of the camera. + /// Pose of the camera. + public PointCloud3DCameraView(PointCloud3D pointCloud3D, ICameraIntrinsics cameraIntrinsics, CoordinateSystem cameraPose) + : base(pointCloud3D, cameraIntrinsics, cameraPose) + { + } + } +} diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CoordinateSystemVelocity3D.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CoordinateSystemVelocity3D.cs index 7077baf80..389ca9523 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CoordinateSystemVelocity3D.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/CoordinateSystemVelocity3D.cs @@ -68,16 +68,18 @@ namespace Microsoft.Psi.Spatial.Euclidean /// The origin coordinate system. /// A destination coordinate system. /// The time it took to reach the destination coordinate system. + /// An optional angle epsilon parameter used to determine when the specified matrix contains a zero-rotation (by default 0.01 degrees). public CoordinateSystemVelocity3D( CoordinateSystem originCoordinateSystem, CoordinateSystem destinationCoordinateSystem, - TimeSpan time) + TimeSpan time, + double angleEpsilon = 0.01 * Math.PI / 180) { this.OriginCoordinateSystem = originCoordinateSystem; var coordinateDifference = destinationCoordinateSystem.TransformBy(originCoordinateSystem.Invert()); var timeInSeconds = time.TotalSeconds; this.LinearVector = coordinateDifference.Origin.ToVector3D().ScaleBy(1.0 / timeInSeconds); - var axisAngleDistance = Vector3D.OfVector(MatrixToAxisAngle(coordinateDifference.GetRotationSubMatrix())); + var axisAngleDistance = Vector3D.OfVector(MatrixToAxisAngle(coordinateDifference.GetRotationSubMatrix(), angleEpsilon)); var angularSpeed = axisAngleDistance.Length / timeInSeconds; this.AxisAngleVector = angularSpeed == 0 ? default : axisAngleDistance.Normalize().ScaleBy(angularSpeed); } diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Images/DepthImageRectangle3D.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/DepthImageRectangle3D.cs similarity index 63% rename from Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Images/DepthImageRectangle3D.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/DepthImageRectangle3D.cs index 3ed994836..02823effe 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Images/DepthImageRectangle3D.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/DepthImageRectangle3D.cs @@ -3,7 +3,6 @@ namespace Microsoft.Psi.Spatial.Euclidean { - using System; using MathNet.Spatial.Euclidean; using Microsoft.Psi; using Microsoft.Psi.Imaging; @@ -11,7 +10,7 @@ namespace Microsoft.Psi.Spatial.Euclidean /// /// Represents a depth image positioned in a 2D rectangle embedded in 3D space. /// - public class DepthImageRectangle3D : IDisposable + public class DepthImageRectangle3D : ImageRectangle3D { /// /// Initializes a new instance of the class. @@ -19,9 +18,8 @@ public class DepthImageRectangle3D : IDisposable /// The rectangle in 3D space to contain the depth image. /// The depth image. public DepthImageRectangle3D(Rectangle3D rectangle, Shared depthImage) + : base(rectangle, depthImage) { - this.Rectangle3D = rectangle; - this.DepthImage = depthImage.AddRef(); } /// @@ -69,16 +67,6 @@ public DepthImageRectangle3D(double scale, Point3D origin, UnitVector3D widthAxi { } - /// - /// Gets the rectangle. - /// - public Rectangle3D Rectangle3D { get; } - - /// - /// Gets the depth image. - /// - public Shared DepthImage { get; } - /// /// Tries to get the nearest pixel value, first projecting the input point into the plane of the 3D rectangle /// to determine image space pixel coordinates. @@ -90,7 +78,7 @@ public bool TryGetPixel(Point3D point, out ushort? pixelValue) { if (this.TryGetPixelCoordinates(point, out int u, out int v)) { - pixelValue = this.DepthImage.Resource.GetPixel(u, v); + pixelValue = this.Image.Resource.GetPixel(u, v); return true; } else @@ -111,61 +99,11 @@ public bool TrySetPixel(Point3D point, ushort pixelValue) { if (this.TryGetPixelCoordinates(point, out int u, out int v)) { - this.DepthImage.Resource.SetPixel(u, v, pixelValue); - return true; - } - else - { - return false; - } - } - - /// - public void Dispose() - { - this.DepthImage?.Dispose(); - } - - /// - /// Get the pixel coordinates that map to a given 3d point location. - /// - /// The 3d point in "global" coordinates. - /// The pixel u coordinate (output). - /// The pixel v coordinate (output). - /// The maximum allowed distance for projecting the point to the rectangular image plane. - /// True if the point could be projected within the bounds of the depth image rectangle, false otherwise. - public bool TryGetPixelCoordinates(Point3D point, out int u, out int v, double maxPlaneDistance = double.MaxValue) - { - // Project the given point to the corresponding plane. - var planeProjectedPoint = point.ProjectOn(Plane.FromPoints(this.Rectangle3D.TopLeft, this.Rectangle3D.TopRight, this.Rectangle3D.BottomRight)); - - // Check if the projected point is too far away from the original point. - if ((planeProjectedPoint - point).Length > maxPlaneDistance) - { - u = v = -1; - return false; - } - - // Construct a width axis pointing left-to-right and a height axis pointing top-to-bottom, - var widthVector = this.Rectangle3D.TopRight - this.Rectangle3D.TopLeft; - var heightVector = this.Rectangle3D.BottomLeft - this.Rectangle3D.TopLeft; - - // Compute the normalized projection to the width and height vectors of the rectangle - var cornerToPoint = planeProjectedPoint - this.Rectangle3D.TopLeft; - var widthVectorProjection = cornerToPoint.DotProduct(widthVector) / widthVector.DotProduct(widthVector); - var heightVectorProjection = cornerToPoint.DotProduct(heightVector) / heightVector.DotProduct(heightVector); - - // Convert to pixel coordinates - u = (int)(widthVectorProjection * this.DepthImage.Resource.Width); - v = (int)(heightVectorProjection * this.DepthImage.Resource.Height); - - if (u >= 0 && v >= 0 && u < this.DepthImage.Resource.Width && v < this.DepthImage.Resource.Height) - { + this.Image.Resource.SetPixel(u, v, pixelValue); return true; } else { - u = v = -1; return false; } } diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Images/EncodedDepthImageRectangle3D.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/EncodedDepthImageRectangle3D.cs similarity index 87% rename from Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Images/EncodedDepthImageRectangle3D.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/EncodedDepthImageRectangle3D.cs index 9a8274783..872849603 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Images/EncodedDepthImageRectangle3D.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/EncodedDepthImageRectangle3D.cs @@ -3,7 +3,6 @@ namespace Microsoft.Psi.Spatial.Euclidean { - using System; using MathNet.Spatial.Euclidean; using Microsoft.Psi; using Microsoft.Psi.Imaging; @@ -11,7 +10,7 @@ namespace Microsoft.Psi.Spatial.Euclidean /// /// Represents an encoded depth image positioned in a 2D rectangle embedded in 3D space. /// - public class EncodedDepthImageRectangle3D : IDisposable + public class EncodedDepthImageRectangle3D : ImageRectangle3D { /// /// Initializes a new instance of the class. @@ -19,9 +18,8 @@ public class EncodedDepthImageRectangle3D : IDisposable /// The rectangle in 3D space to contain the encoded depth image. /// The encoded depth image. public EncodedDepthImageRectangle3D(Rectangle3D rectangle, Shared depthImage) + : base(rectangle, depthImage) { - this.Rectangle3D = rectangle; - this.DepthImage = depthImage.AddRef(); } /// @@ -68,21 +66,5 @@ public EncodedDepthImageRectangle3D(double scale, Point3D origin, UnitVector3D w : this(new Rectangle3D(origin, widthAxis, heightAxis, 0, 0, depthImage.Resource.Width * scale, depthImage.Resource.Height * scale), depthImage) { } - - /// - /// Gets the rectangle. - /// - public Rectangle3D Rectangle3D { get; } - - /// - /// Gets the encoded depth image. - /// - public Shared DepthImage { get; } - - /// - public void Dispose() - { - this.DepthImage?.Dispose(); - } } } diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Images/EncodedImageRectangle3D.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/EncodedImageRectangle3D.cs similarity index 87% rename from Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Images/EncodedImageRectangle3D.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/EncodedImageRectangle3D.cs index de7f45692..57c0cf19f 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Images/EncodedImageRectangle3D.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/EncodedImageRectangle3D.cs @@ -3,7 +3,6 @@ namespace Microsoft.Psi.Spatial.Euclidean { - using System; using MathNet.Spatial.Euclidean; using Microsoft.Psi; using Microsoft.Psi.Imaging; @@ -11,7 +10,7 @@ namespace Microsoft.Psi.Spatial.Euclidean /// /// Represents an encoded image positioned in a 2D rectangle embedded in 3D space. /// - public class EncodedImageRectangle3D : IDisposable + public class EncodedImageRectangle3D : ImageRectangle3D { /// /// Initializes a new instance of the class. @@ -19,9 +18,8 @@ public class EncodedImageRectangle3D : IDisposable /// The rectangle in 3D space to contain the encoded image. /// The encoded image. public EncodedImageRectangle3D(Rectangle3D rectangle, Shared image) + : base(rectangle, image) { - this.Rectangle3D = rectangle; - this.Image = image.AddRef(); } /// @@ -68,21 +66,5 @@ public EncodedImageRectangle3D(double scale, Point3D origin, UnitVector3D widthA : this(new Rectangle3D(origin, widthAxis, heightAxis, 0, 0, image.Resource.Width * scale, image.Resource.Height * scale), image) { } - - /// - /// Gets the rectangle. - /// - public Rectangle3D Rectangle3D { get; } - - /// - /// Gets the encoded image. - /// - public Shared Image { get; } - - /// - public void Dispose() - { - this.Image?.Dispose(); - } } } diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Images/ImageRectangle3D.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/ImageRectangle3D.cs similarity index 69% rename from Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Images/ImageRectangle3D.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/ImageRectangle3D.cs index 2e1dca48e..66defc6c9 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Images/ImageRectangle3D.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/ImageRectangle3D.cs @@ -3,7 +3,6 @@ namespace Microsoft.Psi.Spatial.Euclidean { - using System; using MathNet.Spatial.Euclidean; using Microsoft.Psi; using Microsoft.Psi.Imaging; @@ -11,7 +10,7 @@ namespace Microsoft.Psi.Spatial.Euclidean /// /// Represents an image positioned in a 2D rectangle embedded in 3D space. /// - public class ImageRectangle3D : IDisposable + public class ImageRectangle3D : ImageRectangle3D { /// /// Initializes a new instance of the class. @@ -19,9 +18,8 @@ public class ImageRectangle3D : IDisposable /// The rectangle in 3D space to contain the image. /// The image. public ImageRectangle3D(Rectangle3D rectangle, Shared image) + : base(rectangle, image) { - this.Rectangle3D = rectangle; - this.Image = image.AddRef(); } /// @@ -69,16 +67,6 @@ public ImageRectangle3D(double scale, Point3D origin, UnitVector3D widthAxis, Un { } - /// - /// Gets the rectangle. - /// - public Rectangle3D Rectangle3D { get; } - - /// - /// Gets the image. - /// - public Shared Image { get; } - /// /// Tries to get the nearest pixel value, first projecting the input point into the plane of the 3D rectangle /// to determine image space pixel coordinates. @@ -145,55 +133,5 @@ public bool TrySetPixel(Point3D point, int gray) return false; } } - - /// - public void Dispose() - { - this.Image?.Dispose(); - } - - /// - /// Get the pixel coordinates that map to a given 3d point location. - /// - /// The 3d point in "global" coordinates. - /// The pixel u coordinate (output). - /// The pixel v coordinate (output). - /// The maximum allowed distance for projecting the point to the rectangular image plane. - /// True if the point could be projected within the bounds of the image rectangle, false otherwise. - public bool TryGetPixelCoordinates(Point3D point, out int u, out int v, double maxPlaneDistance = double.MaxValue) - { - // Project the given point to the corresponding plane. - var planeProjectedPoint = point.ProjectOn(Plane.FromPoints(this.Rectangle3D.TopLeft, this.Rectangle3D.TopRight, this.Rectangle3D.BottomRight)); - - // Check if the projected point is too far away from the original point. - if ((planeProjectedPoint - point).Length > maxPlaneDistance) - { - u = v = -1; - return false; - } - - // Construct a width axis pointing left-to-right and a height axis pointing top-to-bottom, - var widthVector = this.Rectangle3D.TopRight - this.Rectangle3D.TopLeft; - var heightVector = this.Rectangle3D.BottomLeft - this.Rectangle3D.TopLeft; - - // Compute the normalized projection to the width and height vectors of the rectangle - var cornerToPoint = planeProjectedPoint - this.Rectangle3D.TopLeft; - var widthVectorProjection = cornerToPoint.DotProduct(widthVector) / widthVector.DotProduct(widthVector); - var heightVectorProjection = cornerToPoint.DotProduct(heightVector) / heightVector.DotProduct(heightVector); - - // Convert to pixel coordinates - u = (int)(widthVectorProjection * this.Image.Resource.Width); - v = (int)(heightVectorProjection * this.Image.Resource.Height); - - if (u >= 0 && v >= 0 && u < this.Image.Resource.Width && v < this.Image.Resource.Height) - { - return true; - } - else - { - u = v = -1; - return false; - } - } } } diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/ImageRectangle3D{T}.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/ImageRectangle3D{T}.cs new file mode 100644 index 000000000..4c4be4274 --- /dev/null +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/ImageRectangle3D{T}.cs @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Spatial.Euclidean +{ + using System; + using MathNet.Spatial.Euclidean; + using Microsoft.Psi; + using Microsoft.Psi.Imaging; + + /// + /// Represents an image positioned in a 2D rectangle embedded in 3D space. + /// + /// Type of image. + public class ImageRectangle3D : IDisposable + where T : class, IImage + { + /// + /// Initializes a new instance of the class. + /// + /// The rectangle in 3D space to contain the image. + /// The image. + public ImageRectangle3D(Rectangle3D rectangle, Shared image) + { + this.Rectangle3D = rectangle; + this.Image = image.AddRef(); + } + + /// + /// Initializes a new instance of the class. + /// + /// The origin of the image rectangle. + /// The horizontal width axis of the image rectangle. + /// The vertical height axis of the image rectangle. + /// The left edge of the image rectangle (relative to origin along the width axis). + /// The bottom edge of the image rectangle (relative to origin along the height axis). + /// The width of the image rectangle. + /// The height of the image rectangle. + /// The image. + /// + /// The edges of the image rectangle are aligned to the specified width and height axes. + /// + public ImageRectangle3D( + Point3D origin, + UnitVector3D widthAxis, + UnitVector3D heightAxis, + double left, + double bottom, + double width, + double height, + Shared image) + : this(new Rectangle3D(origin, widthAxis, heightAxis, left, bottom, width, height), image) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// The scale to use when calculating metric corner offsets from image pixel width and height. + /// The origin of the image rectangle. + /// The horizontal width axis of the image rectangle. + /// The vertical height axis of the image rectangle. + /// The image. + /// + /// The (left, bottom) corner of the image rectangle is set to the origin (0, 0), and (width, height) are calculated + /// from multiplying the image pixel width and height respectively by a scaling parameter. + /// The edges of the image rectangle are aligned to the specified width and height axes. + /// + public ImageRectangle3D(double scale, Point3D origin, UnitVector3D widthAxis, UnitVector3D heightAxis, Shared image) + : this(new Rectangle3D(origin, widthAxis, heightAxis, 0, 0, image.Resource.Width * scale, image.Resource.Height * scale), image) + { + } + + /// + /// Gets the rectangle. + /// + public Rectangle3D Rectangle3D { get; } + + /// + /// Gets the image. + /// + public Shared Image { get; } + + /// + public void Dispose() + { + this.Image?.Dispose(); + } + + /// + /// Get the pixel coordinates that map to a given 3d point location. + /// + /// The 3d point in "global" coordinates. + /// The pixel u coordinate (output). + /// The pixel v coordinate (output). + /// The maximum allowed distance for projecting the point to the rectangular image plane. + /// True if the point could be projected within the bounds of the image rectangle, false otherwise. + public bool TryGetPixelCoordinates(Point3D point, out int u, out int v, double maxPlaneDistance = double.MaxValue) + { + // Project the given point to the corresponding plane. + var planeProjectedPoint = point.ProjectOn(Plane.FromPoints(this.Rectangle3D.TopLeft, this.Rectangle3D.TopRight, this.Rectangle3D.BottomRight)); + + // Check if the projected point is too far away from the original point. + if ((planeProjectedPoint - point).Length > maxPlaneDistance) + { + u = v = -1; + return false; + } + + // Construct a width axis pointing left-to-right and a height axis pointing top-to-bottom, + var widthVector = this.Rectangle3D.TopRight - this.Rectangle3D.TopLeft; + var heightVector = this.Rectangle3D.BottomLeft - this.Rectangle3D.TopLeft; + + // Compute the normalized projection to the width and height vectors of the rectangle + var cornerToPoint = planeProjectedPoint - this.Rectangle3D.TopLeft; + var widthVectorProjection = cornerToPoint.DotProduct(widthVector) / widthVector.DotProduct(widthVector); + var heightVectorProjection = cornerToPoint.DotProduct(heightVector) / heightVector.DotProduct(heightVector); + + // Convert to pixel coordinates + u = (int)(widthVectorProjection * this.Image.Resource.Width); + v = (int)(heightVectorProjection * this.Image.Resource.Height); + + if (u >= 0 && v >= 0 && u < this.Image.Resource.Width && v < this.Image.Resource.Height) + { + return true; + } + else + { + u = v = -1; + return false; + } + } + } +} diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Images/Operators.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/Operators.cs similarity index 67% rename from Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Images/Operators.cs rename to Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/Operators.cs index b76a5598e..1283f4279 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Images/Operators.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/ImageRectangles/Operators.cs @@ -7,7 +7,7 @@ namespace Microsoft.Psi.Spatial.Euclidean using Microsoft.Psi.Imaging; /// - /// Implements operators for processing spatial image types. + /// Implements operators for processing image rectangle 3Ds. /// public static partial class Operators { @@ -17,21 +17,24 @@ public static partial class Operators /// The source stream of image rectangles. /// The image encoder to use. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of encoded image rectangles. public static IProducer Encode( this IProducer source, IImageToStreamEncoder encoder, - DeliveryPolicy deliveryPolicy = null) + DeliveryPolicy deliveryPolicy = null, + string name = null) { return source.Process( - (imageRectangle, envelope, emitter) => + (imageRectangle3D, envelope, emitter) => { - var image = imageRectangle.Image.Resource; + var image = imageRectangle3D.Image.Resource; using var encodedImage = EncodedImagePool.GetOrCreate(image.Width, image.Height, image.PixelFormat); encodedImage.Resource.EncodeFrom(image, encoder); - emitter.Post(new EncodedImageRectangle3D(imageRectangle.Rectangle3D, encodedImage), envelope.OriginatingTime); + emitter.Post(new EncodedImageRectangle3D(imageRectangle3D.Rectangle3D, encodedImage), envelope.OriginatingTime); }, - deliveryPolicy); + deliveryPolicy, + name ?? $"{nameof(Encode)}({encoder.Description})"); } /// @@ -40,21 +43,28 @@ public static partial class Operators /// The source stream of depth image rectangles. /// The depth image encoder to use. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of encoded depth image rectangles. public static IProducer Encode( this IProducer source, IDepthImageToStreamEncoder encoder, - DeliveryPolicy deliveryPolicy = null) + DeliveryPolicy deliveryPolicy = null, + string name = null) { return source.Process( - (depthImageRectangle, envelope, emitter) => + (depthImageRectangle3D, envelope, emitter) => { - var depthImage = depthImageRectangle.DepthImage.Resource; - using var encodedDepthImage = EncodedDepthImagePool.GetOrCreate(depthImage.Width, depthImage.Height); + var depthImage = depthImageRectangle3D.Image.Resource; + using var encodedDepthImage = EncodedDepthImagePool.GetOrCreate( + depthImage.Width, + depthImage.Height, + depthImage.DepthValueSemantics, + depthImage.DepthValueToMetersScaleFactor); encodedDepthImage.Resource.EncodeFrom(depthImage, encoder); - emitter.Post(new EncodedDepthImageRectangle3D(depthImageRectangle.Rectangle3D, encodedDepthImage), envelope.OriginatingTime); + emitter.Post(new EncodedDepthImageRectangle3D(depthImageRectangle3D.Rectangle3D, encodedDepthImage), envelope.OriginatingTime); }, - deliveryPolicy); + deliveryPolicy, + name ?? $"{nameof(Encode)}({encoder.Description})"); } /// @@ -63,21 +73,24 @@ public static partial class Operators /// The source stream of encoded image rectangles. /// The image decoder to use. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of decoded image rectangles. public static IProducer Decode( this IProducer source, IImageFromStreamDecoder decoder, - DeliveryPolicy deliveryPolicy = null) + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Decode)) { return source.Process( - (encodedImageRectangle, envelope, emitter) => + (encodedImageRectangle3D, envelope, emitter) => { - var encodedImage = encodedImageRectangle.Image.Resource; + var encodedImage = encodedImageRectangle3D.Image.Resource; using var image = ImagePool.GetOrCreate(encodedImage.Width, encodedImage.Height, encodedImage.PixelFormat); image.Resource.DecodeFrom(encodedImage, decoder); - emitter.Post(new ImageRectangle3D(encodedImageRectangle.Rectangle3D, image), envelope.OriginatingTime); + emitter.Post(new ImageRectangle3D(encodedImageRectangle3D.Rectangle3D, image), envelope.OriginatingTime); }, - deliveryPolicy); + deliveryPolicy, + name); } /// @@ -86,21 +99,28 @@ public static partial class Operators /// The source stream of encoded depth image rectangles. /// The depth image decoder to use. /// An optional delivery policy. + /// An optional name for the stream operator. /// A stream of decoded depth image rectangles. public static IProducer Decode( this IProducer source, IDepthImageFromStreamDecoder decoder, - DeliveryPolicy deliveryPolicy = null) + DeliveryPolicy deliveryPolicy = null, + string name = nameof(Decode)) { return source.Process( - (encodedDepthImageRectangle, envelope, emitter) => + (encodedDepthImageRectangle3D, envelope, emitter) => { - var encodedDepthImage = encodedDepthImageRectangle.DepthImage.Resource; - using var depthImage = DepthImagePool.GetOrCreate(encodedDepthImage.Width, encodedDepthImage.Height); + var encodedDepthImage = encodedDepthImageRectangle3D.Image.Resource; + using var depthImage = DepthImagePool.GetOrCreate( + encodedDepthImage.Width, + encodedDepthImage.Height, + encodedDepthImage.DepthValueSemantics, + encodedDepthImage.DepthValueToMetersScaleFactor); depthImage.Resource.DecodeFrom(encodedDepthImage, decoder); - emitter.Post(new DepthImageRectangle3D(encodedDepthImageRectangle.Rectangle3D, depthImage), envelope.OriginatingTime); + emitter.Post(new DepthImageRectangle3D(encodedDepthImageRectangle3D.Rectangle3D, depthImage), envelope.OriginatingTime); }, - deliveryPolicy); + deliveryPolicy, + name); } } } diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Mesh3D.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Mesh3D.cs index fe4ef1316..6f1dd6f03 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Mesh3D.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Mesh3D.cs @@ -30,5 +30,11 @@ public Mesh3D(Point3D[] vertices, uint[] triangleIndices) /// Gets mesh triangle indices. /// public uint[] TriangleIndices { get; private set; } + + /// + /// Converts the mesh vertices to a point cloud. + /// + /// A for the mesh vertices. + public PointCloud3D ToPointCloud3D() => new (this.Vertices); } } diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Operators.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Operators.cs index a8f334380..a4267f010 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Operators.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Operators.cs @@ -8,7 +8,7 @@ namespace Microsoft.Psi.Spatial.Euclidean using MathNet.Spatial.Euclidean; /// - /// Implements various operators for manipulating spatial euclidean entities. + /// Implements various operators for manipulating euclidean entities. /// public static partial class Operators { @@ -53,9 +53,10 @@ public static Box3D Transform(this CoordinateSystem coordinateSystem, Box3D box3 /// /// The source stream of coordinate systems. /// An optional delivery policy parameter. + /// An optional name for the stream operator. /// A stream containing the linear velocity of the specified point. - public static IProducer GetLinearVelocity3D(this IProducer source, DeliveryPolicy deliveryPolicy = null) => - source.GetLinearVelocity3D(cs => cs?.Origin, deliveryPolicy); + public static IProducer GetLinearVelocity3D(this IProducer source, DeliveryPolicy deliveryPolicy = null, string name = nameof(GetLinearVelocity3D)) + => source.GetLinearVelocity3D(cs => cs?.Origin, deliveryPolicy, name); /// /// Computes the linear velocity of an object. @@ -64,8 +65,9 @@ public static Box3D Transform(this CoordinateSystem coordinateSystem, Box3D box3 /// The source stream of points. /// A function that specifies the location of the object. /// An optional delivery policy parameter. + /// An optional name for the stream operator. /// A stream containing the linear velocity of the specified point. - public static IProducer GetLinearVelocity3D(this IProducer source, Func getLocation, DeliveryPolicy deliveryPolicy = null) + public static IProducer GetLinearVelocity3D(this IProducer source, Func getLocation, DeliveryPolicy deliveryPolicy = null, string name = nameof(GetLinearVelocity3D)) { var lastPoint3D = default(Point3D?); var lastDateTime = DateTime.MinValue; @@ -82,7 +84,8 @@ public static Box3D Transform(this CoordinateSystem coordinateSystem, Box3D box3 lastPoint3D = point3D; lastDateTime = envelope.OriginatingTime; }, - deliveryPolicy); + deliveryPolicy, + name); } /// diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/PointCloud3D.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/PointCloud3D.cs index 505d91de7..245a95ed9 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/PointCloud3D.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/PointCloud3D.cs @@ -50,18 +50,23 @@ public PointCloud3D(IEnumerable points) /// /// Initializes a new instance of the class. /// - /// This private constructor creates an empty point cloud. - private PointCloud3D() + /// The set of points expressed in a count x 4 matrix. + public PointCloud3D(Matrix points) { + if (points != null && points.RowCount != 4) + { + throw new System.Exception("The points matrix should have 4 rows when constructing a point cloud."); + } + + this.points = points; } /// /// Initializes a new instance of the class. /// - /// The set of points expressed in a count x 4 matrix. - private PointCloud3D(Matrix points) + /// This private constructor creates an empty point cloud. + private PointCloud3D() { - this.points = points; } /// @@ -79,49 +84,49 @@ private PointCloud3D(Matrix points) /// /// The depth image. /// The depth camera intrinsics. - /// An optional parameter representing a scale factor to apply to the depth values. /// An optional parameter to specify how sparsely to sample pixels (by default 1). /// An optional parameter that specifies whether to undistort when projecting through the intrinsics. /// An optional parameter that indicates to return only robust points (where the nearby depth estimates are not zero). /// The corresponding point cloud. - public static PointCloud3D FromDepthImage(Shared depthImage, ICameraIntrinsics depthCameraIntrinsics, double scaleFactor = 1, int sparsity = 1, bool undistort = true, bool robustPointsOnly = false) - => FromDepthImage(depthImage?.Resource, depthCameraIntrinsics, scaleFactor, sparsity, undistort, robustPointsOnly); + public static PointCloud3D FromDepthImage(Shared depthImage, ICameraIntrinsics depthCameraIntrinsics, int sparsity = 1, bool undistort = true, bool robustPointsOnly = false) + => FromDepthImage(depthImage?.Resource, depthCameraIntrinsics, sparsity, undistort, robustPointsOnly); /// /// Create a point cloud from a shared depth image. /// /// The depth image. /// A camera space mapping matrix. - /// An optional parameter representing a scale factor to apply to the depth values (by default 1). /// An optional parameter to specify how sparsely to sample pixels (by default 1). /// An optional parameter that indicates to return only robust points (where the nearby depth estimates are not zero). /// The corresponding point cloud. - public static PointCloud3D FromDepthImage(Shared depthImage, Point3D[,] cameraSpaceMapping, double scalingFactor = 1, int sparsity = 1, bool robustPointsOnly = false) - => FromDepthImage(depthImage?.Resource, cameraSpaceMapping, scalingFactor, sparsity, robustPointsOnly); + public static PointCloud3D FromDepthImage(Shared depthImage, Point3D[,] cameraSpaceMapping, int sparsity = 1, bool robustPointsOnly = false) + => FromDepthImage(depthImage?.Resource, cameraSpaceMapping, sparsity, robustPointsOnly); /// /// Create a point cloud from a depth image. /// /// The depth image. /// The depth camera intrinsics. - /// An optional parameter representing a scale factor to apply to the depth values (by default 1). /// An optional parameter to specify how sparsely to sample pixels (by default 1). /// An optional parameter that specifies whether to undistort when projecting through the intrinsics. /// An optional parameter that indicates to return only robust points (where the nearby depth estimates are not zero). /// The corresponding point cloud. - public static PointCloud3D FromDepthImage(DepthImage depthImage, ICameraIntrinsics depthCameraIntrinsics, double scalingFactor = 1, int sparsity = 1, bool undistort = true, bool robustPointsOnly = false) - => FromDepthImage(depthImage, depthCameraIntrinsics?.GetPixelToCameraSpaceMapping(undistort), scalingFactor, sparsity, robustPointsOnly); + public static PointCloud3D FromDepthImage(DepthImage depthImage, ICameraIntrinsics depthCameraIntrinsics, int sparsity = 1, bool undistort = true, bool robustPointsOnly = false) + => FromDepthImage( + depthImage, + depthCameraIntrinsics?.GetPixelToCameraSpaceMapping(depthImage.DepthValueSemantics, undistort), + sparsity, + robustPointsOnly); /// /// Create a point cloud from a depth image. /// /// The depth image. /// A camera space mapping matrix. - /// An optional parameter representing a scale factor to apply to the depth values (by default 1). /// An optional parameter to specify how sparsely to sample pixels (by default 1). /// An optional parameter that indicates to return only robust points (where the nearby depth estimates are not zero). /// The corresponding point cloud. - public static PointCloud3D FromDepthImage(DepthImage depthImage, Point3D[,] cameraSpaceMapping, double scalingFactor = 1, int sparsity = 1, bool robustPointsOnly = false) + public static PointCloud3D FromDepthImage(DepthImage depthImage, Point3D[,] cameraSpaceMapping, int sparsity = 1, bool robustPointsOnly = false) { if (depthImage == null || cameraSpaceMapping == null) { @@ -171,6 +176,7 @@ public static PointCloud3D FromDepthImage(DepthImage depthImage, Point3D[,] came } // Then iterate again and compute the points + var scalingFactor = depthImage.DepthValueToMetersScaleFactor; var points = Matrix.Build.Dense(4, count); int index = 0; for (int iy = 0; iy < depthImage.Height; iy += sparsity) diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Rectangle3D.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Rectangle3D.cs index 93da9be21..f859eaddd 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Rectangle3D.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/Rectangle3D.cs @@ -5,6 +5,7 @@ namespace Microsoft.Psi.Spatial.Euclidean { using System; using System.Collections.Generic; + using System.Linq; using MathNet.Spatial.Euclidean; /// @@ -166,6 +167,33 @@ public IEnumerable GetCorners() return this.Contains(intersection, tolerance) ? intersection : null; } + /// + /// Computes the closest point in this planar rectangle to a specified 3D point. + /// + /// The 3D point to compute the closest point to. + /// The intersection point, if one exists. + public Point3D ClosestPointTo(Point3D point3D) + { + // compute the plane of the rectangle from three corner points + var candidates = new List(); + + // compute the projection of the point in the plane of the rectangle, and, + // if that projection falls inside the rectangle, that's the closest point + var projectedPoint3D = point3D.ProjectOn(this.GetPlane()); + if (this.Contains(projectedPoint3D)) + { + return projectedPoint3D; + } + + candidates.Add(new LineSegment3D(this.BottomLeft, this.BottomRight).ClosestPointTo(point3D)); + candidates.Add(new LineSegment3D(this.BottomRight, this.TopRight).ClosestPointTo(point3D)); + candidates.Add(new LineSegment3D(this.TopRight, this.TopLeft).ClosestPointTo(point3D)); + candidates.Add(new LineSegment3D(this.TopLeft, this.BottomLeft).ClosestPointTo(point3D)); + + var minDistance = candidates.Min(c => c.DistanceTo(point3D)); + return candidates.First(c => c.DistanceTo(point3D) == minDistance); + } + /// /// Gets the in which the lies. /// @@ -173,6 +201,21 @@ public IEnumerable GetCorners() public Plane GetPlane() => Plane.FromPoints(this.TopLeft, this.TopRight, this.BottomLeft); + /// + /// Get a coordinate system pose, with origin at the center. + /// X-axis points in the facing direction of the normal. + /// Y-axis points in the width direction. + /// Z-axis points in the height direction. + /// + /// The centered coordinate system pose for the rectangle. + public CoordinateSystem GetCenteredCoordinateSystem() + { + var widthVector = (this.BottomRight - this.BottomLeft).Normalize(); + var heightVector = (this.TopLeft - this.BottomLeft).Normalize(); + var normalVector = widthVector.CrossProduct(heightVector); + return new CoordinateSystem(this.GetCenter(), normalVector, widthVector, heightVector); + } + /// /// Determines whether the rectangle contains a specified point. /// diff --git a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/VoxelGrid.cs b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/VoxelGrid.cs index b7bfb3849..f7585f5f5 100644 --- a/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/VoxelGrid.cs +++ b/Sources/Spatial/Microsoft.Psi.Spatial.Euclidean/VoxelGrid.cs @@ -42,18 +42,7 @@ public VoxelGrid(double voxelSize) /// /// The index. /// The voxel. - public Voxel this[(int x, int y, int z) index] - { - get - { - if (!this.voxels.ContainsKey(index)) - { - throw new ArgumentOutOfRangeException("No voxel is available at the specified index."); - } - - return this.voxels[index]; - } - } + public Voxel this[(int x, int y, int z) index] => this.voxels[index]; /// /// Gets the voxel for a specified set of coordinates. diff --git a/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemSpeechIntentDetector.cs b/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemSpeechIntentDetector.cs index 7febd178f..e1aaa0833 100644 --- a/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemSpeechIntentDetector.cs +++ b/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemSpeechIntentDetector.cs @@ -34,8 +34,9 @@ public sealed class SystemSpeechIntentDetector : ConsumerProducer /// The pipeline to add the component to. /// The component configuration. - public SystemSpeechIntentDetector(Pipeline pipeline, SystemSpeechIntentDetectorConfiguration configuration) - : base(pipeline) + /// An optional name for the component. + public SystemSpeechIntentDetector(Pipeline pipeline, SystemSpeechIntentDetectorConfiguration configuration, string name = nameof(SystemSpeechIntentDetector)) + : base(pipeline, name) { this.pipeline = pipeline; this.Configuration = configuration ?? new SystemSpeechIntentDetectorConfiguration(); @@ -53,10 +54,12 @@ public SystemSpeechIntentDetector(Pipeline pipeline, SystemSpeechIntentDetectorC /// /// The pipeline to add the component to. /// The component configuration file. - public SystemSpeechIntentDetector(Pipeline pipeline, string configurationFilename = null) + /// An optional name for the component. + public SystemSpeechIntentDetector(Pipeline pipeline, string configurationFilename = null, string name = nameof(SystemSpeechIntentDetector)) : this( pipeline, - (configurationFilename == null) ? new SystemSpeechIntentDetectorConfiguration() : new ConfigurationHelper(configurationFilename).Configuration) + (configurationFilename == null) ? new SystemSpeechIntentDetectorConfiguration() : new ConfigurationHelper(configurationFilename).Configuration, + name) { } diff --git a/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemSpeechRecognizer.cs b/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemSpeechRecognizer.cs index fbe0b382d..a04fc7290 100644 --- a/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemSpeechRecognizer.cs +++ b/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemSpeechRecognizer.cs @@ -73,8 +73,9 @@ public sealed class SystemSpeechRecognizer : ConsumerProducer /// The pipeline to add the component to. /// The component configuration. - public SystemSpeechRecognizer(Pipeline pipeline, SystemSpeechRecognizerConfiguration configuration) - : base(pipeline) + /// An optional name for the component. + public SystemSpeechRecognizer(Pipeline pipeline, SystemSpeechRecognizerConfiguration configuration, string name = nameof(SystemSpeechRecognizer)) + : base(pipeline, name) { this.Configuration = configuration ?? new SystemSpeechRecognizerConfiguration(); @@ -124,10 +125,12 @@ public SystemSpeechRecognizer(Pipeline pipeline, SystemSpeechRecognizerConfigura /// /// The pipeline to add the component to. /// The component configuration file. - public SystemSpeechRecognizer(Pipeline pipeline, string configurationFilename = null) + /// An optional name for the component. + public SystemSpeechRecognizer(Pipeline pipeline, string configurationFilename = null, string name = nameof(SystemSpeechRecognizer)) : this( pipeline, - (configurationFilename == null) ? new SystemSpeechRecognizerConfiguration() : new ConfigurationHelper(configurationFilename).Configuration) + (configurationFilename == null) ? new SystemSpeechRecognizerConfiguration() : new ConfigurationHelper(configurationFilename).Configuration, + name) { } diff --git a/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemSpeechSynthesizer.cs b/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemSpeechSynthesizer.cs index f8796b6d8..1de96892c 100644 --- a/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemSpeechSynthesizer.cs +++ b/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemSpeechSynthesizer.cs @@ -44,8 +44,9 @@ public sealed class SystemSpeechSynthesizer : ConsumerProducer /// The pipeline to add the component to. /// The component configuration. - public SystemSpeechSynthesizer(Pipeline pipeline, SystemSpeechSynthesizerConfiguration configuration) - : base(pipeline) + /// An optional name for the component. + public SystemSpeechSynthesizer(Pipeline pipeline, SystemSpeechSynthesizerConfiguration configuration, string name = nameof(SystemSpeechSynthesizer)) + : base(pipeline, name) { this.pipeline = pipeline; @@ -74,10 +75,12 @@ public SystemSpeechSynthesizer(Pipeline pipeline, SystemSpeechSynthesizerConfigu /// /// The pipeline to add the component to. /// The component configuration file. - public SystemSpeechSynthesizer(Pipeline pipeline, string configurationFilename = null) + /// An optional name for the component. + public SystemSpeechSynthesizer(Pipeline pipeline, string configurationFilename = null, string name = nameof(SystemSpeechSynthesizer)) : this( pipeline, - (configurationFilename == null) ? new SystemSpeechSynthesizerConfiguration() : new ConfigurationHelper(configurationFilename).Configuration) + (configurationFilename == null) ? new SystemSpeechSynthesizerConfiguration() : new ConfigurationHelper(configurationFilename).Configuration, + name) { } diff --git a/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemVoiceActivityDetector.cs b/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemVoiceActivityDetector.cs index 3d07a22bc..616394edf 100644 --- a/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemVoiceActivityDetector.cs +++ b/Sources/Speech/Microsoft.Psi.Speech.Windows/SystemVoiceActivityDetector.cs @@ -76,8 +76,9 @@ public sealed class SystemVoiceActivityDetector : ConsumerProducer /// The pipeline to add the component to. /// The component configuration. - public SystemVoiceActivityDetector(Pipeline pipeline, SystemVoiceActivityDetectorConfiguration configuration) - : base(pipeline) + /// An optional name for the component. + public SystemVoiceActivityDetector(Pipeline pipeline, SystemVoiceActivityDetectorConfiguration configuration, string name = nameof(SystemVoiceActivityDetector)) + : base(pipeline, name) { this.configuration = configuration ?? new SystemVoiceActivityDetectorConfiguration(); @@ -101,10 +102,12 @@ public SystemVoiceActivityDetector(Pipeline pipeline, SystemVoiceActivityDetecto /// /// The pipeline to add the component to. /// The component configuration file. - public SystemVoiceActivityDetector(Pipeline pipeline, string configurationFilename = null) + /// An optional name for the component. + public SystemVoiceActivityDetector(Pipeline pipeline, string configurationFilename = null, string name = nameof(SystemVoiceActivityDetector)) : this( pipeline, - (configurationFilename == null) ? new SystemVoiceActivityDetectorConfiguration() : new ConfigurationHelper(configurationFilename).Configuration) + (configurationFilename == null) ? new SystemVoiceActivityDetectorConfiguration() : new ConfigurationHelper(configurationFilename).Configuration, + name) { } diff --git a/Sources/Tools/PsiStudio/Microsoft.Psi.PsiStudio/MainWindow.xaml b/Sources/Tools/PsiStudio/Microsoft.Psi.PsiStudio/MainWindow.xaml index 7a8db6b10..71dd771ee 100644 --- a/Sources/Tools/PsiStudio/Microsoft.Psi.PsiStudio/MainWindow.xaml +++ b/Sources/Tools/PsiStudio/Microsoft.Psi.PsiStudio/MainWindow.xaml @@ -24,7 +24,10 @@ + + + @@ -71,8 +74,8 @@ - - + + @@ -123,6 +126,12 @@ + + @@ -191,6 +200,9 @@ + @@ -245,100 +257,113 @@ HorizontalContentAlignment="Stretch"> - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + - - + diff --git a/Sources/Tools/PsiStudio/Microsoft.Psi.PsiStudio/MainWindowViewModel.cs b/Sources/Tools/PsiStudio/Microsoft.Psi.PsiStudio/MainWindowViewModel.cs index 5e177e1eb..cea31dc35 100644 --- a/Sources/Tools/PsiStudio/Microsoft.Psi.PsiStudio/MainWindowViewModel.cs +++ b/Sources/Tools/PsiStudio/Microsoft.Psi.PsiStudio/MainWindowViewModel.cs @@ -76,6 +76,7 @@ public class MainWindowViewModel : ObservableObject private object selectedPropertiesObject; private RelayCommand playPauseCommand; + private RelayCommand goToTimeCommand; private RelayCommand toggleCursorFollowsMouseComand; private RelayCommand nudgeRightCommand; private RelayCommand nudgeLeftCommand; @@ -91,6 +92,8 @@ public class MainWindowViewModel : ObservableObject private RelayCommand createAnnotationStreamCommand; private RelayCommand zoomToSessionExtentsCommand; private RelayCommand zoomToSelectionCommand; + private RelayCommand moveSelectionLeftCommand; + private RelayCommand moveSelectionRightCommand; private RelayCommand clearSelectionCommand; private RelayCommand moveToSelectionStartCommand; private RelayCommand togglePlayRepeatCommand; @@ -112,8 +115,6 @@ public class MainWindowViewModel : ObservableObject private RelayCommand treeSelectedCommand; private RelayCommand closedCommand; private RelayCommand exitCommand; - private RelayCommand autoSaveDatasetsCommand; - private RelayCommand autoLoadMRUDatasetOnStartupCommand; ////private RelayCommand showSettingsWindowComand; @@ -207,6 +208,14 @@ public RelayCommand PlayPauseCommand () => VisualizationContext.Instance.PlayOrPause(), () => this.VisualizationContainer.Navigator.CursorMode != CursorMode.Live); + /// + /// Gets the go-to-time command. + /// + [Browsable(false)] + [IgnoreDataMember] + public RelayCommand GoToTimeCommand + => this.goToTimeCommand ??= new RelayCommand(() => VisualizationContext.Instance.VisualizationContainer.GoToTime()); + /// /// Gets the toggle cursor follows mouse command. /// @@ -337,6 +346,11 @@ public RelayCommand SaveDatasetAsCommand // this should be a relatively quick operation so no need to show progress await VisualizationContext.Instance.DatasetViewModel.SaveAsAsync(filename); + + if (this.AppSettings.AutoLoadMRUDatasetOnStartUp) + { + this.AppSettings.MRUDatasetFilename = filename; + } } }); @@ -396,6 +410,26 @@ public RelayCommand ZoomToSelectionCommand () => this.VisualizationContainer.Navigator.ZoomToSelection(), () => this.VisualizationContainer.Navigator.CanZoomToSelection()); + /// + /// Gets the move selection left command. + /// + [Browsable(false)] + [IgnoreDataMember] + public RelayCommand MoveSelectionLeftCommand + => this.moveSelectionLeftCommand ??= new RelayCommand( + () => this.VisualizationContainer.Navigator.MoveSelectionLeft(), + () => this.VisualizationContainer.Navigator.CanMoveSelectionLeft()); + + /// + /// Gets the move selection right command. + /// + [Browsable(false)] + [IgnoreDataMember] + public RelayCommand MoveSelectionRightCommand + => this.moveSelectionRightCommand ??= new RelayCommand( + () => this.VisualizationContainer.Navigator.MoveSelectionRight(), + () => this.VisualizationContainer.Navigator.CanMoveSelectionRight()); + /// /// Gets the clear selection command. /// @@ -640,35 +674,6 @@ public RelayCommand ExitCommand public RelayCommand CreateAnnotationStreamCommand => this.createAnnotationStreamCommand ??= new RelayCommand(() => this.CreateAnnotationStream()); - /// - /// Gets the auto save datasets command. - /// - [Browsable(false)] - [IgnoreDataMember] - public RelayCommand AutoSaveDatasetsCommand - => this.autoSaveDatasetsCommand ??= new RelayCommand( - () => this.AppSettings.AutoSaveDatasets = !this.AppSettings.AutoSaveDatasets); - - /// - /// Gets the auto load MRU dataset on startup command. - /// - [Browsable(false)] - [IgnoreDataMember] - public RelayCommand AutoLoadMRUDatasetOnStartupCommand - => this.autoLoadMRUDatasetOnStartupCommand ??= new RelayCommand( - () => - { - if (this.AppSettings.AutoLoadMRUDatasetOnStartUp) - { - this.AppSettings.MRUDatasetFilename = null; - this.AppSettings.AutoLoadMRUDatasetOnStartUp = false; - } - else - { - this.AppSettings.AutoLoadMRUDatasetOnStartUp = true; - } - }); - /*/// /// Gets the show settings window command. /// @@ -941,15 +946,18 @@ private void EnsureDerivedStreamTreeNodesExist() // Get the current session var currentSessionViewModel = VisualizationContext.Instance.DatasetViewModel.CurrentSessionViewModel; - foreach (IStreamVisualizationObject derivedStreamVisualizationObject in derivedStreamVisualizationObjects) + if (currentSessionViewModel != null) { - // Get the stream tree node for stream being used by the stream member visualizer. - var streamTreeNode = currentSessionViewModel.FindStreamTreeNode( - derivedStreamVisualizationObject.StreamBinding.PartitionName, - derivedStreamVisualizationObject.StreamBinding.StreamName); + foreach (IStreamVisualizationObject derivedStreamVisualizationObject in derivedStreamVisualizationObjects) + { + // Get the stream tree node for stream being used by the stream member visualizer. + var streamTreeNode = currentSessionViewModel.FindStreamTreeNode( + derivedStreamVisualizationObject.StreamBinding.PartitionName, + derivedStreamVisualizationObject.StreamBinding.StreamName); - // If the session contains the stream, ensure its member children have been created. - streamTreeNode?.EnsureDerivedStreamExists(derivedStreamVisualizationObject.StreamBinding); + // If the session contains the stream, ensure its member children have been created. + streamTreeNode?.EnsureDerivedStreamExists(derivedStreamVisualizationObject.StreamBinding); + } } } } @@ -1085,8 +1093,7 @@ private void LoadAnnotationSchemas() var fileInfos = directoryInfo.GetFiles("*.schema.json"); foreach (var fileInfo in fileInfos) { - var annotationSchema = AnnotationSchema.Load(fileInfo.FullName); - if (annotationSchema != null) + if (AnnotationSchema.TryLoadFrom(fileInfo.FullName, out var annotationSchema)) { this.annotationSchemas.Add(annotationSchema); } diff --git a/Sources/Tools/PsiStudio/Microsoft.Psi.PsiStudio/PsiStudioSettings.cs b/Sources/Tools/PsiStudio/Microsoft.Psi.PsiStudio/PsiStudioSettings.cs index e319ce7ae..2c6f6efc4 100644 --- a/Sources/Tools/PsiStudio/Microsoft.Psi.PsiStudio/PsiStudioSettings.cs +++ b/Sources/Tools/PsiStudio/Microsoft.Psi.PsiStudio/PsiStudioSettings.cs @@ -6,11 +6,9 @@ namespace Microsoft.Psi.PsiStudio using System; using System.Collections.Generic; using System.IO; - using System.Reflection; using System.Windows; using System.Xml; using System.Xml.Serialization; - using Microsoft.Psi.Data; using Microsoft.Psi.Visualization.Windows; /// diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/CameraViewToSpatialCameraViewAdapter.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/CameraViewToSpatialCameraViewAdapter.cs deleted file mode 100644 index 6ba1f75f8..000000000 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/CameraViewToSpatialCameraViewAdapter.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace Microsoft.Psi.Visualization.Adapters -{ - using MathNet.Spatial.Euclidean; - using Microsoft.Psi.Calibration; - using Microsoft.Psi.Imaging; - using Microsoft.Psi.Visualization.Data; - - /// - /// Implements a stream adapter from camera view to spatial camera view with default position. - /// - [StreamAdapter] - public class CameraViewToSpatialCameraViewAdapter : StreamAdapter<(Shared, ICameraIntrinsics), (Shared, ICameraIntrinsics, CoordinateSystem)> - { - /// - public override (Shared, ICameraIntrinsics, CoordinateSystem) GetAdaptedValue((Shared, ICameraIntrinsics) source, Envelope envelope) - => (source.Item1, source.Item2, new CoordinateSystem()); - } -} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/DepthCameraViewToSpatialDepthCameraViewAdapter.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/DepthCameraViewToSpatialDepthCameraViewAdapter.cs deleted file mode 100644 index 1130fa532..000000000 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/DepthCameraViewToSpatialDepthCameraViewAdapter.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace Microsoft.Psi.Visualization.Adapters -{ - using MathNet.Spatial.Euclidean; - using Microsoft.Psi.Calibration; - using Microsoft.Psi.Imaging; - using Microsoft.Psi.Visualization.Data; - - /// - /// Implements a stream adapter from depth camera view to spatial depth camera view with default position. - /// - [StreamAdapter] - public class DepthCameraViewToSpatialDepthCameraViewAdapter : StreamAdapter<(Shared, ICameraIntrinsics), (Shared, ICameraIntrinsics, CoordinateSystem)> - { - /// - public override (Shared, ICameraIntrinsics, CoordinateSystem) GetAdaptedValue((Shared, ICameraIntrinsics) source, Envelope envelope) - => (source.Item1, source.Item2, new CoordinateSystem()); - } -} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedCameraViewToSpatialCameraViewAdapter.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedCameraViewToSpatialCameraViewAdapter.cs deleted file mode 100644 index 37b776500..000000000 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedCameraViewToSpatialCameraViewAdapter.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace Microsoft.Psi.Visualization.Adapters -{ - using MathNet.Spatial.Euclidean; - using Microsoft.Psi.Calibration; - using Microsoft.Psi.Imaging; - using Microsoft.Psi.Visualization.Data; - - /// - /// Implements a stream adapter from camera view (shared encoded image with intrinsics) to spatial camera view (shared image with intrinsics) with default position. - /// - [StreamAdapter] - public class EncodedCameraViewToSpatialCameraViewAdapter : StreamAdapter<(Shared, ICameraIntrinsics), (Shared, ICameraIntrinsics, CoordinateSystem)> - { - private readonly EncodedImageToImageAdapter imageAdapter = new EncodedImageToImageAdapter(); - - /// - public override (Shared, ICameraIntrinsics, CoordinateSystem) GetAdaptedValue((Shared, ICameraIntrinsics) source, Envelope envelope) - => (this.imageAdapter.GetAdaptedValue(source.Item1, envelope), source.Item2, new CoordinateSystem()); - - /// - public override void Dispose((Shared, ICameraIntrinsics, CoordinateSystem) destination) - => this.imageAdapter.Dispose(destination.Item1); - } -} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthCameraViewToSpatialDepthCameraViewAdapter.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthCameraViewToSpatialDepthCameraViewAdapter.cs deleted file mode 100644 index 58ac7aa59..000000000 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthCameraViewToSpatialDepthCameraViewAdapter.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace Microsoft.Psi.Visualization.Adapters -{ - using MathNet.Spatial.Euclidean; - using Microsoft.Psi.Calibration; - using Microsoft.Psi.Imaging; - using Microsoft.Psi.Visualization.Data; - - /// - /// Implements a stream adapter from depth camera view (shared encoded depth image with intrinsics) to spatial depth camera view (shared depth image with intrinsics) with default position. - /// - [StreamAdapter] - public class EncodedDepthCameraViewToSpatialDepthCameraViewAdapter : StreamAdapter<(Shared, ICameraIntrinsics), (Shared, ICameraIntrinsics, CoordinateSystem)> - { - private readonly EncodedDepthImageToDepthImageAdapter depthImageAdapter = new EncodedDepthImageToDepthImageAdapter(); - - /// - public override (Shared, ICameraIntrinsics, CoordinateSystem) GetAdaptedValue((Shared, ICameraIntrinsics) source, Envelope envelope) - => (this.depthImageAdapter.GetAdaptedValue(source.Item1, envelope), source.Item2, new CoordinateSystem()); - - /// - public override void Dispose((Shared, ICameraIntrinsics, CoordinateSystem) destination) - => this.depthImageAdapter.Dispose(destination.Item1); - } -} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthImageToDepthImageAdapter.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthImageToDepthImageAdapter.cs index 8bfe9f8d5..b6feb90b9 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthImageToDepthImageAdapter.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthImageToDepthImageAdapter.cs @@ -19,7 +19,11 @@ public override Shared GetAdaptedValue(Shared sou if (source != null && source.Resource != null) { - sharedDepthImage = DepthImagePool.GetOrCreate(source.Resource.Width, source.Resource.Height); + sharedDepthImage = DepthImagePool.GetOrCreate( + source.Resource.Width, + source.Resource.Height, + source.Resource.DepthValueSemantics, + source.Resource.DepthValueToMetersScaleFactor); var decoder = new DepthImageFromStreamDecoder(); decoder.DecodeFromStream(source.Resource.ToStream(), sharedDepthImage.Resource); } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthImageToImageAdapter.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthImageToImageAdapter.cs index 05cb65907..f47fbd30c 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthImageToImageAdapter.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedDepthImageToImageAdapter.cs @@ -19,7 +19,11 @@ public override Shared GetAdaptedValue(Shared source, if ((source != null) && (source.Resource != null)) { - using var sharedDepthImage = DepthImagePool.GetOrCreate(source.Resource.Width, source.Resource.Height); + using var sharedDepthImage = DepthImagePool.GetOrCreate( + source.Resource.Width, + source.Resource.Height, + source.Resource.DepthValueSemantics, + source.Resource.DepthValueToMetersScaleFactor); sharedImage = ImagePool.GetOrCreate(source.Resource.Width, source.Resource.Height, PixelFormat.Gray_16bpp); var decoder = new DepthImageFromStreamDecoder(); decoder.DecodeFromStream(source.Resource.ToStream(), sharedDepthImage.Resource); diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialCameraViewToSpatialCameraViewAdapter.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialCameraViewToSpatialCameraViewAdapter.cs deleted file mode 100644 index 6076a0848..000000000 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialCameraViewToSpatialCameraViewAdapter.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace Microsoft.Psi.Visualization.Adapters -{ - using MathNet.Spatial.Euclidean; - using Microsoft.Psi.Calibration; - using Microsoft.Psi.Imaging; - using Microsoft.Psi.Visualization.Data; - - /// - /// Implements a stream adapter from encoded spatial camera view (shared encoded image with intrinsics and position) to spatial camera view (shared image with intrinsics and position). - /// - [StreamAdapter] - public class EncodedSpatialCameraViewToSpatialCameraViewAdapter : StreamAdapter<(Shared, ICameraIntrinsics, CoordinateSystem), (Shared, ICameraIntrinsics, CoordinateSystem)> - { - private readonly EncodedImageToImageAdapter imageAdapter = new EncodedImageToImageAdapter(); - - /// - public override (Shared, ICameraIntrinsics, CoordinateSystem) GetAdaptedValue((Shared, ICameraIntrinsics, CoordinateSystem) source, Envelope envelope) - => (this.imageAdapter.GetAdaptedValue(source.Item1, envelope), source.Item2, source.Item3); - - /// - public override void Dispose((Shared, ICameraIntrinsics, CoordinateSystem) destination) - => this.imageAdapter.Dispose(destination.Item1); - } -} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialDepthCameraViewToSpatialDepthCameraViewAdapter.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialDepthCameraViewToSpatialDepthCameraViewAdapter.cs deleted file mode 100644 index a683fb690..000000000 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialDepthCameraViewToSpatialDepthCameraViewAdapter.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace Microsoft.Psi.Visualization.Adapters -{ - using MathNet.Spatial.Euclidean; - using Microsoft.Psi.Calibration; - using Microsoft.Psi.Imaging; - using Microsoft.Psi.Visualization.Data; - - /// - /// Implements a stream adapter from encoded spatial depth camera view (shared encoded depth image with intrinsics and position) to spatial depth camera view (shared depth image with intrinsics and position). - /// - [StreamAdapter] - public class EncodedSpatialDepthCameraViewToSpatialDepthCameraViewAdapter : StreamAdapter<(Shared, ICameraIntrinsics, CoordinateSystem), (Shared, ICameraIntrinsics, CoordinateSystem)> - { - private readonly EncodedDepthImageToDepthImageAdapter depthImageAdapter = new EncodedDepthImageToDepthImageAdapter(); - - /// - public override (Shared, ICameraIntrinsics, CoordinateSystem) GetAdaptedValue((Shared, ICameraIntrinsics, CoordinateSystem) source, Envelope envelope) - => (this.depthImageAdapter.GetAdaptedValue(source.Item1, envelope), source.Item2, source.Item3); - - /// - public override void Dispose((Shared, ICameraIntrinsics, CoordinateSystem) destination) - => this.depthImageAdapter.Dispose(destination.Item1); - } -} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialDepthImageToSpatialDepthImageAdapter.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialDepthImageToSpatialDepthImageAdapter.cs deleted file mode 100644 index f0bd2ca3d..000000000 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/EncodedSpatialDepthImageToSpatialDepthImageAdapter.cs +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT license. - -namespace Microsoft.Psi.Visualization.Adapters -{ - using MathNet.Spatial.Euclidean; - using Microsoft.Psi.Imaging; - using Microsoft.Psi.Visualization.Data; - - /// - /// Implements a stream adapter from encoded spatial depth image (shared encoded depth image with position) to spatial depth image (shared depth image with position). - /// - [StreamAdapter] - public class EncodedSpatialDepthImageToSpatialDepthImageAdapter : StreamAdapter<(Shared, CoordinateSystem), (Shared, CoordinateSystem)> - { - private readonly EncodedDepthImageToDepthImageAdapter depthImageAdapter = new EncodedDepthImageToDepthImageAdapter(); - - /// - public override (Shared, CoordinateSystem) GetAdaptedValue((Shared, CoordinateSystem) source, Envelope envelope) - => (this.depthImageAdapter.GetAdaptedValue(source.Item1, envelope), source.Item2); - - /// - public override void Dispose((Shared, CoordinateSystem) destination) - => this.depthImageAdapter.Dispose(destination.Item1); - } -} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/Ray3DListToNullableAdapter.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/Ray3DListToNullableAdapter.cs new file mode 100644 index 000000000..774830733 --- /dev/null +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/Ray3DListToNullableAdapter.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Visualization.Adapters +{ + using System.Collections.Generic; + using System.Linq; + using MathNet.Spatial.Euclidean; + using Microsoft.Psi.Visualization.Data; + + /// + /// Implements a stream adapter from list of to list of nullable . + /// + [StreamAdapter] + public class Ray3DListToNullableAdapter : StreamAdapter, List> + { + /// + public override List GetAdaptedValue(List source, Envelope envelope) + => source?.Select(p => p as Ray3D?).ToList(); + } +} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/StreamAdapterAttribute.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/StreamAdapterAttribute.cs index 3f8492cab..36fd1b45b 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/StreamAdapterAttribute.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Adapters/StreamAdapterAttribute.cs @@ -11,5 +11,26 @@ namespace Microsoft.Psi.Visualization.Adapters [AttributeUsage(AttributeTargets.Class)] public class StreamAdapterAttribute : Attribute { + /// + /// Initializes a new instance of the class. + /// + /// The name of the stream adapter. + public StreamAdapterAttribute(string name) + { + this.Name = name; + } + + /// + /// Initializes a new instance of the class. + /// + public StreamAdapterAttribute() + { + this.Name = null; + } + + /// + /// Gets the name of the stream adapter. + /// + public string Name { get; private set; } } } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Common/ContextMenuName.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Common/ContextMenuName.cs index db8e5da9c..aa504fd0e 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Common/ContextMenuName.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Common/ContextMenuName.cs @@ -13,6 +13,11 @@ internal class ContextMenuName /// public const string Visualize = "Visualize"; + /// + /// Visualize session. + /// + public const string VisualizeSession = "Visualize Session"; + /// /// Visualize as. /// diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Common/IconSourcePath.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Common/IconSourcePath.cs index 22fb7d4b4..9f7148cb8 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Common/IconSourcePath.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Common/IconSourcePath.cs @@ -33,6 +33,11 @@ public static class IconSourcePath /// public const string PartitionAdd = IconPrefix + "partition-add.png"; + /// + /// Add multiple partitions. + /// + public const string PartitionAddMultiple = IconPrefix + "partition-add-multiple.png"; + /// /// Create partition. /// @@ -74,9 +79,19 @@ public static class IconSourcePath public const string SessionCreate = IconPrefix + "session-create.png"; /// - /// Create session from store. + /// Add session from store. + /// + public const string SessionAddFromStore = IconPrefix + "session-from-store.png"; + + /// + /// Add session from folder. + /// + public const string SessionAddFromFolder = IconPrefix + "session-from-folder.png"; + + /// + /// Add multiple sessions from folder. /// - public const string SessionCreateFromStore = IconPrefix + "session-from-store.png"; + public const string MultipleSessionsAddFromFolder = IconPrefix + "multiple-sessions-from-folder.png"; /// /// Close dataset. @@ -258,6 +273,16 @@ public static class IconSourcePath /// public const string ZoomToSelection = IconPrefix + "zoom-to-selection.png"; + /// + /// Move selection left. + /// + public const string MoveSelectionLeft = IconPrefix + "move-selection-left.png"; + + /// + /// Move selection right. + /// + public const string MoveSelectionRight = IconPrefix + "move-selection-right.png"; + /// /// Clear selection. /// @@ -312,5 +337,10 @@ public static class IconSourcePath /// Toggle visibility. /// public const string ToggleVisibility = IconPrefix + "stream-show-hide.png"; + + /// + /// Go to time button. + /// + public const string GoToTime = IconPrefix + "go-to-time.png"; } } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/go-to-time.png b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/go-to-time.png new file mode 100644 index 0000000000000000000000000000000000000000..38c99d70f30d6ced6fca7f481d370f2e16698ddd GIT binary patch literal 1555 zcmbVMZ)h837=M#AS++K9%2>&^LT@+NGHi0Wq-oN_w3jAb6D3=SMdycM-`u@P&gK5> z?%E~8O4^N4zql$W6E^IJfdip*{y`9w$ifJ7SVe^m5!Nq7@k@1;iJR2-+BJ4Erw;DD z_n!CtJ%4`BJ`p~dr+pcW@07o(r&*0r@EV~WAyKb*SyqUDj@F1A`qVq>AH07hI zC;*qwHs6C*EN@j3&uaj*){JF(6SfLD_zwa z(@ivc#`XDf#R|dqU}pd2#Vc&=^yJFf@h+=vyyJ9l_7h+suB5@orrFx0<2&<()sBSr ztW>Tp-(DCXwwtGqo7^35o^r8r)A@;Ctlf4??d0PAL|~@hyhsAtR>?-UB)Edjls&p^ zd5QyBM|Eq$4}RF;Znn8x;M~!{_ON#TT}%!nhYrQ(21lp(91}+xrwWQL$P%Gq8D`}% zJwlR1Ni0elQxcU3S6e=PS zNxFuybOWmtWL?$@@;{hvTtDCdN0Lf4d^|K3QEa%-^r3M~<57`oBV`3q8KlW$Du;%~ zahQ7zHVP9{5!7XMM3##YGHOsJ(iCoypStD4BF#kg8m1fNQXSs;lF};j>*ZY2r+9xgn2|MQl2Cl zxnyE;QC6{I99>}Zc`_DDt8z{#;tQiqh~!YLm+tGu!F%boa8oHJDQP;CIFyV>NSq-=6!3&YJm=}5eOV7a@_Twv*2mK!#D(bWhJIY;#*8}H(C7cB{$W+f3lc0o zww-?B$+ZGv5`u<*Z5hi(wLgQ7-~TEK#9GZTkTZHALUKkCAfCK`EIf7w*3F(MAZ+vz zb+Hc9;WgJpLAUkS&MVwom$0AyxbN{Om|LLyw%@bwC{9Qh63(U>-4XKmJIvp zaBDj2U|SP2$9{jU+$zY;Lh&E%!hxhe6V?}{RuYm`^NwP literal 0 HcmV?d00001 diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/move-selection-left.png b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/move-selection-left.png new file mode 100644 index 0000000000000000000000000000000000000000..0f9e2cf3f41b09ba3c4643156954d4a66cd63e3d GIT binary patch literal 1573 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`jKx9jP7LeL$-D$|SkfJR9T^xl z_H+M9WCij$3p^r=85sDEfH31!Z9ZwB;4)7a#}EturIQW{wis}@o()Uj{3f95d_L-3 zm0#8kwQI&q&FkEn3csEGqp0>tu!p(fm3*8~c~VcWPWKK81^G_bIXoMFJ+AvEIYp)V z{al^;2?v)6Fiq&Io?Z0&4pZ2A%~hiBF2tD5j9*}QX0qkvrLTY3WKWos{r+0RKR+$Y zBp>FQneEw&c-yY*cjx&f`SSKQi5|sihs-Zp{%wj3;Cp`RbL$lceWt5i=TlC1FU!9a z3Uo1pr>mdKI;VstvvWtp>{m%Sugq(bvP+uyyY!{o{@ch3Ds=OTm6hxiY>HCStXxuy zlZ!G7N;32F6hQpMr2NtnTO}ohirk#MVylYW0$Zhwl9B=|ef_**y~LFKq*T4+{9OHt z!~%UoJp+B1!jxpVx`NW89KHOabp4cM{nVV)+|<01VxU?>xY~k@{F40QjC>qM!=+Iy z0J_B27j6eux8&x+ttc*WEdzQ3pUaCwDhpEegHnt0ON)|IUCV&ZR0jE`z{3@sHjd=ry1^FaR4MADV&l30>zs{~S^Yhb2pWD#Ou zX$3^a7Pq#K*#GDf}QK@ zYvq|&T#}fVoa*Ufs{{-oz0AxMD^mlbL<RwL~E`FU3}=NXg!A)AbXV zfMv&OaB+Bg?c|MuhZT5SB^w1L4z{u4crQhq8SYtX0hH6H5y#a_ct=7oL!X78{zgS#jnfHcGhfF{(!GjDhdv-s9qJG zl<|ywCs|Rc3 nWfP9tSFz9d|6#gVz#hr@Y&}W04uu{N?@e&X($H)+LN4+EU49vc literal 0 HcmV?d00001 diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/move-selection-right.png b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/move-selection-right.png new file mode 100644 index 0000000000000000000000000000000000000000..5274b9d197e6c21f149d64230abadd2938886c98 GIT binary patch literal 1573 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`jKx9jP7LeL$-D$|SkfJR9T^xl z_H+M9WCij$3p^r=85sDEfH31!Z9ZwB-~mq;#}Etuxsy)w9x~u*lRd)FbAZDm^Z0{v zR(B%1sy8u9A84G{`~OPF!ouKL{#|dbd{y2iYaFb7&hpmEduNt07#>@6=d7qg$DGA+ zD_=B9@GH%GoGbZ$@0J6CiU61{2&3v8!9WE|q!wvu>ta^4#+4of~ppStI(k zC}{ur)9kU6zoprj!2x9UcGw7!L~oO zlaR%WG7CyF^YavnQqrsvlk!VTY?YK0Dspr3imfVg3v87#N=gc>^!4+K^%7I^lT!7P z^KIzDWa`f_x()Cl4^;2_Fb5rw5ih*hk;c5#q@=NlIGxBj5 z4VOl-0O%55U$`Au-IALNx1zYjwG8MDd@e5zsVqp<4@xc0FD*(=bu9xrQyJu&0xRdD z)WnkfqLBRj99tzvke3w#U`|$Wc2+PlFtk+A@J&q4%mevD6G>O9OJYf?trAFuu7R1Z zkwu7sr450*V`OnpXXk*T{Itv*padwJD0sR$n;BY~0UfVr z2zIWoua#$BaYzJ=>Vobg+%@SMVaZDd5Jk>n`5V715Q9z9+}0!sI3I@L8(1B zwGhh6$y5NR8LPx(P+qWAN(Lo@)D$JCa3Uo^%0GR-B(!Ql40p>X;4; zE#eZNKF|mH2+!!F`x7kZl3JV$&E_!WK`w3}W(6pZ=K@n1kYis_keHmETB4Aemtw0_ zq-1Zm>H3LFz@lO`xHvq$cJfBS!wNjEl8u5A2V2<_8t?4%Ihl55(!aM81T!va_ny$S zODs-*?Z^Ab{)bb`25tpL(Ts)+vsmwk8V#=I`x}{3&MwO4jc|LE;@9P8J8QNof56u% z6$OV^RIdupao|*6ST@UDOu?VQNZ~t^Ui6ODowIMPU{Crld6vQR`}Yc4SJ%Uizt*gN vbCman)q^$ivI$4+tJr7!|1e!FV2|W{ww|P0he8jC_a-=GX=pYZA(wCfW=kMq literal 0 HcmV?d00001 diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/multiple-sessions-from-folder.png b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/multiple-sessions-from-folder.png new file mode 100644 index 0000000000000000000000000000000000000000..603d5b0c974ad711930434f0f77e3d387a86e2a8 GIT binary patch literal 1469 zcmbVMU1$_n6rOCF5K|N5A5BuB<3uq6A3u7et+l(AuTp6nS_%vgk3Q`bx?fzm zee>7P7UVByf4V*O=haB!k300)qe~@u>+YVG?_bIBdk>m>9u@W8jI1c|3{3yLeR)gE zGtJsn?AyE4xVZHCn;!#b&bD6b=13FZ_6t zv*TM9371S9)mMh5PRk$^;;WBvm_Ov188D#_^94KS5Wu9EFioioBq^WI2lL?|_Og^wccqh!~#bQVTo0ZxN>P42=mw)oj6b>t-DigeFd-pn+|v z09{E5%2Ewz8g-mIVd>OMA%#kT7% z2u;i~8r6hsh$fSBE|Mk?!84wL60!6526H@rE zVAB8@?Aeg9wrrS1lIvszzMe~<;T-muV-{U_?Riqu#yy;|a_quy6F!;HqKX=2!3ULc zxTzFKI)p;UKuJ9&unYw)%LI}BP`EFR6fv!;vKSdiE21j*E23c<@&Hn!Fxqd{^mS~^ z_;pazH~**pm}g}z2j(_yhxJRjfWU}F*sq5W#z690w?amGmuLBIO`F%?YU@ zt1>1G7F=k2g(#04GbVV#;NjQTi#(e_T6r99H>Oqtk;h}D`}1Arn)Y?M<-s?)+|YY+ xxQ%~Ue6#g(Akw&FapCQa2CpgE{N&gnnJr=cR&iS+_jKQ-`7a8;?^rw2{sz3P(0Bj< literal 0 HcmV?d00001 diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/partition-add-multiple.png b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Icons/partition-add-multiple.png new file mode 100644 index 0000000000000000000000000000000000000000..ad067cad9b48c26a48ec795b7fac81fb4432b8ec GIT binary patch literal 1537 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`jKx9jP7LeL$-D$|SkfJR9T^xl z_H+M9WCij$3p^r=85sDEfH31!Z9ZwB;0sR|#}Etuy^{?2nhZqRR;oXc=+pDzn|3gI zhJ>Gk%*-PvPinXud=%q*^hQB>w)=z%#y|D9W@SYARv*$Y7z1P3HCD!TZK3Le; zXMcKk!}foV965yqvS$>pzr3A^yMQga;F7^}Id0Y_{^(oa;y!KVNX* zxjx<9I#>OcPwmUEUMKiXAvAM+PO6aVexTPv5KQ&+=Y)44d{jOOEm zEFRO@Ic*l))=5eEyr8hsfW6hLIy8cN!R|XfthEn9B|n}xvSQU{jrV`~HF{yW)+gCbS^Z4#K*(}|5WeL#F44$rjF6*2Unm|~gA~z?m*s3D8z*Z@vq@=(~Uq7!{ zFEJ%QDOE2yKUcpZu|VHY&p=-ZsxT!PuCAc8C`T{9C|y4#SwA%=H8(Y{q!_5y5U#c$ zBfliSI3pj2(Qs)L3xF>1^@ZDk)h)TXa4U*ST+2%Et1b?yEJ)Q4N-fSWElN&xEdx4J z8RVM+E9aur#FG4?ko^1{TO~)3mlXnFPF8SsRxmO!v{caWO-#N>fV`QxZ!O2`WxW zwo1z{%1s3NIx{yhJypLTFI`Ci9+pn^Dx67n2tbT2&qxItiYHw_ zlojWvm6RtIr8=eqLyNe?rw{ajKEgBl=>7!Dxug~+7iAWJyarPq2%@kG9hw1cpIfU>~)Kj+@> z{CsE5x!(`(+7anq)eQiMXoKnqU6%%9K^L9Zy{<#LguIa*+ri9lYyP6b7sYfg4Z!WQ zi{8RcDlc>f_jv#;eh`e%#jh8yriaHhH9gulg!KF2f9mxm_=j*dyYt0ZCbAH@l`grjNckdLew`yY8vOp*6L9=-`=GW3}I-;K_1RTX~}}RSETuj{|V1x_2^r z>+S&>4$ua6sWaQhk1@@U-O6ln^@2R2W|@j*+MY}$g)RBMBlCQ@T#l6!G2#|^QIaHH zi1YC{M-iM?wS8FOY;S$bf{H!l8jf!en+Yt!JelwnmYp?Qv7EM9+lxV+NHnD|m-=kQujde=ICtits#B}@mJqI+BT(0e7-dHTF?ZWd1 zCMk{Cgv^WfMyn3yN3cgGT!aTEX_)H+HjbQeG4zQ$N{Fej>3}jb+zOe=WSF=hN=!5d zkzofsHqFt(D)h0!1_(|_aq+%UL6XIUoTTDrSrD4g9MO$Jbv|fPloR56Q0zk;`tV=D zI+6>-wIF3}Sg?qB$1bu=JC|~Xn8c-yX>^IX=UOH+?2>|E(u<%?7;PYv5~UOkJ|;H8 z&E;gx_Izj~tf>l1GZZrnT~7AK6Pxo`Y$~s|4;oP*C-k`G(X!8?UzOY$%BSRe{FTPe|F=;vpdsZ8Z{$B6!e0^qM!&cWzak} zK05)io{hUXZQg>@w@(Qb71m|@x4yYuW9ajfX*`Etq4E3SEn4fEZe|bHe>{vE^>Fu46+o^Ru-MvMiQ$oF0ThakuY`A=Kf92t_xzE`DSry^7od1`x2ZuP8`N&Q2{+NJ>r5 z%(GQ`zk9!uLS~AsQn;zFfp39xYDT62(s|s5su(?)1Hb_`sNdc^+B->U=*AZl+LWFOCf^&XRs)DJWnQpS7iK&9QrJkXg Yv5BRnj)IYap@qJIg}$Nj#wRyf0KMEh)&Kwi diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Microsoft.Psi.Visualization.Windows.csproj b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Microsoft.Psi.Visualization.Windows.csproj index d48389020..604c95292 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Microsoft.Psi.Visualization.Windows.csproj +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Microsoft.Psi.Visualization.Windows.csproj @@ -41,6 +41,7 @@ + @@ -54,8 +55,11 @@ + + + @@ -84,6 +88,7 @@ + @@ -121,6 +126,7 @@ + @@ -352,7 +358,9 @@ + + @@ -444,6 +452,8 @@ + + @@ -481,6 +491,8 @@ + + diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Navigation/Navigator.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Navigation/Navigator.cs index 58d52b44d..a78b972ed 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Navigation/Navigator.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Navigation/Navigator.cs @@ -71,7 +71,7 @@ public partial class Navigator : ObservableObject // True if the timeline cursor should follow the mouse cursor when in manual navigation mode, otherwise false private bool cursorFollowsMouse = true; - private RelayCommand copyCursorTimeToClipboardCommand; + private RelayCommand copyToClipboardCommand; /// /// Initializes a new instance of the class. @@ -101,12 +101,12 @@ public Navigator() public event NavigatorTimeChangedHandler CursorChanged; /// - /// Gets the command for copying the cursor time to clipboard. + /// Gets the command for copying a string to clipboard. /// [Browsable(false)] [IgnoreDataMember] - public RelayCommand CopyCursorTimeToClipboardCommand - => this.copyCursorTimeToClipboardCommand ??= new RelayCommand(cursor => Clipboard.SetText(cursor.ToString("M/d/yyyy HH:mm:ss.ffff"))); + public RelayCommand CopyToClipboardCommand + => this.copyToClipboardCommand ??= new RelayCommand(text => Clipboard.SetText(text)); /// /// Gets or the cursor mode. @@ -628,6 +628,77 @@ public void ZoomToSelection() this.viewRange.Set(this.selectionRange.StartTime - padding, this.selectionRange.EndTime + padding); } + /// + /// Move selection left. + /// + public void MoveSelectionLeft() + { + var selectionDuration = this.selectionRange.Duration; + var selectionStartTime = new DateTime(Math.Max((this.selectionRange.StartTime - selectionDuration).Ticks, this.dataRange.StartTime.Ticks)); + var selectionEndTime = selectionStartTime + selectionDuration; + this.selectionRange.Set(selectionStartTime, selectionEndTime); + + this.ShiftViewRangeToAccomodateSelection(); + } + + /// + /// Gets a value indicating whether the selection can be moved left. + /// + /// True if the selection can be moved left. + public bool CanMoveSelectionLeft() + => this.CursorMode != CursorMode.Live && this.SelectionRange.IsFinite && this.SelectionRange.StartTime > this.DataRange.StartTime; + + /// + /// Move selection right. + /// + public void MoveSelectionRight() + { + var selectionDuration = this.selectionRange.Duration; + var selectionEndTime = new DateTime(Math.Min((this.selectionRange.EndTime + selectionDuration).Ticks, this.dataRange.EndTime.Ticks)); + var selectionStartTime = selectionEndTime - selectionDuration; + this.selectionRange.Set(selectionStartTime, selectionEndTime); + + this.ShiftViewRangeToAccomodateSelection(); + } + + /// + /// Gets a value indicating whether the selection can be moved right. + /// + /// True if the selection can be moved right. + public bool CanMoveSelectionRight() + => this.CursorMode != CursorMode.Live && this.SelectionRange.IsFinite && this.SelectionRange.EndTime < this.DataRange.EndTime; + + /// + /// Shift the view range to capture the selection. + /// + public void ShiftViewRangeToAccomodateSelection() + { + var viewRangeDuration = this.viewRange.Duration; + + // If the view range is shorter than the selection + if (viewRangeDuration <= this.selectionRange.Duration) + { + // adjust the desired view range duration to be more than the selection range + viewRangeDuration = TimeSpan.FromTicks((int)(this.selectionRange.Duration.Ticks * 1.1)); + } + + // If the selection range is after the view range + if (this.selectionRange.EndTime > this.viewRange.EndTime) + { + var padding = TimeSpan.FromTicks((int)(viewRangeDuration.Ticks * 0.05)); + var viewRangeEnd = new DateTime(Math.Min((this.selectionRange.StartTime - padding + viewRangeDuration).Ticks, this.dataRange.EndTime.Ticks)); + var viewRangeStart = viewRangeEnd - viewRangeDuration; + this.viewRange.Set(viewRangeStart, viewRangeEnd); + } + else if (this.selectionRange.StartTime < this.viewRange.StartTime) + { + var padding = TimeSpan.FromTicks((int)(viewRangeDuration.Ticks * 0.05)); + var viewRangeStart = new DateTime(Math.Max((this.selectionRange.EndTime + padding - viewRangeDuration).Ticks, this.dataRange.StartTime.Ticks)); + var viewRangeEnd = viewRangeStart + viewRangeDuration; + this.viewRange.Set(viewRangeStart, viewRangeEnd); + } + } + /// /// Gets a value indicating whether the navigator can zoom to selection. /// diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/DatasetViewModel.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/DatasetViewModel.cs index cda051ac8..19a30c03e 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/DatasetViewModel.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/DatasetViewModel.cs @@ -20,6 +20,7 @@ namespace Microsoft.Psi.Visualization.ViewModels using Microsoft.Psi.Visualization.Helpers; using Microsoft.Psi.Visualization.Navigation; using Microsoft.Psi.Visualization.VisualizationObjects; + using Microsoft.Psi.Visualization.Windows; /// /// Defines types of auxiliary dataset information to display. @@ -320,7 +321,9 @@ public class DatasetViewModel : ObservableTreeNodeObject private string auxiliaryInfo = string.Empty; private RelayCommand createSessionCommand; - private RelayCommand createSessionFromStoreCommand; + private RelayCommand addSessionFromStoreCommand; + private RelayCommand addSessionFromFolderCommand; + private RelayCommand addMultipleSessionsFromFolderCommand; private RelayCommand contextMenuOpeningCommand; /// @@ -392,7 +395,7 @@ public DatasetViewModel() public TimeSpan TotalDuration => TimeSpan.FromTicks(this.sessionViewModels.Sum(svm => svm.OriginatingTimeInterval.Span.Ticks)); /// - /// Gets the create session command. + /// Gets the command to create a session. /// [Browsable(false)] [IgnoreDataMember] @@ -400,12 +403,28 @@ public DatasetViewModel() this.createSessionCommand ??= new RelayCommand(() => this.CreateSession()); /// - /// Gets the create session command. + /// Gets the command to add a session from a store. /// [Browsable(false)] [IgnoreDataMember] - public RelayCommand CreateSessionFromStoreCommand => - this.createSessionFromStoreCommand ??= new RelayCommand(() => this.CreateSessionFromStore()); + public RelayCommand AddSessionFromStoreCommand => + this.addSessionFromStoreCommand ??= new RelayCommand(() => this.AddSessionFromStore()); + + /// + /// Gets the command to add a session from a specified folder. + /// + [Browsable(false)] + [IgnoreDataMember] + public RelayCommand AddSessionFromFolderCommand => + this.addSessionFromFolderCommand ??= new RelayCommand(() => this.AddSessionFromFolder()); + + /// + /// Gets the command to add multiple sessions from a specified folder. + /// + [Browsable(false)] + [IgnoreDataMember] + public RelayCommand AddMultipleSessionsFromFolderCommand => + this.addMultipleSessionsFromFolderCommand ??= new RelayCommand(() => this.AddMultipleSessionsFromFolder()); /// /// Gets the command that executes when opening the dataset context menu. @@ -637,9 +656,10 @@ public void CreateSession() } /// - /// Creates a new session from a store. + /// Adds a new session from an existing store. /// - public void CreateSessionFromStore() + /// The session view model for the newly added session. + public SessionViewModel AddSessionFromStore() { var formats = VisualizationContext.Instance.PluginMap.GetStreamReaderExtensions(); var openFileDialog = new Win32.OpenFileDialog @@ -652,10 +672,24 @@ public void CreateSessionFromStore() if (result == true) { var fileInfo = new FileInfo(openFileDialog.FileName); - var name = fileInfo.Name.Split('.')[0]; + var sessionName = fileInfo.Directory.Name; + var storeName = fileInfo.Name.Split('.')[0]; + var storePath = fileInfo.DirectoryName; var readerType = VisualizationContext.Instance.PluginMap.GetStreamReaderType(fileInfo.Extension); - var streamReader = Psi.Data.StreamReader.Create(name, fileInfo.DirectoryName, readerType); - this.CreateSessionFromStore(name, streamReader); + var streamReader = Psi.Data.StreamReader.Create(storeName, storePath, readerType); + var sessionViewModel = this.AddSessionFromStore(sessionName, streamReader); + + // if this is the only session, set it to visualize + if (this.dataset.Sessions.Count == 1) + { + this.VisualizeSession(sessionViewModel); + } + + return sessionViewModel; + } + else + { + return default; } } @@ -665,10 +699,145 @@ public void CreateSessionFromStore() /// The name of the session. /// The stream reader for the data store. /// The partition name. - public void CreateSessionFromStore(string sessionName, IStreamReader streamReader, string partitionName = null) + /// The session view model for the newly added session. + public SessionViewModel AddSessionFromStore(string sessionName, IStreamReader streamReader, string partitionName = null) { sessionName = this.EnsureUniqueSessionName(sessionName); - this.AddSession(this.dataset.AddSessionFromStore(streamReader, sessionName, partitionName)); + return this.AddSession(this.dataset.AddSessionFromStore(streamReader, sessionName, partitionName)); + } + + /// + /// Adds a new session from a folder. + /// + /// The session view model for the newly added session. + public SessionViewModel AddSessionFromFolder() + { + var selectFolderDialog = new System.Windows.Forms.FolderBrowserDialog + { + Description = "Select a folder containing the partitions to be added as a new session.", + ShowNewFolderButton = false, + }; + + // Get the dataset directory name to prepopulate the folder select path + var datasetDirectoryName = string.IsNullOrEmpty(this.dataset.Filename) ? default : new FileInfo(this.dataset.Filename).DirectoryName; + if (datasetDirectoryName != default) + { + selectFolderDialog.SelectedPath = datasetDirectoryName; + } + + var result = selectFolderDialog.ShowDialog(); + if (result == System.Windows.Forms.DialogResult.OK) + { + // get the folder + var selectedFolder = selectFolderDialog.SelectedPath; + + // set the session name to be the directory name + var sessionName = new FileInfo(selectedFolder).Name; + var session = this.dataset.AddSession(sessionName); + var sessionViewModel = this.AddSession(session); + sessionViewModel.AddMultiplePartitionsFromFolder(selectedFolder, out var _); + + // if this is the only session, set it to visualize + if (this.dataset.Sessions.Count == 1) + { + this.VisualizeSession(sessionViewModel); + } + + return sessionViewModel; + } + else + { + return default; + } + } + + /// + /// Adds multiple sessions from a folder. + /// + public void AddMultipleSessionsFromFolder() + { + var selectFolderDialog = new System.Windows.Forms.FolderBrowserDialog + { + Description = "Select a folder containing the sessions to be added to the dataset.", + ShowNewFolderButton = false, + }; + + // Get the dataset directory name to prepopulate the folder select path + var datasetDirectoryName = string.IsNullOrEmpty(this.dataset.Filename) ? default : new FileInfo(this.dataset.Filename).DirectoryName; + if (datasetDirectoryName != default) + { + selectFolderDialog.SelectedPath = datasetDirectoryName; + } + + var result = selectFolderDialog.ShowDialog(); + if (result == System.Windows.Forms.DialogResult.OK) + { + this.AddMultipleSessionsFromFolder(selectFolderDialog.SelectedPath, out var existingSessions, out var existingPartitions); + + if (existingSessions.Count > 0 || existingPartitions.Count > 0) + { + var message = default(string); + if (existingSessions.Count > 0) + { + if (existingPartitions.Count > 0) + { + message = $"{existingSessions.Count} session(s) and {existingPartitions.Count} partition(s) already existed in the dataset and were not added."; + } + else + { + message = $"{existingSessions.Count} session(s) already existed in the dataset and were not added."; + } + } + else if (existingPartitions.Count > 0) + { + message = $"{existingPartitions} partition(s) already existed in the dataset and were not added."; + } + + // Inform the user of partitions that were already present in the session + new MessageBoxWindow(Application.Current.MainWindow, "Existing Sessions or Partitions", message, "Close", null) + .ShowDialog(); + } + } + } + + /// + /// Adds multiple sessions from a specified folder to the dataset. + /// + /// The folder to add sessions from. + /// A list of sessions that were already existing and were not added. + /// A list of partitions that were already existing and were not added. + public void AddMultipleSessionsFromFolder(string folderName, out List existingSessions, out List existingPartitions) + { + existingSessions = new List(); + existingPartitions = new List(); + + // go through the subdirectories + foreach (var sessionFolder in Directory.GetDirectories(folderName)) + { + // set the session name to be the directory name + var sessionName = new FileInfo(sessionFolder).Name; + + // check if the dataset already contains a session by that name + var existingSession = this.dataset.Sessions.FirstOrDefault(s => s.Name == sessionName); + if (existingSession != null) + { + // add it to the list of existing session + existingSessions.Add(existingSession); + } + else + { + this.AddSession(this.dataset.AddSession(sessionName)); + } + + // get the view model + var sessionViewModel = this.sessionViewModels.First(s => s.Name == sessionName); + + // add partitions from the folder + sessionViewModel.AddMultiplePartitionsFromFolder(sessionFolder, out var existingPartitionsInSession); + + // and add any existing partitions to the range. + existingPartitions.AddRange(existingPartitionsInSession); + } } /// @@ -720,8 +889,12 @@ internal void UpdateLivePartitionStatuses() } } - private void AddSession(Session session) => - this.internalSessionViewModels.Add(new SessionViewModel(this, session)); + private SessionViewModel AddSession(Session session) + { + var sessionViewModel = new SessionViewModel(this, session); + this.internalSessionViewModels.Add(sessionViewModel); + return sessionViewModel; + } private string EnsureUniqueSessionName(string sessionName) { @@ -742,8 +915,26 @@ private ContextMenu CreateContextMenu() { var contextMenu = new ContextMenu(); - contextMenu.Items.Add(MenuItemHelper.CreateMenuItem(IconSourcePath.SessionCreate, "Create Session", this.CreateSessionCommand)); - contextMenu.Items.Add(MenuItemHelper.CreateMenuItem(IconSourcePath.SessionCreateFromStore, "Create Session from Store ...", this.CreateSessionFromStoreCommand)); + contextMenu.Items.Add( + MenuItemHelper.CreateMenuItem( + IconSourcePath.SessionCreate, + "Create Session", + this.CreateSessionCommand)); + contextMenu.Items.Add( + MenuItemHelper.CreateMenuItem( + IconSourcePath.SessionAddFromStore, + "Add Session from Store ...", + this.AddSessionFromStoreCommand)); + contextMenu.Items.Add( + MenuItemHelper.CreateMenuItem( + IconSourcePath.SessionAddFromFolder, + "Add Session from Folder ...", + this.AddSessionFromFolderCommand)); + contextMenu.Items.Add( + MenuItemHelper.CreateMenuItem( + IconSourcePath.MultipleSessionsAddFromFolder, + "Add Multiple Sessions from Folder ...", + this.AddMultipleSessionsFromFolderCommand)); contextMenu.Items.Add(new Separator()); diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/DerivedMemberStreamTreeNode.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/DerivedMemberStreamTreeNode.cs index 35dbfa163..27ec75524 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/DerivedMemberStreamTreeNode.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/DerivedMemberStreamTreeNode.cs @@ -179,8 +179,15 @@ protected override void AddDerivedMemberStreamChildren() /// protected override bool CanExpandDerivedMemberStreams() { + // If we have already expanded this node with derived member streams + if (this.InternalChildren.Any(c => c is DerivedMemberStreamTreeNode)) + { + // Then no longer expand + return false; + } + // Get the node type - Type nodeType = TypeResolutionHelper.GetVerifiedType(this.DataTypeName); + var nodeType = TypeResolutionHelper.GetVerifiedType(this.DataTypeName); // If it's an auto-generated nullable, we need to assess whether the inner value-type (inside the nullable) // can expand the members. diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/PartitionViewModel.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/PartitionViewModel.cs index 816789df5..2e90470bd 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/PartitionViewModel.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/PartitionViewModel.cs @@ -16,6 +16,7 @@ namespace Microsoft.Psi.Visualization.ViewModels using Microsoft.Psi.Common; using Microsoft.Psi.Data; using Microsoft.Psi.Persistence; + using Microsoft.Psi.PsiStudio.Common; using Microsoft.Psi.PsiStudio.TypeSpec; using Microsoft.Psi.Visualization; using Microsoft.Psi.Visualization.Base; @@ -853,6 +854,13 @@ private ContextMenu CreateContextMenu() contextMenu.Items.Add(MenuItemHelper.CreateMenuItem(IconSourcePath.PartitionRemove, "Remove", this.RemovePartitionCommand)); contextMenu.Items.Add(new Separator()); + // Add the visualize session context menu if the partition is not in the currently visualized session + if (!this.SessionViewModel.IsCurrentSession) + { + contextMenu.Items.Add(MenuItemHelper.CreateMenuItem(string.Empty, ContextMenuName.VisualizeSession, this.SessionViewModel.VisualizeSessionCommand)); + contextMenu.Items.Add(new Separator()); + } + // Add show partition info menu var showPartitionInfoMenuItem = MenuItemHelper.CreateMenuItem(string.Empty, "Show Partitions Info", null); foreach (var auxiliaryPartitionInfo in Enum.GetValues(typeof(AuxiliaryPartitionInfo))) diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/SessionViewModel.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/SessionViewModel.cs index 59778601b..5ef4a0105 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/SessionViewModel.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/SessionViewModel.cs @@ -12,13 +12,16 @@ namespace Microsoft.Psi.Visualization.ViewModels using System.Runtime.Serialization; using System.Windows; using System.Windows.Controls; + using System.Windows.Input; using GalaSoft.MvvmLight.CommandWpf; + using Microsoft.Psi; using Microsoft.Psi.Data; using Microsoft.Psi.PsiStudio.Common; using Microsoft.Psi.Visualization; using Microsoft.Psi.Visualization.Base; using Microsoft.Psi.Visualization.Data; using Microsoft.Psi.Visualization.Helpers; + using Microsoft.Psi.Visualization.Windows; /// /// Represents a view model of a session. @@ -34,9 +37,11 @@ public class SessionViewModel : ObservableTreeNodeObject private string auxiliaryInfo = string.Empty; - private RelayCommand addPartitionCommand; + private RelayCommand addPartitionFromStoreCommand; + private RelayCommand addMultiplePartitionsFromFolderCommand; private RelayCommand removeSessionCommand; private RelayCommand visualizeSessionCommand; + private RelayCommand mouseDoubleClickCommand; private RelayCommand contextMenuOpeningCommand; /// @@ -111,13 +116,15 @@ public string AuxiliaryInfo /// Gets the originating time of the first message in the session. /// [Browsable(false)] - public DateTime FirstMessageOriginatingTime => this.partitionViewModels.Min(p => p.FirstMessageOriginatingTime); + public DateTime FirstMessageOriginatingTime + => this.partitionViewModels.Count > 0 ? this.partitionViewModels.Min(p => p.FirstMessageOriginatingTime) : default; /// /// Gets the originating time of the last message in the session. /// [Browsable(false)] - public DateTime LastMessageOriginatingTime => this.partitionViewModels.Max(p => p.LastMessageOriginatingTime); + public DateTime LastMessageOriginatingTime + => this.partitionViewModels.Count > 0 ? this.partitionViewModels.Max(p => p.LastMessageOriginatingTime) : default; /// /// Gets the opacity of UI elements associated with this session. UI element opacity is reduced for sessions that are not the current session. @@ -168,7 +175,14 @@ private set /// [Browsable(false)] [IgnoreDataMember] - public RelayCommand AddPartitionCommand => this.addPartitionCommand ??= new RelayCommand(() => this.AddPartition()); + public RelayCommand AddPartitionFromStoreCommand => this.addPartitionFromStoreCommand ??= new RelayCommand(() => this.AddPartitionFromStore()); + + /// + /// Gets the add multiple partitions command. + /// + [Browsable(false)] + [IgnoreDataMember] + public RelayCommand AddMultiplePartitionsFromFolderCommand => this.addMultiplePartitionsFromFolderCommand ??= new RelayCommand(() => this.AddMultiplePartitionsFromFolder()); /// /// Gets the remove session command. @@ -184,6 +198,14 @@ private set [IgnoreDataMember] public RelayCommand VisualizeSessionCommand => this.visualizeSessionCommand ??= new RelayCommand(() => this.DatasetViewModel.VisualizeSession(this)); + /// + /// Gets the mouse double click command. + /// + [Browsable(false)] + [IgnoreDataMember] + public RelayCommand MouseDoubleClickCommand + => this.mouseDoubleClickCommand ??= new RelayCommand(e => this.OnMouseDoubleClick(e)); + /// /// Gets the command that executes when opening the session context menu. /// @@ -245,7 +267,7 @@ public void AddStorePartition(IStreamReader streamReader, string partitionName = /// /// Adds a new partition to the session. /// - public void AddPartition() + public void AddPartitionFromStore() { var formats = VisualizationContext.Instance.PluginMap.GetStreamReaderExtensions(); var openFileDialog = new Win32.OpenFileDialog @@ -254,6 +276,13 @@ public void AddPartition() Filter = string.Join("|", formats.Select(f => $"{f.Name}|*{f.Extensions}")), }; + // Get the path to the last partition added to prepopulate the dialog + var lastPartition = this.session.Partitions.LastOrDefault(); + if (lastPartition != default) + { + openFileDialog.InitialDirectory = lastPartition.StorePath; + } + bool? result = openFileDialog.ShowDialog(Application.Current.MainWindow); if (result == true) { @@ -276,6 +305,77 @@ public void AddPartition() } } + /// + /// Adds multiple partitions from a folder to the session. + /// + public void AddMultiplePartitionsFromFolder() + { + var selectFolderDialog = new System.Windows.Forms.FolderBrowserDialog + { + Description = "Select a folder containing the partitions to be added to the session.", + ShowNewFolderButton = false, + }; + + // Get the path to the last partition added to prepopulate the dialog + var lastPartition = this.session.Partitions.LastOrDefault(); + if (lastPartition != default) + { + selectFolderDialog.SelectedPath = lastPartition.StorePath; + } + + if (selectFolderDialog.ShowDialog() == System.Windows.Forms.DialogResult.OK) + { + this.AddMultiplePartitionsFromFolder(selectFolderDialog.SelectedPath, out var existingPartitions); + + if (existingPartitions.Count > 0) + { + var message = "The following partitions were already in the session and were not added:\r\n\r\n" + + $"{string.Join("\r\n", existingPartitions.Select(p => Path.Combine(p.StorePath, p.StoreName)))}"; + + // Inform the user of partitions that were already present in the session + new MessageBoxWindow( + Application.Current.MainWindow, + "Existing Partitions", + message, + "Close", + null).ShowDialog(); + } + + // Update stream bindings if this is the current session being visualized + if (this.IsCurrentSession) + { + var visualizationContainer = VisualizationContext.Instance.VisualizationContainer; + var sessionExtents = this.OriginatingTimeInterval; + visualizationContainer.Navigator.DataRange.Set(sessionExtents); + visualizationContainer.UpdateStreamSources(this); + } + } + } + + /// + /// Adds multiple partitions from a specified folder to the session. + /// + /// The folder to add partitions from. + /// A list of partitions that were already existing and were not added. + public void AddMultiplePartitionsFromFolder(string folderName, out List existingPartitions) + { + existingPartitions = new List(); + foreach (var store in PsiStore.EnumerateStores(folderName, false)) + { + // Check if the session already contains a partition for this store + var partition = this.session.Partitions.FirstOrDefault(p => p.StoreName == store.Name && p.StorePath == store.Path); + if (partition != default) + { + existingPartitions.Add(partition); + continue; + } + + // Add the new partition, ensuring that the partition name does not clash with an existing one + partition = this.session.AddPsiStorePartition(store.Name, store.Path, this.EnsureUniquePartitionName(store.Name)); + this.AddPartition(partition); + } + } + /// /// Removes a specified partition from the underlying session. /// @@ -351,9 +451,11 @@ internal void UpdateLivePartitionStatuses() } } - private void AddPartition(IPartition partition) + private PartitionViewModel AddPartition(IPartition partition) { - this.internalPartitionViewModels.Add(new PartitionViewModel(this, partition)); + var partitionViewModel = new PartitionViewModel(this, partition); + this.internalPartitionViewModels.Add(partitionViewModel); + return partitionViewModel; } private string EnsureUniquePartitionName(string partitionName) @@ -383,6 +485,16 @@ private void OnDatasetViewModelPropertyChanged(object sender, PropertyChangedEve } } + private void OnMouseDoubleClick(MouseButtonEventArgs e) + { + if (this.DatasetViewModel.CurrentSessionViewModel != this) + { + this.DatasetViewModel.VisualizeSession(this); + this.IsTreeNodeExpanded = true; + e.Handled = true; + } + } + private void UpdateAuxiliaryInfo() { switch (this.DatasetViewModel.ShowAuxiliarySessionInfo) @@ -436,12 +548,17 @@ private ContextMenu CreateContextMenu() // Create the context menu var contextMenu = new ContextMenu(); - contextMenu.Items.Add(MenuItemHelper.CreateMenuItem(IconSourcePath.PartitionAdd, "Add Partition from Existing Store ...", this.AddPartitionCommand)); + contextMenu.Items.Add(MenuItemHelper.CreateMenuItem(IconSourcePath.PartitionAdd, "Add Partition from Store ...", this.AddPartitionFromStoreCommand)); + contextMenu.Items.Add(MenuItemHelper.CreateMenuItem(IconSourcePath.PartitionAddMultiple, "Add Multiple Partitions from Folder ...", this.AddMultiplePartitionsFromFolderCommand)); contextMenu.Items.Add(MenuItemHelper.CreateMenuItem(IconSourcePath.SessionRemove, "Remove", this.RemoveSessionCommand)); contextMenu.Items.Add(new Separator()); - contextMenu.Items.Add(MenuItemHelper.CreateMenuItem(string.Empty, ContextMenuName.Visualize, this.VisualizeSessionCommand)); - contextMenu.Items.Add(new Separator()); + // Add the visualize session context menu if this is not the currently visualized session + if (!this.IsCurrentSession) + { + contextMenu.Items.Add(MenuItemHelper.CreateMenuItem(string.Empty, ContextMenuName.VisualizeSession, this.VisualizeSessionCommand)); + contextMenu.Items.Add(new Separator()); + } // Add run batch processing task menu var runTasksMenuItem = MenuItemHelper.CreateMenuItem(string.Empty, "Run Batch Processing Task", null); diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/StreamContainerTreeNode.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/StreamContainerTreeNode.cs index 3b4b1efd1..89cadb05f 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/StreamContainerTreeNode.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/StreamContainerTreeNode.cs @@ -10,6 +10,7 @@ namespace Microsoft.Psi.Visualization.ViewModels using System.Diagnostics; using System.Linq; using System.Windows.Controls; + using System.Windows.Input; using System.Windows.Media; using GalaSoft.MvvmLight.CommandWpf; using Microsoft.Psi.Diagnostics; @@ -35,6 +36,7 @@ public class StreamContainerTreeNode : ObservableTreeNodeObject private string auxiliaryInfo = string.Empty; + private RelayCommand mouseDoubleClickCommand; private RelayCommand contextMenuOpeningCommand; /// @@ -105,8 +107,7 @@ public StreamContainerTreeNode(PartitionViewModel partitionViewModel, string pat /// Gets the time interval of the stream(s) subsumed by this stream container tree node. /// [Browsable(false)] - public TimeInterval SubsumedTimeInterval - => new TimeInterval(this.SubsumedOpenedTime, this.SubsumedClosedTime); + public TimeInterval SubsumedTimeInterval => new (this.SubsumedOpenedTime, this.SubsumedClosedTime); /// /// Gets the command that executes when opening the stream tree node context menu. @@ -121,6 +122,13 @@ public TimeInterval SubsumedTimeInterval grid.ContextMenu = contextMenu; }); + /// + /// Gets the command that executes when double clicking on the stream tree node. + /// + [Browsable(false)] + public RelayCommand MouseDoubleClickCommand => + this.mouseDoubleClickCommand ??= new RelayCommand(e => this.OnMouseDoubleClick(e)); + /// /// Gets the name to display in the stream tree. /// @@ -198,7 +206,7 @@ public virtual DateTime SubsumedClosedTime /// [Browsable(false)] public virtual TimeInterval SubsumedOriginatingTimeInterval => - new TimeInterval(this.SubsumedFirstMessageOriginatingTime, this.SubsumedLastMessageOriginatingTime); + new (this.SubsumedFirstMessageOriginatingTime, this.SubsumedLastMessageOriginatingTime); /// /// Gets the originating time of the first message in the stream(s) subsumed by the tree node. @@ -261,7 +269,7 @@ public virtual long SubsumedMessageCount => this.children.Where(c => c is not DerivedStreamTreeNode).Sum(c => c.SubsumedMessageCount); /// - /// Gets the total number of messages in the stream(s) subsumed by the tree node. + /// Gets the average message latency for the stream(s) subsumed by the tree node. /// [DisplayName("Subsumed Avg. Message Latency (ms)")] [Description("The average latency (in milliseconds) of messages in the stream(s) subsumed by the tree node.")] @@ -271,7 +279,7 @@ public virtual long SubsumedMessageCount double.NaN; /// - /// Gets the total number of messages in the stream(s) subsumed by the tree node. + /// Gets the average message size for the stream(s) subsumed by the tree node. /// [DisplayName("Subsumed Avg. Message Size")] [Description("The average size (in bytes) of messages in the stream(s) subsumed by the tree node.")] @@ -440,6 +448,14 @@ protected virtual void OnDatasetViewModelPropertyChanged(object sender, Property } } + /// + /// Handler for a double-click event on the stream container tree node. + /// + /// The mouse button event arguments. + protected virtual void OnMouseDoubleClick(MouseButtonEventArgs e) + { + } + /// /// Updates the auxiliary info to be displayed. /// @@ -491,6 +507,17 @@ protected virtual void UpdateAuxiliaryInfo() /// The context menu to populate. protected virtual void PopulateContextMenu(ContextMenu contextMenu) { + // Add the visualize session context menu if the stream is not in the currently visualized session + if (!this.SessionViewModel.IsCurrentSession) + { + if (contextMenu.Items.Count > 0) + { + contextMenu.Items.Add(new Separator()); + } + + contextMenu.Items.Add(MenuItemHelper.CreateMenuItem(string.Empty, ContextMenuName.VisualizeSession, this.SessionViewModel.VisualizeSessionCommand)); + } + this.PopulateContextMenuWithExpandAndCollapseAll(contextMenu); this.PopulateContextMenuWithShowStreamInfo(contextMenu); } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/StreamTreeNode.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/StreamTreeNode.cs index 76844694c..ba3c44612 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/StreamTreeNode.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/ViewModels/StreamTreeNode.cs @@ -9,6 +9,7 @@ namespace Microsoft.Psi.Visualization.ViewModels using System.Linq; using System.Reflection; using System.Windows.Controls; + using System.Windows.Input; using Microsoft.Psi.Audio; using Microsoft.Psi.Data.Annotations; using Microsoft.Psi.PsiStudio.Common; @@ -611,6 +612,20 @@ protected override void PopulateContextMenu(ContextMenu contextMenu) base.PopulateContextMenu(contextMenu); } + /// + protected override void OnMouseDoubleClick(MouseButtonEventArgs e) + { + base.OnMouseDoubleClick(e); + + if (this.CanExpandDerivedMemberStreams()) + { + this.AddDerivedMemberStreamChildren(); + this.ExpandAll(); + this.IsTreeNodeExpanded = true; + e.Handled = true; + } + } + /// /// Populates a specified context menu with visualizers. /// @@ -711,7 +726,7 @@ protected void PopulateContextMenuWithExpandMembers(ContextMenu contextMenu) this.AddDerivedMemberStreamChildren(); this.ExpandAll(); }), - isEnabled: this.CanExpandDerivedMemberStreams() && !this.InternalChildren.Any(c => c is DerivedMemberStreamTreeNode))); + isEnabled: this.CanExpandDerivedMemberStreams())); } /// @@ -720,7 +735,14 @@ protected void PopulateContextMenuWithExpandMembers(ContextMenu contextMenu) /// True if the stream tree node can expand derived members. protected virtual bool CanExpandDerivedMemberStreams() { - Type nodeType = TypeResolutionHelper.GetVerifiedType(this.DataTypeName); + // If we have already expanded this node with derived member streams + if (this.InternalChildren.Any(c => c is DerivedMemberStreamTreeNode)) + { + // Then no longer expand + return false; + } + + var nodeType = TypeResolutionHelper.GetVerifiedType(this.DataTypeName); if (nodeType != null) { diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/CanvasVisualizationPanelView.xaml b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/CanvasVisualizationPanelView.xaml index edc1983cc..0c20cb3f3 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/CanvasVisualizationPanelView.xaml +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/CanvasVisualizationPanelView.xaml @@ -1,7 +1,7 @@  - - + diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/CanvasVisualizationPanelView.xaml.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/CanvasVisualizationPanelView.xaml.cs index 36fc495ae..137dce2ed 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/CanvasVisualizationPanelView.xaml.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/CanvasVisualizationPanelView.xaml.cs @@ -3,16 +3,12 @@ namespace Microsoft.Psi.Visualization.Views { - using System.Collections.Generic; - using System.Windows.Controls; - using GalaSoft.MvvmLight.CommandWpf; - using Microsoft.Psi.Visualization.Helpers; using Microsoft.Psi.Visualization.VisualizationPanels; /// /// Interaction logic for CanvasVisualizationPanelView.xaml. /// - public partial class CanvasVisualizationPanelView : VisualizationPanelView + public partial class CanvasVisualizationPanelView : InstantVisualizationPanelView { /// /// Initializes a new instance of the class. @@ -26,78 +22,5 @@ public CanvasVisualizationPanelView() /// Gets the visualization panel. /// protected CanvasVisualizationPanel VisualizationPanel => (CanvasVisualizationPanel)this.DataContext; - - /// - public override void AppendContextMenuItems(List menuItems) - { - // Add Set Cursor Epsilon menu with sub-menu items - var setCursorEpsilonMenuItem = MenuItemHelper.CreateMenuItem( - string.Empty, - "Set Cursor Epsilon (on All Visualizers)", - null, - this.VisualizationPanel.VisualizationObjects.Count > 0); - - setCursorEpsilonMenuItem.Items.Add( - MenuItemHelper.CreateMenuItem( - null, - "Infinite Past", - new RelayCommand( - () => - { - foreach (var visualizationObject in this.VisualizationPanel.VisualizationObjects) - { - visualizationObject.CursorEpsilonNegMs = int.MaxValue; - visualizationObject.CursorEpsilonPosMs = 0; - } - }), - true)); - setCursorEpsilonMenuItem.Items.Add( - MenuItemHelper.CreateMenuItem( - null, - "Last 5 seconds", - new RelayCommand( - () => - { - foreach (var visualizationObject in this.VisualizationPanel.VisualizationObjects) - { - visualizationObject.CursorEpsilonNegMs = 5000; - visualizationObject.CursorEpsilonPosMs = 0; - } - }), - true)); - setCursorEpsilonMenuItem.Items.Add( - MenuItemHelper.CreateMenuItem( - null, - "Last 1 second", - new RelayCommand( - () => - { - foreach (var visualizationObject in this.VisualizationPanel.VisualizationObjects) - { - visualizationObject.CursorEpsilonNegMs = 1000; - visualizationObject.CursorEpsilonPosMs = 0; - } - }), - true)); - setCursorEpsilonMenuItem.Items.Add( - MenuItemHelper.CreateMenuItem( - null, - "Last 50 milliseconds", - new RelayCommand( - () => - { - foreach (var visualizationObject in this.VisualizationPanel.VisualizationObjects) - { - visualizationObject.CursorEpsilonNegMs = 50; - visualizationObject.CursorEpsilonPosMs = 0; - } - }), - true)); - - menuItems.Add(setCursorEpsilonMenuItem); - menuItems.Add(null); - - base.AppendContextMenuItems(menuItems); - } } } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/InstantVisualizationPanelView.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/InstantVisualizationPanelView.cs new file mode 100644 index 000000000..9b9a55d07 --- /dev/null +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/InstantVisualizationPanelView.cs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Visualization.Views +{ + using System.Collections.Generic; + using System.Linq; + using System.Windows; + using System.Windows.Controls; + using GalaSoft.MvvmLight.CommandWpf; + using Microsoft.Psi.Visualization.Helpers; + using Microsoft.Psi.Visualization.VisualizationPanels; + using Microsoft.Psi.Visualization.Windows; + + /// + /// Represents the base class for all instant visualization panel views. + /// + public abstract class InstantVisualizationPanelView : VisualizationPanelView + { + /// + public override void AppendContextMenuItems(List menuItems) + { + if (this.DataContext is InstantVisualizationPanel visualizationPanel) + { + // Add Set Cursor Epsilon menu with sub-menu items + var setCursorEpsilonMenuItem = MenuItemHelper.CreateMenuItem( + string.Empty, + "Set Default Cursor Epsilon", + null); + + _ = setCursorEpsilonMenuItem.Items.Add( + MenuItemHelper.CreateMenuItem( + null, + "Infinite Past", + new RelayCommand(() => this.UpdateDefaultCursorEpsilon(visualizationPanel, "Infinite Past", int.MaxValue, 0), true))); + setCursorEpsilonMenuItem.Items.Add( + MenuItemHelper.CreateMenuItem( + null, + "Last 5 seconds", + new RelayCommand(() => this.UpdateDefaultCursorEpsilon(visualizationPanel, "Last 5 seconds", 5000, 0), true))); + setCursorEpsilonMenuItem.Items.Add( + MenuItemHelper.CreateMenuItem( + null, + "Last 1 second", + new RelayCommand(() => this.UpdateDefaultCursorEpsilon(visualizationPanel, "Last 1 second", 1000, 0), true))); + setCursorEpsilonMenuItem.Items.Add( + MenuItemHelper.CreateMenuItem( + null, + "Last 50 milliseconds", + new RelayCommand(() => this.UpdateDefaultCursorEpsilon(visualizationPanel, "Last 50 milliseconds", 50, 0), true))); + + menuItems.Add(setCursorEpsilonMenuItem); + } + + base.AppendContextMenuItems(menuItems); + } + + private void UpdateDefaultCursorEpsilon(InstantVisualizationPanel visualizationPanel, string name, int negMs, int posMs) + { + visualizationPanel.DefaultCursorEpsilonNegMs = negMs; + visualizationPanel.DefaultCursorEpsilonPosMs = posMs; + + var anyVisualizersWithDifferentCursorEpsilon = + visualizationPanel.VisualizationObjects.Any(vo => vo.CursorEpsilonNegMs != 50 || vo.CursorEpsilonPosMs != 0); + + if (anyVisualizersWithDifferentCursorEpsilon) + { + var result = new MessageBoxWindow( + Application.Current.MainWindow, + "Update visualizers?", + $"Some of the visualizers in this panel have a cursor epsilon that is different from {name}. Would you also like to change the cursor epsilon for these visualizers to {name}?", + "Yes", + "No").ShowDialog(); + if (result == true) + { + foreach (var vo in visualizationPanel.VisualizationObjects) + { + vo.CursorEpsilonNegMs = negMs; + vo.CursorEpsilonPosMs = posMs; + } + } + } + } + } +} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/InstantVisualizationPlaceholderPanelView.xaml b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/InstantVisualizationPlaceholderPanelView.xaml index 1b440fb6f..b705ef518 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/InstantVisualizationPlaceholderPanelView.xaml +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/InstantVisualizationPlaceholderPanelView.xaml @@ -1,7 +1,7 @@  - - + diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/InstantVisualizationPlaceholderPanelView.xaml.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/InstantVisualizationPlaceholderPanelView.xaml.cs index 7cd44d638..333e3181d 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/InstantVisualizationPlaceholderPanelView.xaml.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/InstantVisualizationPlaceholderPanelView.xaml.cs @@ -10,7 +10,7 @@ namespace Microsoft.Psi.Visualization.Views /// /// Interaction logic for InstantVisualizationPlaceholderPanelView.xaml. /// - public partial class InstantVisualizationPlaceholderPanelView : VisualizationPanelView + public partial class InstantVisualizationPlaceholderPanelView : InstantVisualizationPanelView { /// /// Initializes a new instance of the class. @@ -24,12 +24,5 @@ public InstantVisualizationPlaceholderPanelView() /// Gets the visualization panel. /// protected InstantVisualizationPlaceholderPanel VisualizationPanel => (InstantVisualizationPlaceholderPanel)this.DataContext; - - /// - public override void AppendContextMenuItems(List menuItems) - { - // Placeholder objects are not deletable nor clearable, so do not call - // the base class to have these menuitems added to the context menu. - } } } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/StreamVisualizationObjectCanvasView.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/StreamVisualizationObjectCanvasView.cs index 7bf822e22..e89c11f42 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/StreamVisualizationObjectCanvasView.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/StreamVisualizationObjectCanvasView.cs @@ -33,7 +33,6 @@ public abstract class StreamVisualizationObjectCanvasView public StreamVisualizationObjectCanvasView() { - this.DataContextChanged += this.OnDataContextChanged; this.SizeChanged += this.OnSizeChanged; this.transformGroup.Children.Add(this.translateTransform); diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/VisualizationPanelView.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/VisualizationPanelView.cs index efbaadf4c..3f9387085 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/VisualizationPanelView.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/VisualizationPanelView.cs @@ -45,14 +45,38 @@ public virtual void AppendContextMenuItems(List menuItems) null, visualizationPanel.VisualizationObjects.Count > 0)); - menuItems.Add( + // Add copy to clipboard menu with sub-menu items + var copyToClipboardMenuItem = MenuItemHelper.CreateMenuItem( + string.Empty, + "Copy to Clipboard", + null); + + copyToClipboardMenuItem.Items.Add( MenuItemHelper.CreateMenuItem( null, - $"Copy Cursor Time to Clipboard", - visualizationPanel.Navigator.CopyCursorTimeToClipboardCommand, + "Cursor Time", + visualizationPanel.Navigator.CopyToClipboardCommand, null, true, - visualizationPanel.Navigator.Cursor)); + visualizationPanel.Navigator.Cursor.ToString("M/d/yyyy HH:mm:ss.ffff"))); + copyToClipboardMenuItem.Items.Add( + MenuItemHelper.CreateMenuItem( + null, + "Session Name & Cursor Time", + visualizationPanel.Navigator.CopyToClipboardCommand, + null, + VisualizationContext.Instance.DatasetViewModel?.CurrentSessionViewModel != null, + VisualizationContext.Instance.DatasetViewModel.CurrentSessionViewModel.Name.ToString() + "@" + visualizationPanel.Navigator.Cursor.ToString("M/d/yyyy HH:mm:ss.ffff"))); + + menuItems.Add(copyToClipboardMenuItem); + + menuItems.Add( + MenuItemHelper.CreateMenuItem( + null, + $"Go To Time ...", + visualizationPanel.Container.GoToTimeCommand, + null, + true)); } } } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/DiagnosticsVisualization/PipelineDiagnosticsVisualizationObjectView.xaml.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/DiagnosticsVisualization/PipelineDiagnosticsVisualizationObjectView.xaml.cs index 2882c5a08..d170347e6 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/DiagnosticsVisualization/PipelineDiagnosticsVisualizationObjectView.xaml.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/DiagnosticsVisualization/PipelineDiagnosticsVisualizationObjectView.xaml.cs @@ -89,6 +89,7 @@ public override void AppendContextMenuItems(List menuItems) PipelineDiagnosticsVisualizationObject.HeatmapStats.AvgMessageProcessTime => "Message Process Time (Average)", PipelineDiagnosticsVisualizationObject.HeatmapStats.TotalMessageEmittedCount => "Total Messages Emitted (Count)", PipelineDiagnosticsVisualizationObject.HeatmapStats.TotalMessageDroppedCount => "Total Messages Dropped (Count)", + PipelineDiagnosticsVisualizationObject.HeatmapStats.TotalMessageDroppedPercentage => "Total Messages Dropped (%)", PipelineDiagnosticsVisualizationObject.HeatmapStats.TotalMessageProcessedCount => "Total Messages Processed (Count)", PipelineDiagnosticsVisualizationObject.HeatmapStats.AvgMessageSize => "Message Size (Average)", _ => throw new NotImplementedException(), @@ -116,6 +117,7 @@ protected override void OnLoaded(object sender, RoutedEventArgs e) this.graphViewer.ObjectUnderMouseCursorChanged += this.GraphViewer_ObjectUnderMouseCursorChanged; this.SizeChanged += this.DiagnosticsVisualizationObjectView_SizeChanged; this.presenter = new PipelineDiagnosticsVisualizationPresenter(this, this.PipelineDiagnosticsVisualizationObject); + this.presenter.UpdateGraph(this.PipelineDiagnosticsVisualizationObject.CurrentData, true); } /// diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/DiagnosticsVisualization/PipelineDiagnosticsVisualizationPresenter.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/DiagnosticsVisualization/PipelineDiagnosticsVisualizationPresenter.cs index bbf765cf1..c68f4a7ad 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/DiagnosticsVisualization/PipelineDiagnosticsVisualizationPresenter.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/Visuals2D/DiagnosticsVisualization/PipelineDiagnosticsVisualizationPresenter.cs @@ -8,6 +8,7 @@ namespace Microsoft.Psi.Visualization.Views.Visuals2D using System.Linq; using System.Text; using Microsoft.Msagl.Drawing; + using Microsoft.Psi.Data; using Microsoft.Psi.Diagnostics; using Microsoft.Psi.PsiStudio.TypeSpec; using Microsoft.Psi.Visualization.VisualizationObjects; @@ -289,6 +290,16 @@ private static Edge GetEdgeById(int id, Graph graph) private static bool IsBridgeToExporter(PipelineDiagnostics.PipelineElementDiagnostics node) { + if (node.TypeName == nameof(PsiExporter)) + { + return true; + } + + if (node.ConnectorBridgeToPipelineElement == null) + { + return false; + } + var bridgeEmitters = node.ConnectorBridgeToPipelineElement.Emitters; var typeName = bridgeEmitters.Length == 1 ? bridgeEmitters[0].PipelineElement.TypeName : string.Empty; return typeName == "MessageConnector`1" || typeName == "MessageEnvelopeConnector`1"; @@ -354,6 +365,8 @@ private void UpdateReceiverDiagnostics(PipelineDiagnostics.ReceiverDiagnostics r PipelineDiagnosticsVisualizationObject.HeatmapStats.TotalMessageEmittedCount => i => i.TotalMessageEmittedCount, PipelineDiagnosticsVisualizationObject.HeatmapStats.TotalMessageProcessedCount => i => i.TotalMessageProcessedCount, PipelineDiagnosticsVisualizationObject.HeatmapStats.TotalMessageDroppedCount => i => i.TotalMessageDroppedCount, + PipelineDiagnosticsVisualizationObject.HeatmapStats.TotalMessageDroppedPercentage => i => + i.TotalMessageEmittedCount > 0 ? 100 * i.TotalMessageDroppedCount / (double)i.TotalMessageEmittedCount : double.NaN, PipelineDiagnosticsVisualizationObject.HeatmapStats.AvgMessageSize => i => { var avg = i.AvgMessageSize; @@ -476,9 +489,10 @@ private Node BuildVisualNode(PipelineDiagnostics.PipelineElementDiagnostics node vis.Label.FontColor = this.LabelColor(fillColor); vis.Attr.Color = fillColor; vis.Attr.FillColor = fillColor; - if (vis.LabelText == "Join" || vis.LabelText == "Fuse") + if ((vis.LabelText.StartsWith("Join(") && vis.LabelText.EndsWith(")")) || + (vis.LabelText.StartsWith("Fuse(") && vis.LabelText.EndsWith(")"))) { - this.SetJoinVisualAttributes(vis, node.Name); + this.SetFuseVisualAttributes(vis, node.DiagnosticState); } return vis; @@ -555,7 +569,7 @@ private void SetConnectorVisualAttributes(Node vis, string label) this.SetVisualAttributes(vis, Shape.Circle, this.ConnectorColor, "☍", label); } - private void SetJoinVisualAttributes(Node vis, string label) + private void SetFuseVisualAttributes(Node vis, string label) { this.SetVisualAttributes(vis, Shape.Circle, this.JoinColor, "+", label); } @@ -755,9 +769,12 @@ private Graph BuildVisualGraph(PipelineDiagnostics diagnostics, Dictionary @@ -118,7 +125,6 @@ public LabeledRectangleListVisualizationObjectCanvasItemView() this.Label.Width = Math.Max(0, rectangle.Width * canvasView.ScaleTransform.ScaleX); this.Label.Height = 30; this.Label.ToolTip = tooltip; - this.Label.Visibility = (label == default) ? Visibility.Collapsed : Visibility.Visible; // update the render transform for the label (this.Label.RenderTransform as TranslateTransform).X = (rectangle.Left + canvasView.TranslateTransform.X) * canvasView.ScaleTransform.ScaleX; diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYVisualizationPanelView.xaml b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYVisualizationPanelView.xaml index 74d38d0ad..90f1ad5c3 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYVisualizationPanelView.xaml +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYVisualizationPanelView.xaml @@ -1,7 +1,7 @@  - - + diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYVisualizationPanelView.xaml.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYVisualizationPanelView.xaml.cs index 9d0171b5a..6c6ffeb26 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYVisualizationPanelView.xaml.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYVisualizationPanelView.xaml.cs @@ -12,7 +12,7 @@ namespace Microsoft.Psi.Visualization.Views /// /// Interaction logic for XYVisualizationPanelView.xaml. /// - public partial class XYVisualizationPanelView : VisualizationPanelView + public partial class XYVisualizationPanelView : InstantVisualizationPanelView { /// /// Initializes a new instance of the class. @@ -37,74 +37,6 @@ public override void AppendContextMenuItems(List menuItems) null, this.VisualizationPanel.AxisComputeMode == AxisComputeMode.Manual)); - // Add Set Cursor Epsilon menu with sub-menu items - menuItems.Add(null); - var setCursorEpsilonMenuItem = MenuItemHelper.CreateMenuItem( - string.Empty, - "Set Cursor Epsilon (on All Visualizers)", - null, - this.VisualizationPanel.VisualizationObjects.Count > 0); - - setCursorEpsilonMenuItem.Items.Add( - MenuItemHelper.CreateMenuItem( - null, - "Infinite Past", - new RelayCommand( - () => - { - foreach (var visualizationObject in this.VisualizationPanel.VisualizationObjects) - { - visualizationObject.CursorEpsilonNegMs = int.MaxValue; - visualizationObject.CursorEpsilonPosMs = 0; - } - }), - true)); - setCursorEpsilonMenuItem.Items.Add( - MenuItemHelper.CreateMenuItem( - null, - "Last 5 seconds", - new RelayCommand( - () => - { - foreach (var visualizationObject in this.VisualizationPanel.VisualizationObjects) - { - visualizationObject.CursorEpsilonNegMs = 5000; - visualizationObject.CursorEpsilonPosMs = 0; - } - }), - true)); - setCursorEpsilonMenuItem.Items.Add( - MenuItemHelper.CreateMenuItem( - null, - "Last 1 second", - new RelayCommand( - () => - { - foreach (var visualizationObject in this.VisualizationPanel.VisualizationObjects) - { - visualizationObject.CursorEpsilonNegMs = 1000; - visualizationObject.CursorEpsilonPosMs = 0; - } - }), - true)); - setCursorEpsilonMenuItem.Items.Add( - MenuItemHelper.CreateMenuItem( - null, - "Last 50 milliseconds", - new RelayCommand( - () => - { - foreach (var visualizationObject in this.VisualizationPanel.VisualizationObjects) - { - visualizationObject.CursorEpsilonNegMs = 50; - visualizationObject.CursorEpsilonPosMs = 0; - } - }), - true)); - - menuItems.Add(setCursorEpsilonMenuItem); - menuItems.Add(null); - base.AppendContextMenuItems(menuItems); } } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYZVisualizationPanelView.xaml b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYZVisualizationPanelView.xaml index f6bc47580..80dbbad21 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYZVisualizationPanelView.xaml +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYZVisualizationPanelView.xaml @@ -1,7 +1,7 @@  - - + diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYZVisualizationPanelView.xaml.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYZVisualizationPanelView.xaml.cs index 2fb4ed663..b59af4adf 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYZVisualizationPanelView.xaml.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Views/XYZVisualizationPanelView.xaml.cs @@ -7,18 +7,15 @@ namespace Microsoft.Psi.Visualization.Views using System.Collections.Generic; using System.Collections.Specialized; using System.Windows; - using System.Windows.Controls; using System.Windows.Media.Animation; using System.Windows.Media.Media3D; - using GalaSoft.MvvmLight.CommandWpf; - using Microsoft.Psi.Visualization.Helpers; using Microsoft.Psi.Visualization.VisualizationObjects; using Microsoft.Psi.Visualization.VisualizationPanels; /// /// Interaction logic for XYZVisualizationPanelView.xaml. /// - public partial class XYZVisualizationPanelView : VisualizationPanelView + public partial class XYZVisualizationPanelView : InstantVisualizationPanelView { private Storyboard cameraStoryboard; @@ -40,79 +37,6 @@ public XYZVisualizationPanelView() /// protected XYZVisualizationPanel VisualizationPanel => this.DataContext as XYZVisualizationPanel; - /// - public override void AppendContextMenuItems(List menuItems) - { - // Add Set Cursor Epsilon menu with sub-menu items - var setCursorEpsilonMenuItem = MenuItemHelper.CreateMenuItem( - string.Empty, - "Set Cursor Epsilon (on All Visualizers)", - null, - this.VisualizationPanel.VisualizationObjects.Count > 0); - - setCursorEpsilonMenuItem.Items.Add( - MenuItemHelper.CreateMenuItem( - null, - "Infinite Past", - new RelayCommand( - () => - { - foreach (var visualizationObject in this.VisualizationPanel.VisualizationObjects) - { - visualizationObject.CursorEpsilonNegMs = int.MaxValue; - visualizationObject.CursorEpsilonPosMs = 0; - } - }), - true)); - setCursorEpsilonMenuItem.Items.Add( - MenuItemHelper.CreateMenuItem( - null, - "Last 5 seconds", - new RelayCommand( - () => - { - foreach (var visualizationObject in this.VisualizationPanel.VisualizationObjects) - { - visualizationObject.CursorEpsilonNegMs = 5000; - visualizationObject.CursorEpsilonPosMs = 0; - } - }), - true)); - setCursorEpsilonMenuItem.Items.Add( - MenuItemHelper.CreateMenuItem( - null, - "Last 1 second", - new RelayCommand( - () => - { - foreach (var visualizationObject in this.VisualizationPanel.VisualizationObjects) - { - visualizationObject.CursorEpsilonNegMs = 1000; - visualizationObject.CursorEpsilonPosMs = 0; - } - }), - true)); - setCursorEpsilonMenuItem.Items.Add( - MenuItemHelper.CreateMenuItem( - null, - "Last 50 milliseconds", - new RelayCommand( - () => - { - foreach (var visualizationObject in this.VisualizationPanel.VisualizationObjects) - { - visualizationObject.CursorEpsilonNegMs = 50; - visualizationObject.CursorEpsilonPosMs = 0; - } - }), - true)); - - menuItems.Add(setCursorEpsilonMenuItem); - menuItems.Add(null); - - base.AppendContextMenuItems(menuItems); - } - private void AddVisualForVisualizationObject(VisualizationObject visualizationObject) { Visual3D visual = ((I3DVisualizationObject)visualizationObject).Visual3D; diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationContext.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationContext.cs index c4040efc2..c594ab1e1 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationContext.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationContext.cs @@ -265,7 +265,7 @@ public void VisualizeStream(StreamTreeNode streamTreeNode, VisualizerMetadata vi } } - // If the target visualization panel is an instant visualization placeholder panel, replace if with a real panel of the correct type + // If the target visualization panel is an instant visualization placeholder panel, replace it with a real panel of the correct type else if ((visualizationPanel is InstantVisualizationPlaceholderPanel placeholderPanel) && (visualizationPanel.ParentPanel is InstantVisualizationContainer instantVisualizationContainer)) { VisualizationPanel replacementPanel = VisualizationPanelFactory.CreateVisualizationPanel(visualizerMetadata.VisualizationPanelType); @@ -356,6 +356,11 @@ public async Task OpenDatasetAsync(string filename, bool showStatusWindow, bool } catch (Exception e) { + // create an empty dataset + this.DatasetViewModels.Clear(); + this.DatasetViewModel = new DatasetViewModel(); + this.DatasetViewModels.Add(this.DatasetViewModel); + // catch and display any exceptions that occurred during the open dataset operation var exception = e.InnerException ?? e; MessageBox.Show(exception.Message, exception.GetType().Name, MessageBoxButton.OK, MessageBoxImage.Error); diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationVisualizationObject.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationVisualizationObject.cs index 12ebc5a99..fb9434035 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationVisualizationObject.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Annotations/TimeIntervalAnnotationVisualizationObject.cs @@ -439,6 +439,12 @@ protected override void OnStreamBound() { base.OnStreamBound(); this.AnnotationSchema = DataManager.Instance.GetSupplementalMetadata(this.StreamSource); + + if (this.AnnotationSchema == null) + { + throw new Exception("Cannot find annotation schema."); + } + this.GenerateLegendValue(); } @@ -611,7 +617,7 @@ private void DeleteAllAnnotationsOnTrack(string track) /// The name of the track. private void RenameTrack(string track) { - var dlg = new GetParameterWindow(Application.Current.MainWindow, "New Track Name", track); + var dlg = new GetParameterWindow(Application.Current.MainWindow, "Rename Track", "New Track Name", track); if (dlg.ShowDialog() == true) { var newTrackName = dlg.ParameterValue; diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/AudioVisualizationObject.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/AudioVisualizationObject.cs index 8813aab4c..13398684f 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/AudioVisualizationObject.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/AudioVisualizationObject.cs @@ -6,7 +6,7 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects using System.ComponentModel; using System.Runtime.Serialization; using System.Windows; - using GalaSoft.MvvmLight.Command; + using GalaSoft.MvvmLight.CommandWpf; using Microsoft.Psi.Visualization; using Microsoft.Psi.Visualization.Helpers; using Microsoft.Psi.Visualization.Summarizers; diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/LabeledRectangleListVisualizationObject.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/LabeledRectangleListVisualizationObject.cs index 10bb5876d..b9f37d08e 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/LabeledRectangleListVisualizationObject.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/LabeledRectangleListVisualizationObject.cs @@ -28,6 +28,11 @@ public class LabeledRectangleListVisualizationObject : XYValueEnumerableVisualiz /// private double lineWidth = 1; + /// + /// Whether to show the label. + /// + private bool showLabel = true; + /// /// Gets or sets the line color. /// @@ -52,6 +57,18 @@ public double LineWidth set { this.Set(nameof(this.LineWidth), ref this.lineWidth, value); } } + /// + /// Gets or sets a value indicating whether to show the label. + /// + [DataMember] + [DisplayName("Show Label")] + [Description("Indicates whether to show the label.")] + public bool ShowLabel + { + get { return this.showLabel; } + set { this.Set(nameof(this.ShowLabel), ref this.showLabel, value); } + } + /// [IgnoreDataMember] public override DataTemplate DefaultViewTemplate => XamlHelper.CreateTemplate(this.GetType(), typeof(LabeledRectangleListVisualizationObjectView)); diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/ModelVisual3DVisualizationObject.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/ModelVisual3DVisualizationObject.cs index 24613350b..f83f1b9ab 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/ModelVisual3DVisualizationObject.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/ModelVisual3DVisualizationObject.cs @@ -287,7 +287,7 @@ protected void UpdateChildVisibility(Visual3D child, bool visible) /// A newly created message using the envelope of the current value. protected Message? SynthesizeMessage(T data) { - if (!this.CurrentValue.HasValue) + if (!this.CurrentValue.HasValue || this.CurrentValue.Value == null) { return null; } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/PipelineDiagnosticsVisualizationObject.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/PipelineDiagnosticsVisualizationObject.cs index 713d52d35..d3bb21bee 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/PipelineDiagnosticsVisualizationObject.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/PipelineDiagnosticsVisualizationObject.cs @@ -87,6 +87,11 @@ public enum HeatmapStats /// TotalMessageDroppedCount, + /// + /// Message dropped percentage heatmap visualization. + /// + TotalMessageDroppedPercentage, + /// /// Message processed count heatmap visualization. /// diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Ray3DListVisualizationObject.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Ray3DListVisualizationObject.cs new file mode 100644 index 000000000..255ce462f --- /dev/null +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/Ray3DListVisualizationObject.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Visualization.VisualizationObjects +{ + using MathNet.Spatial.Euclidean; + + /// + /// Implements a visualization object that can display lists of 3D rays. + /// + [VisualizationObject("3D Rays")] + public class Ray3DListVisualizationObject : ModelVisual3DListVisualizationObject + { + } +} diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/StreamVisualizationObject{TData}.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/StreamVisualizationObject{TData}.cs index 5bd221909..fb54e9894 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/StreamVisualizationObject{TData}.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/StreamVisualizationObject{TData}.cs @@ -7,7 +7,7 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects using System.ComponentModel; using System.Runtime.Serialization; using System.Windows; - using GalaSoft.MvvmLight.Command; + using GalaSoft.MvvmLight.CommandWpf; using Microsoft.Psi; using Microsoft.Psi.PsiStudio.TypeSpec; using Microsoft.Psi.Visualization; diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/UpdatableVisual3DList{TVisual3D}.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/UpdatableVisual3DList{TVisual3D}.cs index 8c3e5ddbf..4104c065d 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/UpdatableVisual3DList{TVisual3D}.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/UpdatableVisual3DList{TVisual3D}.cs @@ -95,7 +95,7 @@ public TVisual3D GetNext() while (this.visuals.Count <= this.currentIndex) { // Create the new visual. - TVisual3D visual = new TVisual3D(); + TVisual3D visual = new (); // Initialize the visual if an initializer method was specified. this.newVisualHandler?.Invoke(visual); diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/VisualizationContainer.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/VisualizationContainer.cs index b81b4e723..ba2737b43 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/VisualizationContainer.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/VisualizationContainer.cs @@ -13,7 +13,7 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects using System.Text; using System.Windows; using System.Windows.Data; - using GalaSoft.MvvmLight.Command; + using GalaSoft.MvvmLight.CommandWpf; using Microsoft.Psi.Data; using Microsoft.Psi.Data.Helpers; using Microsoft.Psi.Visualization.Navigation; @@ -35,6 +35,7 @@ public class VisualizationContainer : ObservableObject // Current Visualization Container version private const double CurrentVisualizationContainerVersion = 5.0d; + private RelayCommand goToTimeCommand; private RelayCommand zoomToSessionExtentsCommand; private RelayCommand zoomToSelectionCommand; private RelayCommand clearSelectionCommand; @@ -86,19 +87,9 @@ public VisualizationContainer() [Browsable(false)] [IgnoreDataMember] public RelayCommand ZoomToSelectionCommand - { - get - { - if (this.zoomToSelectionCommand == null) - { - this.zoomToSelectionCommand = new RelayCommand( - () => this.Navigator.ZoomToSelection(), - () => this.Navigator.CanZoomToSelection()); - } - - return this.zoomToSelectionCommand; - } - } + => this.zoomToSelectionCommand ??= new RelayCommand( + () => this.Navigator.ZoomToSelection(), + () => this.Navigator.CanZoomToSelection()); /// /// Gets the clear selection command. @@ -106,19 +97,9 @@ public RelayCommand ZoomToSelectionCommand [Browsable(false)] [IgnoreDataMember] public RelayCommand ClearSelectionCommand - { - get - { - if (this.clearSelectionCommand == null) - { - this.clearSelectionCommand = new RelayCommand( - () => this.Navigator.ClearSelection(), - () => this.Navigator.CanClearSelection()); - } - - return this.clearSelectionCommand; - } - } + => this.clearSelectionCommand ??= new RelayCommand( + () => this.Navigator.ClearSelection(), + () => this.Navigator.CanClearSelection()); /// /// Gets the zoom to session extents command. @@ -126,19 +107,17 @@ public RelayCommand ClearSelectionCommand [Browsable(false)] [IgnoreDataMember] public RelayCommand ZoomToSessionExtentsCommand - { - get - { - if (this.zoomToSessionExtentsCommand == null) - { - this.zoomToSessionExtentsCommand = new RelayCommand( - () => this.Navigator.ZoomToDataRange(), - () => VisualizationContext.Instance.IsDatasetLoaded() && this.Navigator.CursorMode != CursorMode.Live); - } + => this.zoomToSessionExtentsCommand ??= new RelayCommand( + () => this.Navigator.ZoomToDataRange(), + () => VisualizationContext.Instance.IsDatasetLoaded() && this.Navigator.CursorMode != CursorMode.Live); - return this.zoomToSessionExtentsCommand; - } - } + /// + /// Gets the go to time command. + /// + [Browsable(false)] + [IgnoreDataMember] + public RelayCommand GoToTimeCommand + => this.goToTimeCommand ??= new RelayCommand(() => this.GoToTime()); /// /// Gets or sets the current visualization panel. @@ -518,6 +497,72 @@ public void ZoomToRange(TimeInterval timeInterval) this.Navigator.Cursor = timeInterval.Left; } + /// + /// Goes to a time specified by the user. + /// + public void GoToTime() + { + var getTime = new GetParameterWindow( + Application.Current.MainWindow, + "Go To Time", + "Time", + string.Empty, + value => + { + if (DateTime.TryParse(value, out var dateTime)) + { + if (dateTime >= this.Navigator.DataRange.StartTime && + dateTime <= this.Navigator.DataRange.EndTime) + { + return (true, string.Empty); + } + else + { + return (false, "The specified date-time is outside the range of the current session."); + } + } + else + { + return (false, "Cannot convert the specified time to a valid date time."); + } + }); + + if (getTime.ShowDialog() == true) + { + var cursor = DateTime.Parse(getTime.ParameterValue); + + // If the cursor falls outside the current view range, shift the view range + if (cursor <= this.Navigator.ViewRange.StartTime) + { + var viewDurationTicks = this.Navigator.ViewRange.Duration.Ticks; + var viewStartTime = cursor - TimeSpan.FromTicks(viewDurationTicks / 2); + if (viewStartTime < this.Navigator.DataRange.StartTime) + { + viewStartTime = this.Navigator.DataRange.StartTime; + } + + var viewEndTime = viewStartTime + this.Navigator.ViewRange.Duration; + this.Navigator.ViewRange.Set(viewStartTime, viewEndTime); + } + else if (cursor >= this.Navigator.ViewRange.EndTime) + { + var viewDurationTicks = this.Navigator.ViewRange.Duration.Ticks; + var viewEndTime = cursor + TimeSpan.FromTicks(viewDurationTicks / 2); + if (viewEndTime > this.Navigator.DataRange.EndTime) + { + viewEndTime = this.Navigator.DataRange.EndTime; + } + + var viewStartTime = viewEndTime - this.Navigator.ViewRange.Duration; + this.Navigator.ViewRange.Set(viewStartTime, viewEndTime); + } + + this.Navigator.CursorFollowsMouse = true; + this.Navigator.Cursor = cursor; + this.Navigator.CursorFollowsMouse = false; + } + } + private static bool SeekToLayoutElement(JsonTextReader jsonReader) { // Move the json text reader to the "Layout" node in the document diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/VisualizationObject.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/VisualizationObject.cs index 9a9fefb2e..47a9c7b1b 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/VisualizationObject.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationObjects/VisualizationObject.cs @@ -7,7 +7,7 @@ namespace Microsoft.Psi.Visualization.VisualizationObjects using System.ComponentModel; using System.Runtime.Serialization; using System.Windows; - using GalaSoft.MvvmLight.Command; + using GalaSoft.MvvmLight.CommandWpf; using Microsoft.Psi.PsiStudio.TypeSpec; using Microsoft.Psi.Visualization; using Microsoft.Psi.Visualization.Base; diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/CanvasVisualizationPanel.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/CanvasVisualizationPanel.cs index 4f9e0d771..ddaeb691d 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/CanvasVisualizationPanel.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/CanvasVisualizationPanel.cs @@ -4,20 +4,15 @@ namespace Microsoft.Psi.Visualization.VisualizationPanels { using System.Collections.Generic; - using System.ComponentModel; - using System.Runtime.Serialization; using System.Windows; using Microsoft.Psi.Visualization.Helpers; using Microsoft.Psi.Visualization.Views; - using Xceed.Wpf.Toolkit.PropertyGrid.Attributes; /// - /// Represents a visualization panel that 2D visualizers can be rendered in. + /// Represents a visualization panel that instant visualizers can be rendered in. /// - public class CanvasVisualizationPanel : VisualizationPanel + public class CanvasVisualizationPanel : InstantVisualizationPanel { - private int relativeWidth = 100; - /// /// Initializes a new instance of the class. /// @@ -26,18 +21,6 @@ public CanvasVisualizationPanel() this.Name = "Canvas Panel"; } - /// - /// Gets or sets the name of the relative width for the panel. - /// - [DataMember] - [PropertyOrder(7)] - [Description("The relative width for the panel.")] - public int RelativeWidth - { - get { return this.relativeWidth; } - set { this.Set(nameof(this.RelativeWidth), ref this.relativeWidth, value); } - } - /// public override List CompatiblePanelTypes => new () { VisualizationPanelType.Canvas }; diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationContainer.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationContainer.cs index 98b765df9..a15cc7856 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationContainer.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationContainer.cs @@ -134,6 +134,15 @@ public void ReplaceChildVisualizationPanel(VisualizationPanel oldPanel, Visualiz newPanel.SetParentContainer(this.Container); newPanel.SetParentPanel(this); newPanel.Width = oldPanel.Width; + + // Transfer default properties to the new panel + if (newPanel is InstantVisualizationPanel newInstantPanel && + oldPanel is InstantVisualizationPanel oldInstantPanel) + { + newInstantPanel.DefaultCursorEpsilonNegMs = oldInstantPanel.DefaultCursorEpsilonNegMs; + newInstantPanel.DefaultCursorEpsilonPosMs = oldInstantPanel.DefaultCursorEpsilonPosMs; + } + newPanel.PropertyChanged += this.OnChildVisualizationPanelPropertyChanged; // Replace the panel diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationPanel.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationPanel.cs new file mode 100644 index 000000000..b2d441d98 --- /dev/null +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationPanel.cs @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace Microsoft.Psi.Visualization.VisualizationPanels +{ + using System.ComponentModel; + using System.Runtime.Serialization; + using Microsoft.Psi.Visualization.VisualizationObjects; + using Xceed.Wpf.Toolkit.PropertyGrid.Attributes; + + /// + /// Represents the base class that instant visualization panels derive from. + /// + public abstract class InstantVisualizationPanel : VisualizationPanel + { + private int relativeWidth = 100; + private int defaultCursorEpsilonPosMs = 0; + private int defaultCursorEpsilonNegMs = 500; + + /// + /// Gets or sets the default cursor epsilon for the panel. + /// + [DataMember] + [DisplayName("Default Cursor Epsilon Past (ms)")] + [Description("The default past cursor epsilon for the panel.")] + public int DefaultCursorEpsilonNegMs + { + get { return this.defaultCursorEpsilonNegMs; } + set { this.Set(nameof(this.DefaultCursorEpsilonNegMs), ref this.defaultCursorEpsilonNegMs, value); } + } + + /// + /// Gets or sets the default cursor epsilon for the panel. + /// + [DataMember] + [DisplayName("Default Cursor Epsilon Future (ms)")] + [Description("The default future cursor epsilon for the panel.")] + public int DefaultCursorEpsilonPosMs + { + get { return this.defaultCursorEpsilonPosMs; } + set { this.Set(nameof(this.DefaultCursorEpsilonPosMs), ref this.defaultCursorEpsilonPosMs, value); } + } + + /// + /// Gets or sets the name of the relative width for the panel. + /// + [DataMember] + [DisplayName("Relative Width")] + [Description("The relative width for the panel.")] + public int RelativeWidth + { + get { return this.relativeWidth; } + set { this.Set(nameof(this.RelativeWidth), ref this.relativeWidth, value); } + } + + /// + public override void AddVisualizationObject(VisualizationObject visualizationObject) + { + base.AddVisualizationObject(visualizationObject); + + visualizationObject.CursorEpsilonNegMs = this.defaultCursorEpsilonNegMs; + visualizationObject.CursorEpsilonPosMs = this.defaultCursorEpsilonPosMs; + } + } +} \ No newline at end of file diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationPlaceholderPanel.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationPlaceholderPanel.cs index 4b0594ef0..03ce0c112 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationPlaceholderPanel.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/InstantVisualizationPlaceholderPanel.cs @@ -7,17 +7,16 @@ namespace Microsoft.Psi.Visualization.VisualizationPanels using System.ComponentModel; using System.Runtime.Serialization; using System.Windows; - using GalaSoft.MvvmLight.Command; + using GalaSoft.MvvmLight.CommandWpf; using Microsoft.Psi.Visualization.Helpers; using Microsoft.Psi.Visualization.Views; /// /// Represents a placeholder for another instant visualization panel. /// - public class InstantVisualizationPlaceholderPanel : VisualizationPanel + public class InstantVisualizationPlaceholderPanel : InstantVisualizationPanel { private readonly InstantVisualizationContainer instantVisualizationContainer; - private int relativeWidth = 100; /// /// Initializes a new instance of the class. @@ -36,17 +35,6 @@ public InstantVisualizationPlaceholderPanel(InstantVisualizationContainer instan [IgnoreDataMember] public RelayCommand RemoveCellCommand => new RelayCommand(() => this.instantVisualizationContainer.CreateRemoveCellCommand(this)); - /// - /// Gets or sets the name of the relative width for the panel. - /// - [DataMember] - [Description("The relative width for the panel.")] - public int RelativeWidth - { - get { return this.relativeWidth; } - set { this.Set(nameof(this.RelativeWidth), ref this.relativeWidth, value); } - } - /// public override List CompatiblePanelTypes => new List() { VisualizationPanelType.Canvas, VisualizationPanelType.XY, VisualizationPanelType.XYZ }; diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/XYVisualizationPanel.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/XYVisualizationPanel.cs index 78f48687f..12334289b 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/XYVisualizationPanel.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/XYVisualizationPanel.cs @@ -11,7 +11,6 @@ namespace Microsoft.Psi.Visualization.VisualizationPanels using System.Linq; using System.Runtime.Serialization; using System.Windows; - using System.Windows.Controls; using System.Windows.Input; using GalaSoft.MvvmLight.CommandWpf; using Microsoft.Psi.Visualization.Helpers; @@ -21,7 +20,7 @@ namespace Microsoft.Psi.Visualization.VisualizationPanels /// /// Represents a visualization panel that 2D visualizers can be rendered in. /// - public class XYVisualizationPanel : CanvasVisualizationPanel + public class XYVisualizationPanel : InstantVisualizationPanel { /// /// The scale factor used when zooming into or out of the panel with the mouse wheel. @@ -37,15 +36,13 @@ public class XYVisualizationPanel : CanvasVisualizationPanel private Point mousePosition = new (0, 0); private AxisComputeMode axisComputeMode = AxisComputeMode.Auto; - // The viewport window within which the child visualization objects are displayed. - private Grid viewport = null; + // The current dimensions of the viewport window within which the child visualization objects are displayed. + private double viewportWidth; + private double viewportHeight; // The padding which defines the active display area of the panel. private Thickness viewportPadding; - // The items control to which visualization objects will be added. - private Grid itemsControl = null; - private RelayCommand viewportLoadedCommand; private RelayCommand mouseRightButtonDownCommand; private RelayCommand mouseRightButtonUpCommand; @@ -200,8 +197,12 @@ public RelayCommand ViewportLoadedCommand this.viewportLoadedCommand = new RelayCommand( e => { - this.viewport = e.Source as Grid; - this.itemsControl = (this.viewport.Children[0] as Border).Child as Grid; + // Event source is the viewport + var viewport = e.Source as FrameworkElement; + + // Initialize the display area + this.viewportWidth = viewport.ActualWidth; + this.viewportHeight = viewport.ActualHeight; this.ZoomToDisplayArea(); }); } @@ -226,8 +227,11 @@ public RelayCommand MouseWheelCommand { if (e.Delta != 0) { + // Event source is the items control Grid element + var itemsControl = e.Source as FrameworkElement; + // Get the current mouse location in the items control - Point mouseLocation = Mouse.GetPosition(this.itemsControl); + var mouseLocation = Mouse.GetPosition(itemsControl); // Get the current X Axis and Y Axis logical dimensions double xAxisLogicalWidth = this.XAxis.Maximum - this.XAxis.Minimum; @@ -247,8 +251,8 @@ public RelayCommand MouseWheelCommand // Calculate the new minimum X and Y logical values of the axes such // that the mouse will still be above the same point in the 2D image - double xAxisLogicalMinimum = this.MousePosition.X - mouseLocation.X * xAxisLogicalWidth / this.itemsControl.ActualWidth; - double yAxisLogicalMinimum = this.MousePosition.Y - mouseLocation.Y * yAxisLogicalHeight / this.itemsControl.ActualHeight; + double xAxisLogicalMinimum = this.MousePosition.X - mouseLocation.X * xAxisLogicalWidth / itemsControl.ActualWidth; + double yAxisLogicalMinimum = this.MousePosition.Y - mouseLocation.Y * yAxisLogicalHeight / itemsControl.ActualHeight; // Switch to manual axis compute mode this.AxisComputeMode = AxisComputeMode.Manual; @@ -279,7 +283,7 @@ public RelayCommand MouseRightButtonDownCommand this.mouseRightButtonDownCommand = new RelayCommand( e => { - this.mouseRButtonDownPosition = Mouse.GetPosition(this.itemsControl); + this.mouseRButtonDownPosition = Mouse.GetPosition(e.Source as FrameworkElement); }); } @@ -329,12 +333,15 @@ public RelayCommand MouseMoveCommand this.mouseMoveCommand = new RelayCommand( e => { + // Event source is the items control Grid element + var itemsControl = e.Source as FrameworkElement; + // Get the current mouse position - Point newMousePosition = e.GetPosition(this.itemsControl); + var newMousePosition = e.GetPosition(itemsControl); // Get the current scale factor between the axes logical bounds and the items control size. - double scaleX = (this.XAxis.Maximum - this.XAxis.Minimum) / this.itemsControl.ActualWidth; - double scaleY = (this.YAxis.Maximum - this.YAxis.Minimum) / this.itemsControl.ActualHeight; + double scaleX = (this.XAxis.Maximum - this.XAxis.Minimum) / itemsControl.ActualWidth; + double scaleY = (this.YAxis.Maximum - this.YAxis.Minimum) / itemsControl.ActualHeight; // Set the mouse position in locical/image co-ordinates this.MousePosition = new Point(newMousePosition.X * scaleX + this.XAxis.Minimum, newMousePosition.Y * scaleY + this.YAxis.Minimum); @@ -533,7 +540,11 @@ private void YAxisPropertyBrowser_PropertyChanged(object sender, PropertyChanged } private void OnViewportSizeChanged(SizeChangedEventArgs e) - => this.ZoomToDisplayArea(); + { + this.viewportWidth = e.NewSize.Width; + this.viewportHeight = e.NewSize.Height; + this.ZoomToDisplayArea(); + } private void CalculateAxisRangesAuto() { @@ -572,26 +583,23 @@ private void CalculateAxisRangesAuto() private void ZoomToDisplayArea() { - if (this.viewport != null) - { - // Calculate the aspect ratios of the value ranges and the items control size - double valueRangeAspectRatio = (this.XAxis.Maximum - this.XAxis.Minimum) / (this.YAxis.Maximum - this.YAxis.Minimum); - double viewportAspectRatio = this.viewport.ActualWidth / this.viewport.ActualHeight; + // Calculate the aspect ratios of the value ranges and the items control size + double valueRangeAspectRatio = (this.XAxis.Maximum - this.XAxis.Minimum) / (this.YAxis.Maximum - this.YAxis.Minimum); + double viewportAspectRatio = this.viewportWidth / this.viewportHeight; - if (valueRangeAspectRatio > viewportAspectRatio) - { - // Add padding the the Y axis min/max to maintain the aspect ratio - double scaleFactor = this.viewport.ActualWidth / (this.XAxis.Maximum - this.XAxis.Minimum); - double padding = (this.viewport.ActualHeight - (this.YAxis.Maximum - this.YAxis.Minimum) * scaleFactor) / 2.0d; - this.ViewportPadding = new Thickness(0, padding, 0, padding); - } - else - { - // Add padding the the X axis min/max to maintain the aspect ratio - double scaleFactor = this.viewport.ActualHeight / (this.YAxis.Maximum - this.YAxis.Minimum); - double padding = (this.viewport.ActualWidth - (this.XAxis.Maximum - this.XAxis.Minimum) * scaleFactor) / 2.0d; - this.ViewportPadding = new Thickness(padding, 0, padding, 0); - } + if (valueRangeAspectRatio > viewportAspectRatio) + { + // Add padding the the Y axis min/max to maintain the aspect ratio + double scaleFactor = this.viewportWidth / (this.XAxis.Maximum - this.XAxis.Minimum); + double padding = (this.viewportHeight - (this.YAxis.Maximum - this.YAxis.Minimum) * scaleFactor) / 2.0d; + this.ViewportPadding = new Thickness(0, padding, 0, padding); + } + else + { + // Add padding the the X axis min/max to maintain the aspect ratio + double scaleFactor = this.viewportHeight / (this.YAxis.Maximum - this.YAxis.Minimum); + double padding = (this.viewportWidth - (this.XAxis.Maximum - this.XAxis.Minimum) * scaleFactor) / 2.0d; + this.ViewportPadding = new Thickness(padding, 0, padding, 0); } } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/XYZVisualizationPanel.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/XYZVisualizationPanel.cs index 3f175e868..73cea040b 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/XYZVisualizationPanel.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizationPanels/XYZVisualizationPanel.cs @@ -19,9 +19,8 @@ namespace Microsoft.Psi.Visualization.VisualizationPanels /// /// Represents a visualization panel that 3D visualizers can be rendered in. /// - public class XYZVisualizationPanel : VisualizationPanel + public class XYZVisualizationPanel : InstantVisualizationPanel { - private int relativeWidth = 100; private double majorDistance = 5; private double minorDistance = 5; private double thickness = 0.01; @@ -136,18 +135,6 @@ public virtual RelayCommand MouseRightButtonUpCommand set { this.Set(nameof(this.CameraAnimation), ref this.cameraAnimation, value); } } - /// - /// Gets or sets the name of the relative width for the panel. - /// - [DataMember] - [DisplayName("Relative Width")] - [Description("The relative width for the panel.")] - public int RelativeWidth - { - get { return this.relativeWidth; } - set { this.Set(nameof(this.RelativeWidth), ref this.relativeWidth, value); } - } - /// /// Gets or sets the major distance. /// diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizerMetadata.cs b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizerMetadata.cs index 07a3374b6..0008740e9 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizerMetadata.cs +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/VisualizerMetadata.cs @@ -5,9 +5,11 @@ namespace Microsoft.Psi.Visualization { using System; using System.Collections.Generic; + using System.Linq; using System.Reflection; using Microsoft.Psi.PsiStudio.Common; using Microsoft.Psi.Visualization.Adapters; + using Microsoft.Psi.Visualization.Data; using Microsoft.Psi.Visualization.Summarizers; using Microsoft.Psi.Visualization.VisualizationObjects; using Microsoft.Psi.Visualization.VisualizationPanels; @@ -102,14 +104,14 @@ public class VisualizerMetadata public static List Create(Type visualizationObjectType, Dictionary summarizers, List dataAdapters, VisualizationLogWriter logWriter) { // Get the visualization object attribute - VisualizationObjectAttribute visualizationObjectAttribute = GetVisualizationObjectAttribute(visualizationObjectType, logWriter); + var visualizationObjectAttribute = GetVisualizationObjectAttribute(visualizationObjectType, logWriter); if (visualizationObjectAttribute == null) { return null; } // Get the visualization panel type attribute - VisualizationPanelTypeAttribute visualizationPanelTypeAttribute = GetVisualizationPanelTypeAttribute(visualizationObjectType, logWriter); + var visualizationPanelTypeAttribute = GetVisualizationPanelTypeAttribute(visualizationObjectType, logWriter); if (visualizationPanelTypeAttribute == null) { return null; @@ -117,14 +119,14 @@ public static List Create(Type visualizationObjectType, Dict // Get the message data type for the visualization object. We will get nothing back // if visualizationObjectType does not ultimately derive from VisualizationObject - Type visualizationObjectDataType = GetVisualizationObjectDataType(visualizationObjectType, logWriter); + var visualizationObjectDataType = GetVisualizationObjectDataType(visualizationObjectType, logWriter); if (visualizationObjectDataType == null) { return null; } // Get the summarizer type (if the visualizer uses a summarizer) - SummarizerMetadata summarizerMetadata = null; + var summarizerMetadata = default(SummarizerMetadata); if (visualizationObjectAttribute.SummarizerType != null) { if (summarizers.ContainsKey(visualizationObjectAttribute.SummarizerType)) @@ -157,7 +159,7 @@ public static List Create(Type visualizationObjectType, Dict // 2) Otherwise, use the visualization object's data type Type dataType = summarizerMetadata != null ? summarizerMetadata.InputType : visualizationObjectDataType; - List metadatas = new List(); + var metadatas = new List(); // Add the visualization metadata using no adapter Create(metadatas, dataType, visualizationObjectType, visualizationObjectAttribute, visualizationPanelTypeAttribute, null); @@ -203,9 +205,65 @@ internal VisualizerMetadata GetCloneWithNewStreamAdapterType(Type streamAdapterT VisualizationPanelTypeAttribute visualizationPanelTypeAttribute, StreamAdapterMetadata adapterMetadata) { - var commandTitle = (visualizationObjectAttribute.IsUniversalVisualizer || adapterMetadata == null) ? - $"{ContextMenuName.Visualize} {visualizationObjectAttribute.CommandText}" : - $"{ContextMenuName.VisualizeAs} {visualizationObjectAttribute.CommandText}"; + // First, check for the case where the list of visualizer metadatas already contains a + // visualizer with an adapter with the same type signature. If so, we need to generate + // the command names by appending a specification of the adapter name. + static bool HasSameAdapterTypes(Type streamAdapterType, Type otherStreamAdapterType) + { + if (streamAdapterType == null || otherStreamAdapterType == null) + { + return false; + } + + if (streamAdapterType.BaseType.IsGenericType && streamAdapterType.BaseType.GetGenericTypeDefinition() == typeof(StreamAdapter<,>)) + { + var streamAdapterGenericArguments = streamAdapterType.BaseType.GetGenericArguments(); + if (otherStreamAdapterType.BaseType.IsGenericType && otherStreamAdapterType.BaseType.GetGenericTypeDefinition() == typeof(StreamAdapter<,>)) + { + var otherStreamAdapterGenericArguments = otherStreamAdapterType.BaseType.GetGenericArguments(); + return streamAdapterGenericArguments[0] == otherStreamAdapterGenericArguments[0] && + streamAdapterGenericArguments[1] == otherStreamAdapterGenericArguments[1]; + } + } + + return false; + } + + var sameAdapterVisualizerMetadatas = metadatas.Where(m => HasSameAdapterTypes(m.StreamAdapterType, adapterMetadata?.AdapterType)); + + var commandTitle = default(string); + var inNewPanelCommandTitle = default(string); + + // If there are other stream adapters with the same type signature + if (sameAdapterVisualizerMetadatas.Any()) + { + // Then elaborate the name of the command to include the name of the stream adapter. + var streamAdapterAttribute = adapterMetadata.AdapterType.GetCustomAttribute(); + var viaStreamAdapterName = string.IsNullOrEmpty(streamAdapterAttribute.Name) ? " (via unnamed adapter)" : $" (via {streamAdapterAttribute.Name} adapter)"; + commandTitle = $"{ContextMenuName.VisualizeAs} {visualizationObjectAttribute.CommandText}{viaStreamAdapterName}"; + inNewPanelCommandTitle = $"{ContextMenuName.VisualizeAs} {visualizationObjectAttribute.CommandText} in New Panel{viaStreamAdapterName}"; + + // Also for all the matching adapters, elaborate the names of the corresponding commands + foreach (var sameAdapterOtherVisualizerMetadata in sameAdapterVisualizerMetadatas) + { + var otherStreamAdapterAttribute = sameAdapterOtherVisualizerMetadata.StreamAdapterType.GetCustomAttribute(); + var viaOtherStreamAdapterName = string.IsNullOrEmpty(otherStreamAdapterAttribute.Name) ? " (via unnamed adapter)" : $" (via {otherStreamAdapterAttribute.Name} adapter)"; + + if (!sameAdapterOtherVisualizerMetadata.CommandText.EndsWith(viaOtherStreamAdapterName)) + { + sameAdapterOtherVisualizerMetadata.CommandText += viaOtherStreamAdapterName; + } + } + } + else + { + commandTitle = (visualizationObjectAttribute.IsUniversalVisualizer || adapterMetadata == null) ? + $"{ContextMenuName.Visualize} {visualizationObjectAttribute.CommandText}" : + $"{ContextMenuName.VisualizeAs} {visualizationObjectAttribute.CommandText}"; + inNewPanelCommandTitle = (visualizationObjectAttribute.IsUniversalVisualizer || adapterMetadata == null) ? + $"{ContextMenuName.Visualize} {visualizationObjectAttribute.CommandText} in New Panel" : + $"{ContextMenuName.VisualizeAs} {visualizationObjectAttribute.CommandText} in New Panel"; + } metadatas.Add(new VisualizerMetadata( dataType, @@ -219,10 +277,6 @@ internal VisualizerMetadata GetCloneWithNewStreamAdapterType(Type streamAdapterT false, visualizationObjectAttribute.IsUniversalVisualizer)); - var inNewPanelCommandTitle = (visualizationObjectAttribute.IsUniversalVisualizer || adapterMetadata == null) ? - $"{ContextMenuName.Visualize} {visualizationObjectAttribute.CommandText} in New Panel" : - $"{ContextMenuName.VisualizeAs} {visualizationObjectAttribute.CommandText} in New Panel"; - metadatas.Add(new VisualizerMetadata( dataType, visualizationObjectType, @@ -242,25 +296,25 @@ private static VisualizationObjectAttribute GetVisualizationObjectAttribute(Type if (visualizationObjectAttribute == null) { - logWriter.WriteError("Visualization object {0} could not be loaded because it is not decorated with a VisualizationObjectAttribute", visualizationObjectType.Name); + logWriter.WriteError($"Visualization object {0} could not be loaded because it is not decorated with a {nameof(VisualizationObjectAttribute)}.", visualizationObjectType.Name); return null; } if (string.IsNullOrWhiteSpace(visualizationObjectAttribute.CommandText)) { - logWriter.WriteError("Visualization object {0} could not be loaded because its VisualizationObjectAttribute does not specify a Text property", visualizationObjectType.Name); + logWriter.WriteError($"Visualization object {0} could not be loaded because its {nameof(VisualizationObjectAttribute)} does not specify a {nameof(VisualizationObjectAttribute.CommandText)} property.", visualizationObjectType.Name); return null; } if (string.IsNullOrWhiteSpace(visualizationObjectAttribute.IconSourcePath) && string.IsNullOrWhiteSpace(visualizationObjectAttribute.NewPanelIconSourcePath)) { - logWriter.WriteError("Visualization object {0} could not be loaded because its VisualizationObjectAttribute does not specify either an IconSourcePath property or a NewPanelIconSourcePath property", visualizationObjectType.Name); + logWriter.WriteError($"Visualization object {0} could not be loaded because its {nameof(VisualizationObjectAttribute)} does not specify either an {nameof(VisualizationObjectAttribute.IconSourcePath)} property or a {nameof(VisualizationObjectAttribute.NewPanelIconSourcePath)} property.", visualizationObjectType.Name); return null; } if (!visualizationObjectAttribute.VisualizationFormatString.Contains(VisualizationObjectAttribute.DefaultVisualizationFormatString)) { - logWriter.WriteError("Visualization object {0} could not be loaded because its VisualizationObjectAttribute has an invalid value for the VisualizationFormatString property", visualizationObjectType.Name); + logWriter.WriteError($"Visualization object {0} could not be loaded because its {nameof(VisualizationObjectAttribute)} has an invalid value for the {nameof(VisualizationObjectAttribute.DefaultVisualizationFormatString)} property.", visualizationObjectType.Name); return null; } diff --git a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Windows/GetParameterWindow.xaml b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Windows/GetParameterWindow.xaml index aaa58d85a..290bbf007 100644 --- a/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Windows/GetParameterWindow.xaml +++ b/Sources/Visualization/Microsoft.Psi.Visualization.Windows/Windows/GetParameterWindow.xaml @@ -41,19 +41,22 @@ + - - + + - public event PropertyChangedEventHandler PropertyChanged; - /// - /// Gets or sets the parameter name. - /// - public string ParameterName { get; set; } - /// /// Gets or sets the parameter value. /// @@ -70,6 +68,25 @@ public bool IsValid } } + /// + /// Gets or sets the error message. + /// + public string ErrorMessage + { + get => this.errorMessage; + set + { + if (this.errorMessage != value) + { + this.errorMessage = value; + if (this.PropertyChanged != null) + { + this.PropertyChanged.Invoke(this, new PropertyChangedEventArgs(nameof(this.ErrorMessage))); + } + } + } + } + private void OKButton_Click(object sender, RoutedEventArgs e) { this.DialogResult = true; @@ -85,7 +102,9 @@ private void ValidateFormValues(object sender, RoutedEventArgs e) private void Validate() { - this.IsValid = this.validator(this.ParameterValue).Item1; + (var isValid, var errorMessage) = this.validator(this.ParameterValue); + this.IsValid = isValid; + this.ErrorMessage = string.IsNullOrEmpty(this.ParameterValue) ? string.Empty : errorMessage; } } } \ No newline at end of file diff --git a/Sources/Visualization/Test.Psi.Visualization/Properties/AssemblyInfo.cs b/Sources/Visualization/Test.Psi.Visualization/Properties/AssemblyInfo.cs index a7f51db35..46ed7e122 100644 --- a/Sources/Visualization/Test.Psi.Visualization/Properties/AssemblyInfo.cs +++ b/Sources/Visualization/Test.Psi.Visualization/Properties/AssemblyInfo.cs @@ -10,6 +10,6 @@ [assembly: AssemblyCopyright("Copyright (C) Microsoft Corporation. All rights reserved.")] [assembly: ComVisible(false)] [assembly: Guid("7cd463a8-61bb-4937-aa96-02da13e622d0")] -[assembly: AssemblyVersion("0.16.92.1")] -[assembly: AssemblyFileVersion("0.16.92.1")] -[assembly: AssemblyInformationalVersion("0.16.92.1-beta")] +[assembly: AssemblyVersion("0.17.52.1")] +[assembly: AssemblyFileVersion("0.17.52.1")] +[assembly: AssemblyInformationalVersion("0.17.52.1-beta")]