diff --git a/worker/include/RTC/SimulcastConsumer.hpp b/worker/include/RTC/SimulcastConsumer.hpp
index fad2728c..3c55a66f 100644
--- a/worker/include/RTC/SimulcastConsumer.hpp
+++ b/worker/include/RTC/SimulcastConsumer.hpp
@@ -111,7 +111,8 @@ namespace RTC
 		int16_t currentSpatialLayer{ -1 };
 		int16_t tsReferenceSpatialLayer{ -1 }; // Used for RTP TS sync.
 		std::unique_ptr<RTC::Codecs::EncodingContext> encodingContext;
-		uint32_t tsOffset{ 0u }; // RTP Timestamp offset.
+		uint32_t tsOffset{ 0u };          // RTP Timestamp offset.
+		uint32_t tsReferenceOffset{ 0u }; // RTP Timestamp offset from the tsReferenceSpatialLayer.
 		bool keyFrameForTsOffsetRequested{ false };
 		uint64_t lastBweDowngradeAtMs{ 0u }; // Last time we moved to lower spatial layer due to BWE.
 	};
diff --git a/worker/src/RTC/SimulcastConsumer.cpp b/worker/src/RTC/SimulcastConsumer.cpp
index eefc7c63..60a0941b 100644
--- a/worker/src/RTC/SimulcastConsumer.cpp
+++ b/worker/src/RTC/SimulcastConsumer.cpp
@@ -700,9 +700,27 @@ namespace RTC
 			uint32_t tsOffset{ 0u };
 
 			// Sync our RTP stream's RTP timestamp.
-			if (spatialLayer == this->tsReferenceSpatialLayer)
+			if (-1 == this->tsReferenceSpatialLayer)
+			{
+				if (shouldSwitchCurrentSpatialLayer)
+				{
+					// If we're actually switching streams, push the RTP timestamp up (we don't have a way to
+					// offset based on abs time yet) otherwise, if it's the first packet, we can just keep the
+					// timestamps
+					if (this->rtpStream->GetMaxPacketTs())
+						tsOffset = packet->GetTimestamp() - this->rtpStream->GetMaxPacketTs() -
+						           33 * this->rtpStream->GetClockRate() / 1000;
+					else
+						tsOffset = 0u;
+				}
+				else
+				{
+					tsOffset = this->tsOffset; // status quo
+				}
+			}
+			else if (spatialLayer == this->tsReferenceSpatialLayer)
 			{
-				tsOffset = 0u;
+				tsOffset = this->tsReferenceOffset;
 			}
 			// If this is not the RTP stream we use as TS reference, do NTP based RTP TS synchronization.
 			else
@@ -730,7 +748,7 @@ namespace RTC
 
 				// Apply offset. This is the difference that later must be removed from the
 				// sending RTP packet.
-				tsOffset = newTs2 - ts1;
+				tsOffset = this->tsReferenceOffset + newTs2 - ts1;
 			}
 
 			// When switching to a new stream it may happen that the timestamp of this
@@ -807,6 +825,10 @@ namespace RTC
 
 			this->tsOffset = tsOffset;
 
+			// We need this update here and below, it picks up and carries forward any adjustments we made above
+			if (spatialLayer == this->tsReferenceSpatialLayer)
+				this->tsReferenceOffset = this->tsOffset;
+
 			// Sync our RTP stream's sequence number.
 			this->rtpSeqManager.Sync(packet->GetSequenceNumber() - 1);
 
@@ -821,6 +843,9 @@ namespace RTC
 			// Update current spatial layer.
 			this->currentSpatialLayer = this->targetSpatialLayer;
 
+			if (this->currentSpatialLayer == this->tsReferenceSpatialLayer)
+				this->tsReferenceOffset = this->tsOffset;
+
 			// Update target and current temporal layer.
 			this->encodingContext->SetTargetTemporalLayer(this->targetTemporalLayer);
 			this->encodingContext->SetCurrentTemporalLayer(packet->GetTemporalLayer());
@@ -853,6 +878,16 @@ namespace RTC
 				EmitLayersChange();
 		}
 
+		if (-1 == this->tsReferenceSpatialLayer)
+		{
+			auto* producerCurrentRtpStream = GetProducerCurrentRtpStream();
+			if (producerCurrentRtpStream && producerCurrentRtpStream->GetSenderReportNtpMs())
+			{
+				this->tsReferenceSpatialLayer = this->currentSpatialLayer;
+				this->tsReferenceOffset       = this->tsOffset;
+			}
+		}
+
 		// Update RTP seq number and timestamp based on NTP offset.
 		uint16_t seq;
 		uint32_t timestamp = packet->GetTimestamp() - this->tsOffset;
@@ -1323,14 +1358,18 @@ namespace RTC
 	{
 		MS_TRACE();
 
 		// If we don't have yet a RTP timestamp reference, set it now.
-		if (newTargetSpatialLayer != -1 && this->tsReferenceSpatialLayer == -1)
-		{
-			MS_DEBUG_TAG(
-			  simulcast, "using spatial layer %" PRIi16 " as RTP timestamp reference", newTargetSpatialLayer);
-
-			this->tsReferenceSpatialLayer = newTargetSpatialLayer;
-		}
+		// if (newTargetSpatialLayer != -1 && this->tsReferenceSpatialLayer == -1)
+		//{
+		//	MS_DEBUG_TAG(
+		//	  simulcast, "using spatial layer %" PRIi16 " as RTP timestamp reference",
+		// newTargetSpatialLayer);
+		//
+		//	this->tsReferenceSpatialLayer = newTargetSpatialLayer;
+		//}
 
 		if (newTargetSpatialLayer == -1)
 		{