FaultInjection: Adds ApplyPercentage for ServerErrorRule

Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/FaultInjectionServerErrorResult.cs

@@ -14,19 +14,27 @@ public sealed class FaultInjectionServerErrorResult : IFaultInjectionResult
         private readonly int times;
         private readonly TimeSpan delay;
         private readonly bool suppressServiceRequests;
+        private readonly double applyPercentage;
 
         /// <summary>
         /// Creates a new FaultInjectionServerErrorResult.
         /// </summary>
         /// <param name="serverErrorType">Specifies the server error type.</param>
         /// <param name="times">Specifies the number of times a rule can be applied on a single operation.</param>
         /// <param name="delay">Specifies the injected delay for the server error.</param>
-        public FaultInjectionServerErrorResult(FaultInjectionServerErrorType serverErrorType, int times, TimeSpan delay, bool suppressServiceRequests)
+        /// <param name="applyPercentage" Specifies the percentage of how many times the rule will be applied.</param>
+        public FaultInjectionServerErrorResult(
+            FaultInjectionServerErrorType serverErrorType, 
+            int times, 
+            TimeSpan delay, 
+            bool suppressServiceRequests,
+            double applyPercentage = 1)
         {
             this.serverErrorType = serverErrorType;
             this.times = times;
             this.delay = delay;
             this.suppressServiceRequests = suppressServiceRequests;
+            this.applyPercentage = applyPercentage;
         }
 
         /// <summary>
@@ -68,17 +76,27 @@ public bool GetSuppressServiceRequests()
             return this.suppressServiceRequests;
         }
 
+        /// <summary>
+        /// Gets the percentage of how many times the rule will be applied.
+        /// </summary>
+        /// <returns></returns>
+        public double GetApplyPercentage()
+        {
+            return this.applyPercentage;
+        }
+
         /// <summary>
         /// To String method
         /// </summary>
         /// <returns>a string represeting the <see cref="FaultInjectionServerErrorResult"/>.</returns>
         public override string ToString()
         {
             return String.Format(
-                "FaultInjectionServerErrorResult{{ serverErrorType: {0}, times: {1}, delay: {2}}}",
+                "FaultInjectionServerErrorResult{{ serverErrorType: {0}, times: {1}, delay: {2}, applicationPercentage: {3}}",
                 this.serverErrorType,
                 this.times,
-                this.delay);
+                this.delay,
+                this.applyPercentage);
         }
     }
 }

Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/FaultInjectionServerErrorResultBuilder.cs

@@ -15,6 +15,7 @@ public sealed class FaultInjectionServerErrorResultBuilder
         private TimeSpan delay;
         private bool suppressServiceRequest;
         private bool isDelaySet = false;
+        private double applyPercentage = 1;
 
         /// <summary>
         /// Creates a <see cref="FaultInjectionServerErrorResult"/>.
@@ -67,6 +68,17 @@ public FaultInjectionServerErrorResultBuilder WithSuppressServiceRequest(bool su
             return this;
         }
 
+        public FaultInjectionServerErrorResultBuilder WithApplyPercentage(double applyPercentage)
+        {
+            if (applyPercentage <= 0 || applyPercentage > 1)
+            {
+                throw new ArgumentOutOfRangeException(nameof(applyPercentage), "Argument 'applyPercentage' must be within the range (0, 1].");
+            }
+
+            this.applyPercentage = applyPercentage;
+            return this;
+        }
+
         /// <summary>
         /// Creates a new <see cref="FaultInjectionServerErrorResult"/>.
         /// </summary>
@@ -84,7 +96,8 @@ public FaultInjectionServerErrorResult Build()
                 this.serverErrorType,
                 this.times,
                 this.delay,
-                this.suppressServiceRequest);
+                this.suppressServiceRequest,
+                this.applyPercentage);
         }
     }
 }

Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/implementataion/ChaosInterceptor.cs

@@ -117,7 +117,12 @@ public async Task<(bool, StoreResponse?)> OnRequestCallAsync(ChannelCallArgument
         /// <param name="serverUri"></param>
         /// <param name="openingRequest"></param>
         /// <param name="channel"></param>
-        public async Task OnChannelOpenAsync(Guid activityId, Guid connectionCorrilationId, Uri serverUri, DocumentServiceRequest openingRequest, Channel channel)
+        public async Task OnChannelOpenAsync(
+            Guid activityId,
+            Guid connectionCorrilationId,
+            Uri serverUri,
+            DocumentServiceRequest openingRequest,
+            Channel channel)
         {
             FaultInjectionServerErrorRule? serverConnectionDelayRule = this.ruleStore?.FindRntbdServerConnectionDelayRule(
                 serverUri,
@@ -131,8 +136,11 @@ public async Task OnChannelOpenAsync(Guid activityId, Guid connectionCorrilation
 
                 this.applicationContext.AddRuleExecution(serverConnectionDelayRule.GetId(), activityId);
 
-                DefaultTrace.TraceInformation("FaultInjection: FaultInjection Rule {0} Inserted {1} duration connection delay for request {2}",
-                                    serverConnectionDelayRule.GetId(), serverConnectionDelayRule.GetDelay(), activityId);
+                DefaultTrace.TraceInformation(
+                    "FaultInjection: FaultInjection Rule {0} Inserted {1} duration connection delay for request {2}",
+                    serverConnectionDelayRule.GetId(),
+                    serverConnectionDelayRule.GetDelay(),
+                    activityId);
 
                 TimeSpan connectionDelay = serverConnectionDelayRule.GetDelay();
                 await Task.Delay(connectionDelay);
@@ -161,8 +169,11 @@ public async Task OnBeforeConnectionWriteAsync(ChannelCallArguments args)
                 this.applicationContext.AddRuleExecution(serverSendDelayRule.GetId(), args.CommonArguments.ActivityId);
                 TimeSpan delay = serverSendDelayRule.GetDelay();
 
-                DefaultTrace.TraceInformation("FaultInjection: FaultInjection Rule {0} Inserted {1} duration response delay for request {2}",
-                                    serverSendDelayRule.GetId(), delay, args.CommonArguments.ActivityId);
+                DefaultTrace.TraceInformation(
+                    "FaultInjection: FaultInjection Rule {0} Inserted {1} duration response delay for request {2}",
+                    serverSendDelayRule.GetId(),
+                    delay,
+                    args.CommonArguments.ActivityId);
 
                 await Task.Delay(delay);
             }
@@ -181,8 +192,11 @@ public async Task OnAfterConnectionWriteAsync(ChannelCallArguments args)
                 this.applicationContext.AddRuleExecution(serverResponseDelayRule.GetId(), args.CommonArguments.ActivityId);
                 TimeSpan delay = serverResponseDelayRule.GetDelay();
 
-                DefaultTrace.TraceInformation("FaultInjection: FaultInjection Rule {0} Inserted {1} duration response delay for request {2}",
-                                    serverResponseDelayRule.GetId(), delay, args.CommonArguments.ActivityId);
+                DefaultTrace.TraceInformation(
+                    "FaultInjection: FaultInjection Rule {0} Inserted {1} duration response delay for request {2}",
+                     serverResponseDelayRule.GetId(),
+                     delay,
+                     args.CommonArguments.ActivityId);
 
                 await Task.Delay(delay);
             }

Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/implementataion/FaultInjectionRuleProcessor.cs

@@ -161,8 +161,8 @@ private async Task<IFaultInjectionRuleInternal> GetEffectiveServerErrorRule(Faul
                     result.GetTimes(),
                     result.GetDelay(),
                     result.GetSuppressServiceRequests(),
+                    result.GetApplyPercentage(),
                     this.applicationContext));
-
         }
 
         private async Task<IFaultInjectionRuleInternal> GetEffectiveConnectionErrorRule(FaultInjectionRule rule)

Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/implementataion/FaultInjectionServerErrorResultInternal.cs

@@ -19,6 +19,7 @@ internal class FaultInjectionServerErrorResultInternal
         private readonly int times;
         private readonly TimeSpan delay;
         private readonly bool suppressServiceRequest;
+        private readonly double applicationPercentage;
         private readonly FaultInjectionApplicationContext applicationContext;
 
         /// <summary>
@@ -27,18 +28,21 @@ internal class FaultInjectionServerErrorResultInternal
         /// <param name="serverErrorType"></param>
         /// <param name="times"></param>
         /// <param name="delay"></param>
+        /// <param name="applicationPercentage"></param>
         /// <param name="applicationContext"></param>
         public FaultInjectionServerErrorResultInternal(
             FaultInjectionServerErrorType serverErrorType, 
             int times, 
             TimeSpan delay, 
             bool suppressServiceRequest,
+            double applicationPercentage,
             FaultInjectionApplicationContext applicationContext)
         {
             this.serverErrorType = serverErrorType;
             this.times = times;
             this.delay = delay;
             this.suppressServiceRequest = suppressServiceRequest;
+            this.applicationPercentage = applicationPercentage;
             this.applicationContext = applicationContext;
         }
 
@@ -79,6 +83,15 @@ public bool GetSuppressServiceRequest()
             return this.suppressServiceRequest;
         }
 
+        /// <summary>
+        /// Returns the percentage of how many times the rule will be applied.
+        /// </summary>
+        /// <returns></returns>
+        public double GetApplicationPercentage()
+        {
+            return this.applicationPercentage;
+        }
+
         /// <summary>
         /// Determins if the rule can be applied.
         /// </summary>

Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/src/implementataion/FaultInjectionServerErrorRule.cs

@@ -72,6 +72,10 @@ public bool IsApplicable(ChannelCallArguments args)
             {
                 return false;
             }
+            else if (Random.Shared.NextDouble() < this.result.GetApplicationPercentage())
+            {
+                return false;
+            }
             else
             {
                 Interlocked.Increment(ref this.hitCount);

Microsoft.Azure.Cosmos.Samples/Tools/FaultInjection/tests/FaultInjectionDirectModeTests.cs

@@ -1203,6 +1203,61 @@ private async Task Timeout_FaultInjectionServerErrorRule_IncludePrimaryTest()
             }
         }
 
+        [TestMethod]
+        [Owner("nalutripician")]
+        [Description("Tests apply percent")]
+        public async Task Timeout_FaultInjectionServerErrorRule_ThresholdTest()
+        {
+            string thresholdRuleId = "hitCountRule-" + Guid.NewGuid().ToString();
+            FaultInjectionRule thresholdRule = new FaultInjectionRuleBuilder(
+                id: thresholdRuleId,
+                condition:
+                    new FaultInjectionConditionBuilder()
+                        .WithOperationType(FaultInjectionOperationType.ReadItem)
+                        .Build(),
+                result:
+                    FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.Gone)
+                        .WithApplyPercentage(.5)
+                        .WithTimes(1)
+                        .Build())
+                .WithDuration(TimeSpan.FromMinutes(5))
+                .Build();
+            thresholdRule.Disable();
+
+            try
+            {
+                FaultInjector faultInjector = new FaultInjector(new List<FaultInjectionRule> { thresholdRule });
+
+                await this.Initialize(faultInjector, true);
+
+                JObject createdItem = JObject.FromObject(new { id = Guid.NewGuid().ToString(), Pk = Guid.NewGuid().ToString() });
+
+                ItemResponse<JObject>? itemResponse = this.container != null
+                    ? await this.container.CreateItemAsync<JObject>(createdItem)
+                    : null;
+                Assert.IsNotNull(itemResponse);
+
+                CosmosDiagnostics? cosmosDiagnostics;
+
+                thresholdRule.Enable();
+
+                for (int i = 0; i < 100; i++)
+                {
+                    cosmosDiagnostics = this.container != null
+                        ? await this.PerformDocumentOperation(this.container, OperationType.Read, createdItem)
+                        : null;
+                    Assert.IsNotNull(cosmosDiagnostics);
+                }
+
+                Assert.IsTrue(thresholdRule.GetHitCount() >= 38, "This is Expected to fail 0.602% of the time");
+                Assert.IsTrue(thresholdRule.GetHitCount() <= 62, "This is Expected to fail 0.602% of the time");
+            }
+            finally
+            {
+                thresholdRule.Disable();
+            }
+        }
+
         [TestMethod]
         [Owner("nalutripician")]
         [Description("Tests fault injection connection error rules")]