Adding pagination limits. (#2153)

## Why make this change?

This change allows users to use the runtime config to provide limits on
the number of records that can be retrieved through paginated calls. It
also gives them the ability to define a default size which will be the
size returned when no pagination input is given (no first in case of
graphql and no limit in case of rest calls).

## What is this change?
It adds the nullable paginationoptions property to runtimeoptions of
runtimeConfig.
Default page size is set to 100. 
Max page size is set to 100,000 ( can be altered by the user in their
runtimeconfig).

A call with -1 pagination input will result in max page size records
being returned. Any call with a pagination number higher than max page
size will be rejected with bad request.

In config:
default-page-size is an integer
default-page-size default is 100
default-page-size value -1 means "same as max-page-size"
default-page-size value 0 is an error
default-page-size value less than -1 is an error
default-page-size value more than than max-page-size is an error
max-page-size is an integer
max-page-size default is 100,000
max-page-size value -1 means "same as int.MaxValue"
max-page-size value 0 is an error
max-page-size value less than -1 is an error
max-page-size value more than int.MaxValue is an error

In a query (REST or GQL):
$first=-1 means "whatever the max-page-size value is"
$first=less than -1 is an error
$first=0 is an error
$first=(any value more than max-page-size) is an error

Sample configuration file:
```json
{
  "runtime": {
    "pagination": {
      "default-page-size": -1,
      "max-page-size": 1233
    }
  }
}
```

## How was this tested?
1. For default page value of 100, invalid page value of 0 or <-1 the
existing test cases should cover.
2. Added tests for both GQL and REST for above conditions.

---------

Co-authored-by: Sean Leonard <sean.leonard@microsoft.com>
Co-authored-by: aaronburtle <93220300+aaronburtle@users.noreply.github.com>

src/Config/ObjectModel/PaginationOptions.cs

@@ -0,0 +1,112 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System.Diagnostics.CodeAnalysis;
+using System.Net;
+using System.Text.Json.Serialization;
+using Azure.DataApiBuilder.Service.Exceptions;
+
+namespace Azure.DataApiBuilder.Config.ObjectModel;
+
+/// <summary>
+/// Pagination options for the dab setup.
+/// Properties are nullable to support DAB CLI merge config
+/// expected behavior.
+/// </summary>
+public record PaginationOptions
+{
+    /// <summary>
+    /// Default page size.
+    /// </summary>
+    public const uint DEFAULT_PAGE_SIZE = 100;
+
+    /// <summary>
+    /// Max page size.
+    /// </summary>
+    public const uint MAX_PAGE_SIZE = 100000;
+
+    /// <summary>
+    /// The default page size for pagination.
+    /// </summary>
+    [JsonPropertyName("default-page-size")]
+    public int? DefaultPageSize { get; init; } = null;
+
+    /// <summary>
+    /// The max page size for pagination.
+    /// </summary>
+    [JsonPropertyName("max-page-size")]
+    public int? MaxPageSize { get; init; } = null;
+
+    [JsonConstructor]
+    public PaginationOptions(int? DefaultPageSize = null, int? MaxPageSize = null)
+    {
+        if (MaxPageSize is not null)
+        {
+            ValidatePageSize((int)MaxPageSize);
+            this.MaxPageSize = MaxPageSize == -1 ? Int32.MaxValue : (int)MaxPageSize;
+            UserProvidedMaxPageSize = true;
+        }
+        else
+        {
+            this.MaxPageSize = (int)MAX_PAGE_SIZE;
+        }
+
+        if (DefaultPageSize is not null)
+        {
+            ValidatePageSize((int)DefaultPageSize);
+            this.DefaultPageSize = DefaultPageSize == -1 ? (int)this.MaxPageSize : (int)DefaultPageSize;
+            UserProvidedDefaultPageSize = true;
+        }
+        else
+        {
+            this.DefaultPageSize = (int)DEFAULT_PAGE_SIZE;
+        }
+
+        if (this.DefaultPageSize > this.MaxPageSize)
+        {
+            throw new DataApiBuilderException(
+                message: "Pagination options invalid. The default page size cannot be greater than max page size",
+                statusCode: HttpStatusCode.ServiceUnavailable,
+                subStatusCode: DataApiBuilderException.SubStatusCodes.ConfigValidationError);
+        }
+    }
+
+    /// <summary>
+    /// Flag which informs CLI and JSON serializer whether to write default page size.
+    /// property and value to the runtime config file.
+    /// When user doesn't provide the default-page-size property/value, which signals DAB to use the default,
+    /// the DAB CLI should not write the default value to a serialized config.
+    /// This is because the user's intent is to use DAB's default value which could change
+    /// and DAB CLI writing the property and value would lose the user's intent.
+    /// This is because if the user were to use the CLI created config, a default-page-size
+    /// property/value specified would be interpreted by DAB as "user explicitly default-page-size."
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
+    [MemberNotNullWhen(true, nameof(DefaultPageSize))]
+    public bool UserProvidedDefaultPageSize { get; init; } = false;
+
+    /// <summary>
+    /// Flag which informs CLI and JSON serializer whether to write max-page-size
+    /// property and value to the runtime config file.
+    /// When user doesn't provide the max-page-size property/value, which signals DAB to use the default,
+    /// the DAB CLI should not write the default value to a serialized config.
+    /// This is because the user's intent is to use DAB's default value which could change
+    /// and DAB CLI writing the property and value would lose the user's intent.
+    /// This is because if the user were to use the CLI created config, a max-page-size
+    /// property/value specified would be interpreted by DAB as "user explicitly max-page-size."
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
+    [MemberNotNullWhen(true, nameof(MaxPageSize))]
+    public bool UserProvidedMaxPageSize { get; init; } = false;
+
+    private static void ValidatePageSize(int pageSize)
+    {
+        if (pageSize < -1 || pageSize == 0 || pageSize > Int32.MaxValue)
+        {
+            throw new DataApiBuilderException(
+                message: "Pagination options invalid. Page size arguments cannot be 0, exceed max int value or be less than -1",
+                statusCode: HttpStatusCode.ServiceUnavailable,
+                subStatusCode: DataApiBuilderException.SubStatusCodes.ConfigValidationError);
+        }
+    }
+}

src/Config/ObjectModel/RuntimeConfig.cs

@@ -471,4 +471,45 @@ public bool IsMultipleCreateOperationEnabled()
                Runtime.GraphQL.MultipleMutationOptions.MultipleCreateOptions is not null &&
                Runtime.GraphQL.MultipleMutationOptions.MultipleCreateOptions.Enabled);
     }
+
+    public uint DefaultPageSize()
+    {
+        return (uint?)Runtime?.Pagination?.DefaultPageSize ?? PaginationOptions.DEFAULT_PAGE_SIZE;
+    }
+
+    public uint MaxPageSize()
+    {
+        return (uint?)Runtime?.Pagination?.MaxPageSize ?? PaginationOptions.MAX_PAGE_SIZE;
+    }
+
+    /// <summary>
+    /// Get the pagination limit from the runtime configuration.
+    /// </summary>
+    /// <param name="first">The pagination input from the user. Example: $first=10</param>
+    /// <returns></returns>
+    /// <exception cref="DataApiBuilderException"></exception>
+    public uint GetPaginationLimit(int? first)
+    {
+        uint defaultPageSize = this.DefaultPageSize();
+        uint maxPageSize = this.MaxPageSize();
+
+        if (first is not null)
+        {
+            if (first < -1 || first == 0 || first > maxPageSize)
+            {
+                throw new DataApiBuilderException(
+                message: $"Invalid number of items requested, {nameof(first)} argument must be either -1 or a positive number within the max page size limit of {maxPageSize}. Actual value: {first}",
+                statusCode: HttpStatusCode.BadRequest,
+                subStatusCode: DataApiBuilderException.SubStatusCodes.BadRequest);
+            }
+            else
+            {
+                return (first == -1 ? maxPageSize : (uint)first);
+            }
+        }
+        else
+        {
+            return defaultPageSize;
+        }
+    }
 }

src/Config/ObjectModel/RuntimeOptions.cs

@@ -14,6 +14,7 @@ public record RuntimeOptions
     public string? BaseRoute { get; init; }
     public TelemetryOptions? Telemetry { get; init; }
     public EntityCacheOptions? Cache { get; init; }
+    public PaginationOptions? Pagination { get; init; }
 
     [JsonConstructor]
     public RuntimeOptions(
@@ -22,14 +23,16 @@ public record RuntimeOptions
         HostOptions? Host,
         string? BaseRoute = null,
         TelemetryOptions? Telemetry = null,
-        EntityCacheOptions? Cache = null)
+        EntityCacheOptions? Cache = null,
+        PaginationOptions? Pagination = null)
     {
         this.Rest = Rest;
         this.GraphQL = GraphQL;
         this.Host = Host;
         this.BaseRoute = BaseRoute;
         this.Telemetry = Telemetry;
         this.Cache = Cache;
+        this.Pagination = Pagination;
     }
 
     /// <summary>

src/Core/Models/GraphQLFilterParsers.cs

@@ -315,6 +315,7 @@ public GQLFilterParser(RuntimeConfigProvider runtimeConfigProvider, IMetadataPro
         CosmosExistsQueryStructure existsQuery = new(
             ctx,
             new Dictionary<string, object?>(),
+            _configProvider,
             metadataProvider,
             queryStructure.AuthorizationResolver,
             this,

src/Core/Models/RestRequestContexts/RestRequestContext.cs

@@ -88,7 +88,7 @@ protected RestRequestContext(string entityName, DatabaseObject dbo)
     /// Based on request this property may or may not be populated.
     /// </summary>
 
-    public uint? First { get; set; }
+    public int? First { get; set; }
     /// <summary>
     /// Is the result supposed to be multiple values or not.
     /// </summary>

src/Core/Resolvers/CosmosExistsQueryStructure.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 using Azure.DataApiBuilder.Auth;
+using Azure.DataApiBuilder.Core.Configurations;
 using Azure.DataApiBuilder.Core.Models;
 using Azure.DataApiBuilder.Core.Services;
 using HotChocolate.Resolvers;
@@ -15,13 +16,15 @@ public class CosmosExistsQueryStructure : CosmosQueryStructure
         /// </summary>
         public CosmosExistsQueryStructure(IMiddlewareContext context,
             IDictionary<string, object?> parameters,
+            RuntimeConfigProvider runtimeConfigProvider,
             ISqlMetadataProvider metadataProvider,
             IAuthorizationResolver authorizationResolver,
             GQLFilterParser gQLFilterParser,
             IncrementingInteger? counter = null,
             List<Predicate>? predicates = null)
             : base(context,
                   parameters,
+                  runtimeConfigProvider,
                   metadataProvider,
                   authorizationResolver,
                   gQLFilterParser,

src/Core/Resolvers/CosmosQueryEngine.cs

@@ -69,7 +69,7 @@ DabCacheService cache
 
             ISqlMetadataProvider metadataStoreProvider = _metadataProviderFactory.GetMetadataProvider(dataSourceName);
 
-            CosmosQueryStructure structure = new(context, parameters, metadataStoreProvider, _authorizationResolver, _gQLFilterParser);
+            CosmosQueryStructure structure = new(context, parameters, _runtimeConfigProvider, metadataStoreProvider, _authorizationResolver, _gQLFilterParser);
             RuntimeConfig runtimeConfig = _runtimeConfigProvider.GetConfig();
 
             string queryString = _queryBuilder.Build(structure);
@@ -201,7 +201,7 @@ DabCacheService cache
             // TODO: add support for TOP and Order-by push-down
 
             ISqlMetadataProvider metadataStoreProvider = _metadataProviderFactory.GetMetadataProvider(dataSourceName);
-            CosmosQueryStructure structure = new(context, parameters, metadataStoreProvider, _authorizationResolver, _gQLFilterParser);
+            CosmosQueryStructure structure = new(context, parameters, _runtimeConfigProvider, metadataStoreProvider, _authorizationResolver, _gQLFilterParser);
             CosmosClient client = _clientProvider.Clients[dataSourceName];
             Container container = client.GetDatabase(structure.Database).GetContainer(structure.Container);
             QueryDefinition querySpec = new(_queryBuilder.Build(structure));

src/Core/Resolvers/CosmosQueryStructure.cs

@@ -5,6 +5,7 @@
 using Azure.DataApiBuilder.Auth;
 using Azure.DataApiBuilder.Config.DatabasePrimitives;
 using Azure.DataApiBuilder.Config.ObjectModel;
+using Azure.DataApiBuilder.Core.Configurations;
 using Azure.DataApiBuilder.Core.Models;
 using Azure.DataApiBuilder.Core.Services;
 using Azure.DataApiBuilder.Core.Services.MetadataProviders;
@@ -36,10 +37,12 @@ public class CosmosQueryStructure : BaseQueryStructure
         public string Container { get; internal set; }
         public string Database { get; internal set; }
         public string? Continuation { get; internal set; }
-        public int? MaxItemCount { get; internal set; }
+        public uint? MaxItemCount { get; internal set; }
         public string? PartitionKeyValue { get; internal set; }
         public List<OrderByColumn> OrderByColumns { get; internal set; }
 
+        public RuntimeConfigProvider RuntimeConfigProvider { get; internal set; }
+
         public string GetTableAlias()
         {
             return $"table{TableCounter.Next()}";
@@ -48,6 +51,7 @@ public string GetTableAlias()
         public CosmosQueryStructure(
             IMiddlewareContext context,
             IDictionary<string, object?> parameters,
+            RuntimeConfigProvider provider,
             ISqlMetadataProvider metadataProvider,
             IAuthorizationResolver authorizationResolver,
             GQLFilterParser gQLFilterParser,
@@ -58,6 +62,7 @@ public string GetTableAlias()
             _context = context;
             SourceAlias = _containerAlias;
             DatabaseObject.Name = _containerAlias;
+            RuntimeConfigProvider = provider;
             Init(parameters);
         }
 
@@ -116,7 +121,6 @@ private void Init(IDictionary<string, object?> queryParams)
 
             IsPaginated = QueryBuilder.IsPaginationType(underlyingType);
             OrderByColumns = new();
-
             if (IsPaginated)
             {
                 FieldNode? fieldNode = ExtractItemsQueryField(selection.SyntaxNode);
@@ -155,13 +159,21 @@ private void Init(IDictionary<string, object?> queryParams)
                     (CosmosSqlMetadataProvider)MetadataProvider);
             }
 
+            RuntimeConfigProvider.TryGetConfig(out RuntimeConfig? runtimeConfig);
             // first and after will not be part of query parameters. They will be going into headers instead.
             // TODO: Revisit 'first' while adding support for TOP queries
             if (queryParams.ContainsKey(QueryBuilder.PAGE_START_ARGUMENT_NAME))
             {
-                MaxItemCount = (int?)queryParams[QueryBuilder.PAGE_START_ARGUMENT_NAME];
+                object? firstArgument = queryParams[QueryBuilder.PAGE_START_ARGUMENT_NAME];
+                MaxItemCount = runtimeConfig?.GetPaginationLimit((int?)firstArgument);
+
                 queryParams.Remove(QueryBuilder.PAGE_START_ARGUMENT_NAME);
             }
+            else
+            {
+                // set max item count to default value.
+                MaxItemCount = runtimeConfig?.DefaultPageSize();
+            }
 
             if (queryParams.ContainsKey(QueryBuilder.PAGINATION_TOKEN_ARGUMENT_NAME))
             {

src/Core/Resolvers/Sql Query Structures/SqlQueryStructure.cs

@@ -17,7 +17,6 @@
 using HotChocolate.Language;
 using HotChocolate.Resolvers;
 using Microsoft.AspNetCore.Http;
-
 namespace Azure.DataApiBuilder.Core.Resolvers
 {
     /// <summary>
@@ -59,15 +58,10 @@ public class SqlQueryStructure : BaseSqlQueryStructure
         /// </summary>
         public Dictionary<string, string> ColumnLabelToParam { get; }
 
-        /// <summary>
-        /// Default limit when no first param is specified for list queries
-        /// </summary>
-        private const uint DEFAULT_LIST_LIMIT = 100;
-
         /// <summary>
         /// The maximum number of results this query should return.
         /// </summary>
-        private uint? _limit = DEFAULT_LIST_LIMIT;
+        private uint? _limit = PaginationOptions.DEFAULT_PAGE_SIZE;
 
         /// <summary>
         /// If this query is built because of a GraphQL query (as opposed to
@@ -198,7 +192,9 @@ public class SqlQueryStructure : BaseSqlQueryStructure
             }
 
             AddColumnsForEndCursor();
-            _limit = context.First is not null ? context.First + 1 : DEFAULT_LIST_LIMIT + 1;
+            runtimeConfigProvider.TryGetConfig(out RuntimeConfig? runtimeConfig);
+            _limit = runtimeConfig?.GetPaginationLimit((int?)context.First) + 1;
+
             ParametrizeColumns();
         }
 
@@ -334,24 +330,19 @@ private List<OrderByColumn> PrimaryKeyAsOrderByColumns()
                 IsListQuery = outputType.IsListType();
             }
 
-            if (IsListQuery && queryParams.ContainsKey(QueryBuilder.PAGE_START_ARGUMENT_NAME))
+            if (IsListQuery)
             {
-                // parse first parameter for all list queries
-                object? firstObject = queryParams[QueryBuilder.PAGE_START_ARGUMENT_NAME];
-
-                if (firstObject != null)
+                runtimeConfigProvider.TryGetConfig(out RuntimeConfig? runtimeConfig);
+                if (queryParams.ContainsKey(QueryBuilder.PAGE_START_ARGUMENT_NAME))
                 {
-                    int first = (int)firstObject;
-
-                    if (first <= 0)
-                    {
-                        throw new DataApiBuilderException(
-                        message: $"Invalid number of items requested, {QueryBuilder.PAGE_START_ARGUMENT_NAME} argument must be an integer greater than 0 for {schemaField.Name}. Actual value: {first.ToString()}",
-                        statusCode: HttpStatusCode.BadRequest,
-                        subStatusCode: DataApiBuilderException.SubStatusCodes.BadRequest);
-                    }
-
-                    _limit = (uint)first;
+                    // parse first parameter for all list queries
+                    object? firstObject = queryParams[QueryBuilder.PAGE_START_ARGUMENT_NAME];
+                    _limit = runtimeConfig?.GetPaginationLimit((int?)firstObject);
+                }
+                else
+                {
+                    // if first is not passed, we should use the default page size.
+                    _limit = runtimeConfig?.DefaultPageSize();
                 }
             }