Consolidate AdditionalProperties with raw data field (Phase 1) (#4510)

* initial commit to change the unverifiable types to binarydata

* update the structure to split additional properties and raw data field

* additional properties and raw data field could coexist now, but still no filters

* implement the filter for additional properties property

* regenerate

* update for array and dictionary

* fix issue in some hlc package

* fix in HLC when unknown is object there

* update schema object type accordingly

* regen

* reenable some test cases, and fix the emitter issue to support ...Record

* regen after merge

* support generate additional properties from record spread and add test cases

* added an array case

* regen

* fix the union type issue

* fix the replace BinaryData with union issue

* fix

* update cadl-ranch version and implement half of the new test cases

* add other cases

* fix failed case

* regen

* remove workaround

* regen after merge

* adding similar things to schemaobjecttype

* fix test cases

* fix a disalignment

* update a mgmt test to ensure this is working properly

* Update src/AutoRest.CSharp/Common/Output/Builders/BuilderHelpers.cs

Co-authored-by: Wei Hu <live1206@gmail.com>

* Update src/AutoRest.CSharp/Common/Output/Builders/BuilderHelpers.cs

Co-authored-by: Wei Hu <live1206@gmail.com>

---------

Co-authored-by: Wei Hu <live1206@gmail.com>

src/AutoRest.CSharp/Common/Generation/Types/TypeFactory.cs

@@ -22,12 +22,13 @@ namespace AutoRest.CSharp.Generation.Types
     internal class TypeFactory
     {
         private readonly OutputLibrary _library;
-        private readonly Type _unknownType;
+
+        public Type UnknownType { get; }
 
         public TypeFactory(OutputLibrary library, Type unknownType)
         {
             _library = library;
-            _unknownType = unknownType;
+            UnknownType = unknownType;
         }
 
         private Type AzureResponseErrorType => typeof(ResponseError);
@@ -91,7 +92,7 @@ public TypeFactory(OutputLibrary library, Type unknownType)
                 _ => new CSharpType(typeof(object), inputType.IsNullable),
             },
             InputGenericType genericType => new CSharpType(genericType.Type, CreateType(genericType.ArgumentType)).WithNullable(inputType.IsNullable),
-            InputIntrinsicType { Kind: InputIntrinsicTypeKind.Unknown } => _unknownType,
+            InputIntrinsicType { Kind: InputIntrinsicTypeKind.Unknown } => UnknownType,
             CodeModelType cmt => CreateType(cmt.Schema, cmt.IsNullable),
             _ => throw new Exception("Unknown type")
         };
@@ -139,8 +140,8 @@ private Type GetListType(Schema schema)
             AllSchemaTypes.Unixtime => typeof(DateTimeOffset),
             AllSchemaTypes.Uri => typeof(Uri),
             AllSchemaTypes.Uuid => typeof(Guid),
-            AllSchemaTypes.Any => _unknownType,
-            AllSchemaTypes.AnyObject => ToXMsFormatType(format) ?? _unknownType,
+            AllSchemaTypes.Any => UnknownType,
+            AllSchemaTypes.AnyObject => ToXMsFormatType(format) ?? UnknownType,
             AllSchemaTypes.Binary => typeof(byte[]),
             _ => null
         };

src/AutoRest.CSharp/Common/Output/Builders/BuilderHelpers.cs

@@ -5,7 +5,6 @@
 using System.Collections.Generic;
 using System.Globalization;
 using System.Linq;
-using System.Security;
 using System.Text;
 using AutoRest.CSharp.Common.Input;
 using AutoRest.CSharp.Generation.Types;
@@ -358,5 +357,66 @@ public static MethodSignatureModifiers MapModifiers(ISymbol symbol)
             }
             return modifiers;
         }
+
+        public static CSharpType CreateAdditionalPropertiesPropertyType(CSharpType originalType, CSharpType unknownType)
+        {
+            // TODO -- we only construct additional properties when the type is verifiable, because we always need the property to fall into the bucket of serialized additional raw data field when it does not fit the additional properties.
+            var arguments = originalType.Arguments;
+            var keyType = arguments[0];
+            var valueType = arguments[1];
+
+            return originalType.MakeGenericType(new[] { ReplaceUnverifiableType(keyType, unknownType), ReplaceUnverifiableType(valueType, unknownType) });
+        }
+
+        private static CSharpType ReplaceUnverifiableType(CSharpType type, CSharpType unknownType)
+        {
+            // when the type is System.Object Or BinaryData
+            if (type.EqualsIgnoreNullable(unknownType))
+            {
+                return type;
+            }
+
+            // when the type is a verifiable type
+            if (IsVerifiableType(type))
+            {
+                return type;
+            }
+
+            // when the type is a union
+            if (type.IsUnion)
+            {
+                return type;
+            }
+
+            // otherwise the type is not a verifiable type
+            // replace for list
+            if (type.IsList)
+            {
+                return type.MakeGenericType(new[] { ReplaceUnverifiableType(type.Arguments[0], unknownType) });
+            }
+            // replace for dictionary
+            if (type.IsDictionary)
+            {
+                return type.MakeGenericType(new[] { ReplaceUnverifiableType(type.Arguments[0], unknownType), ReplaceUnverifiableType(type.Arguments[1], unknownType) });
+            }
+            // for the other cases, wrap them in a union
+            return CSharpType.FromUnion(new[] { type }, type.IsNullable);
+        }
+
+        private static readonly HashSet<Type> _verifiableTypes = new HashSet<Type>
+        {
+            // The following types are constructed by the `TryGetXXX` methods on `JsonElement`.
+            typeof(byte), typeof(byte[]), typeof(sbyte),
+            typeof(DateTime), typeof(DateTimeOffset),
+            typeof(decimal), typeof(double), typeof(short), typeof(int), typeof(long), typeof(float),
+            typeof(ushort), typeof(uint), typeof(ulong),
+            typeof(Guid),
+            // The following types have a firm JsonValueKind to verify
+            typeof(string), typeof(bool)
+        };
+
+        public static bool IsVerifiableType(Type type) => _verifiableTypes.Contains(type);
+
+        public static bool IsVerifiableType(CSharpType type) => type is { IsFrameworkType: true, FrameworkType: { } frameworkType } && IsVerifiableType(frameworkType);
     }
 }