[data views] Allow data views created on hidden and system indices - …

…second attempt (#168882)

## Summary

Previously, the 'Allow hidden and system indices' advanced option when
creating a data view was only a UI convenience. It allowed you to see
which hidden and system indices you were matching but they would be
would be selected just the same once the data view was loaded. At some
point something changed and now there are system and hidden indices that
require `expandWildcards: hidden` to be passed to field caps in order to
see anything. `allowHidden: boolean` is added to the DataView and
DataViewSpec and passed through when field caps requests are made.

This is primarily a tool for troubleshooting. For instance, instead of
hitting a full data stream across a number of data tiers you can select
a specific index to compare its performance.

NOTE: This is a second attempt. What I learned - the whole
`expand_wildcards` param is literal - you can directly query a hidden
index without `expandWildcards: hidden` since its not using a wildcard.
Tests now use a wildcard.

Closes: #164652

---------

Co-authored-by: Lukas Olson <lukas@elastic.co>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

src/plugins/data/common/search/search_source/fetch/get_search_params.test.ts

@@ -9,6 +9,7 @@
 import { UI_SETTINGS } from '../../../constants';
 import { GetConfigFn } from '../../../types';
 import { getSearchParams, getSearchParamsFromRequest } from './get_search_params';
+import { createStubDataView } from '@kbn/data-views-plugin/common/data_views/data_view.stub';
 
 function getConfigStub(config: any = {}): GetConfigFn {
   return (key) => config[key];
@@ -46,4 +47,50 @@ describe('getSearchParams', () => {
       query: 123,
     });
   });
+
+  test('sets expand_wildcards=all if data view has allowHidden=true', () => {
+    const getConfig = getConfigStub({
+      [UI_SETTINGS.COURIER_SET_REQUEST_PREFERENCE]: 'custom',
+      [UI_SETTINGS.COURIER_CUSTOM_REQUEST_PREFERENCE]: 'aaa',
+    });
+    const index = createStubDataView({
+      spec: {
+        allowHidden: true,
+      },
+    });
+    const searchParams = getSearchParamsFromRequest(
+      {
+        index,
+        body: {
+          query: 123,
+          track_total_hits: true,
+        },
+      },
+      { getConfig }
+    );
+    expect(searchParams).toHaveProperty('expand_wildcards', 'all');
+  });
+
+  test('does not set expand_wildcards if data view has allowHidden=false', () => {
+    const getConfig = getConfigStub({
+      [UI_SETTINGS.COURIER_SET_REQUEST_PREFERENCE]: 'custom',
+      [UI_SETTINGS.COURIER_CUSTOM_REQUEST_PREFERENCE]: 'aaa',
+    });
+    const index = createStubDataView({
+      spec: {
+        allowHidden: false,
+      },
+    });
+    const searchParams = getSearchParamsFromRequest(
+      {
+        index,
+        body: {
+          query: 123,
+          track_total_hits: true,
+        },
+      },
+      { getConfig }
+    );
+    expect(searchParams).not.toHaveProperty('expand_wildcards', 'all');
+  });
 });

src/plugins/data/common/search/search_source/fetch/get_search_params.ts

@@ -42,8 +42,8 @@ export function getSearchParamsFromRequest(
   return {
     index: searchRequest.index.title || searchRequest.index,
     body,
-    // @ts-expect-error `track_total_hits` not allowed at top level for `typesWithBodyKey`
     track_total_hits,
+    ...(searchRequest.index?.allowHidden && { expand_wildcards: 'all' }),
     ...searchParams,
   };
 }

src/plugins/data_view_editor/public/components/advanced_params_content/advanced_params_content.tsx

@@ -30,14 +30,16 @@ interface AdvancedParamsContentProps {
   disableAllowHidden: boolean;
   disableId: boolean;
   onAllowHiddenChange?: (value: boolean) => void;
+  defaultVisible?: boolean;
 }
 
 export const AdvancedParamsContent = ({
   disableAllowHidden,
   disableId,
   onAllowHiddenChange,
+  defaultVisible = false,
 }: AdvancedParamsContentProps) => (
-  <AdvancedParamsSection>
+  <AdvancedParamsSection defaultVisible={defaultVisible}>
     <EuiFlexGroup>
       <EuiFlexItem>
         <UseField<boolean, IndexPatternConfig>

src/plugins/data_view_editor/public/components/advanced_params_content/advanced_params_section.tsx

@@ -13,10 +13,11 @@ import { EuiButtonEmpty, EuiSpacer } from '@elastic/eui';
 
 interface Props {
   children: React.ReactNode;
+  defaultVisible: boolean;
 }
 
-export const AdvancedParamsSection = ({ children }: Props) => {
-  const [isVisible, setIsVisible] = useState<boolean>(false);
+export const AdvancedParamsSection = ({ children, defaultVisible = false }: Props) => {
+  const [isVisible, setIsVisible] = useState<boolean>(defaultVisible);
 
   const toggleIsVisible = useCallback(() => {
     setIsVisible(!isVisible);

src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx

@@ -105,6 +105,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
             title: editData.getIndexPattern(),
             id: editData.id,
             name: editData.name,
+            allowHidden: editData.getAllowHidden(),
             ...(editData.timeFieldName
               ? {
                   timestampField: { label: editData.timeFieldName, value: editData.timeFieldName },
@@ -124,6 +125,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
         timeFieldName: formData.timestampField?.value,
         id: formData.id,
         name: formData.name,
+        allowHidden: formData.allowHidden,
       };
 
       if (type === INDEX_PATTERN_TYPE.ROLLUP && rollupIndex) {
@@ -293,6 +295,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
             onAllowHiddenChange={() => {
               form.getFields().title.validate();
             }}
+            defaultVisible={editData?.getAllowHidden()}
           />
         </Form>
         <Footer

src/plugins/data_view_editor/public/components/data_view_flyout_content_container.tsx

@@ -50,10 +50,11 @@ const DataViewFlyoutContentContainer = ({
     try {
       let saveResponse;
       if (editData) {
-        const { name = '', timeFieldName, title = '' } = dataViewSpec;
+        const { name = '', timeFieldName, title = '', allowHidden = false } = dataViewSpec;
         editData.setIndexPattern(title);
         editData.name = name;
         editData.timeFieldName = timeFieldName;
+        editData.setAllowHidden(allowHidden);
         saveResponse = editData.isPersisted()
           ? await dataViews.updateSavedObject(editData)
           : editData;

src/plugins/data_view_editor/public/data_view_editor_service.ts

@@ -342,6 +342,7 @@ export class DataViewEditorService {
 
     const getFieldsOptions: GetFieldsOptions = {
       pattern: this.indexPattern,
+      allowHidden: this.allowHidden,
     };
     if (this.type === INDEX_PATTERN_TYPE.ROLLUP) {
       getFieldsOptions.type = INDEX_PATTERN_TYPE.ROLLUP;

src/plugins/data_views/common/content_management/v1/cm_services.ts

@@ -46,6 +46,7 @@ const dataViewAttributesSchema = schema.object(
     allowNoIndex: schema.maybe(schema.boolean()),
     runtimeFieldMap: schema.maybe(schema.any()),
     name: schema.maybe(schema.string()),
+    allowHidden: schema.maybe(schema.boolean()),
   },
   { unknowns: 'forbid' }
 );

src/plugins/data_views/common/data_views/abstract_data_views.ts

@@ -126,6 +126,8 @@ export abstract class AbstractDataView {
 
   protected scriptedFields: DataViewFieldBase[];
 
+  private allowHidden: boolean = false;
+
   constructor(config: AbstractDataViewDeps) {
     const { spec = {}, fieldFormats, shortDotsEnable = false, metaFields = [] } = config;
 
@@ -178,8 +180,13 @@ export abstract class AbstractDataView {
     this.runtimeFieldMap = cloneDeep(spec.runtimeFieldMap) || {};
     this.namespaces = spec.namespaces || [];
     this.name = spec.name || '';
+    this.allowHidden = spec.allowHidden || false;
   }
 
+  getAllowHidden = () => this.allowHidden;
+
+  setAllowHidden = (allowHidden: boolean) => (this.allowHidden = allowHidden);
+
   /**
    * Get name of Data View
    */
@@ -325,6 +332,7 @@ export abstract class AbstractDataView {
       allowNoIndex: this.allowNoIndex ? this.allowNoIndex : undefined,
       runtimeFieldMap: stringifyOrUndefined(this.runtimeFieldMap),
       name: this.name,
+      allowHidden: this.allowHidden,
     };
   }
 

src/plugins/data_views/common/data_views/data_view.ts

@@ -160,6 +160,7 @@ export class DataView extends AbstractDataView implements DataViewBase {
       fieldAttrs,
       allowNoIndex: this.allowNoIndex,
       name: this.name,
+      allowHidden: this.getAllowHidden(),
     };
 
     // Filter undefined values from the spec

src/plugins/data_views/common/data_views/data_views.ts

@@ -531,6 +531,8 @@ export class DataViewsService {
       allowNoIndex: true,
       ...options,
       pattern: indexPattern.title as string,
+      allowHidden:
+        (indexPattern as DataViewSpec).allowHidden || (indexPattern as DataView)?.getAllowHidden(),
     });
 
   private getFieldsAndIndicesForDataView = async (dataView: DataView) => {
@@ -541,6 +543,7 @@ export class DataViewsService {
       allowNoIndex: true,
       pattern: dataView.getIndexPattern(),
       metaFields,
+      allowHidden: dataView.getAllowHidden(),
     });
   };
 
@@ -553,6 +556,7 @@ export class DataViewsService {
       rollupIndex: options.rollupIndex,
       allowNoIndex: true,
       indexFilter: options.indexFilter,
+      allowHidden: options.allowHidden,
     });
   };
 
@@ -704,6 +708,7 @@ export class DataViewsService {
         fieldAttrs,
         allowNoIndex,
         name,
+        allowHidden,
       },
     } = savedObject;
 
@@ -731,6 +736,7 @@ export class DataViewsService {
       allowNoIndex,
       runtimeFieldMap: parsedRuntimeFieldMap,
       name,
+      allowHidden,
     };
   };
 
@@ -763,6 +769,7 @@ export class DataViewsService {
         type,
         rollupIndex: typeMeta?.params?.rollup_index,
         allowNoIndex: spec.allowNoIndex,
+        allowHidden: spec.allowHidden,
       },
       spec.fieldAttrs,
       displayErrors

src/plugins/data_views/common/types.ts

@@ -157,6 +157,10 @@ export interface DataViewAttributes {
    * Name of the data view. Human readable name used to differentiate data view.
    */
   name?: string;
+  /**
+   * Allow hidden and system indices when loading field list
+   */
+  allowHidden?: boolean;
 }
 
 /**
@@ -309,6 +313,7 @@ export interface GetFieldsOptions {
   indexFilter?: QueryDslQueryContainer;
   includeUnmapped?: boolean;
   fields?: string[];
+  allowHidden?: boolean;
 }
 
 /**
@@ -517,6 +522,10 @@ export type DataViewSpec = {
    * Name of the data view. Human readable name used to differentiate data view.
    */
   name?: string;
+  /**
+   * Allow hidden and system indices when loading field list
+   */
+  allowHidden?: boolean;
 };
 
 // eslint-disable-next-line @typescript-eslint/consistent-type-definitions

src/plugins/data_views/public/data_views/data_views_api_client.ts

@@ -60,6 +60,7 @@ export class DataViewsApiClient implements IDataViewsApiClient {
       indexFilter,
       includeUnmapped,
       fields,
+      allowHidden,
     } = options;
     return this._request<FieldsForWildcardResponse>(
       FIELDS_FOR_WILDCARD_PATH,
@@ -71,6 +72,7 @@ export class DataViewsApiClient implements IDataViewsApiClient {
         allow_no_index: allowNoIndex,
         include_unmapped: includeUnmapped,
         fields,
+        allow_hidden: allowHidden,
       },
       indexFilter ? JSON.stringify({ index_filter: indexFilter }) : undefined
     ).then((response) => {

src/plugins/data_views/server/content_management/data_views_storage.ts

@@ -37,6 +37,7 @@ export class DataViewsStorage extends SOContentStorage<DataViewCrudTypes> {
         'runtimeFieldMap',
         'allowNoIndex',
         'name',
+        'allowHidden',
       ],
       mSearchAdditionalSearchFields: ['name'],
       logger,

src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts

@@ -71,12 +71,23 @@ export class IndexPatternsFetcher {
     rollupIndex?: string;
     indexFilter?: QueryDslQueryContainer;
     fields?: string[];
+    allowHidden?: boolean;
   }): Promise<{ fields: FieldDescriptor[]; indices: string[] }> {
-    const { pattern, metaFields = [], fieldCapsOptions, type, rollupIndex, indexFilter } = options;
+    const {
+      pattern,
+      metaFields = [],
+      fieldCapsOptions,
+      type,
+      rollupIndex,
+      indexFilter,
+      allowHidden,
+    } = options;
     const allowNoIndices = fieldCapsOptions
       ? fieldCapsOptions.allow_no_indices
       : this.allowNoIndices;
 
+    const expandWildcards = allowHidden ? 'all' : 'open';
+
     const fieldCapsResponse = await getFieldCapabilities({
       callCluster: this.elasticsearchClient,
       indices: pattern,
@@ -87,6 +98,7 @@ export class IndexPatternsFetcher {
       },
       indexFilter,
       fields: options.fields || ['*'],
+      expandWildcards,
     });
 
     if (this.rollupsEnabled && type === 'rollup' && rollupIndex) {

src/plugins/data_views/server/fetcher/lib/es_api.ts

@@ -7,6 +7,7 @@
  */
 
 import { ElasticsearchClient } from '@kbn/core/server';
+import { ExpandWildcard } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { QueryDslQueryContainer } from '../../../common/types';
 import { convertEsError } from './errors';
 
@@ -45,6 +46,7 @@ interface FieldCapsApiParams {
   fieldCapsOptions?: { allow_no_indices: boolean; include_unmapped?: boolean };
   indexFilter?: QueryDslQueryContainer;
   fields?: string[];
+  expandWildcards?: ExpandWildcard;
 }
 
 /**
@@ -69,6 +71,7 @@ export async function callFieldCapsApi(params: FieldCapsApiParams) {
       include_unmapped: false,
     },
     fields = ['*'],
+    expandWildcards,
   } = params;
   try {
     return await callCluster.fieldCaps(
@@ -77,6 +80,7 @@ export async function callFieldCapsApi(params: FieldCapsApiParams) {
         fields,
         ignore_unavailable: true,
         index_filter: indexFilter,
+        expand_wildcards: expandWildcards,
         ...fieldCapsOptions,
       },
       { meta: true }