Merge branch 'master' into bump-node-16.8.0

.buildkite/scripts/common/env.sh

@@ -56,8 +56,8 @@ else
 fi
 
 # These are for backwards-compatibility
-export GIT_COMMIT="$BUILDKITE_COMMIT"
-export GIT_BRANCH="$BUILDKITE_BRANCH"
+export GIT_COMMIT="${BUILDKITE_COMMIT:-}"
+export GIT_BRANCH="${BUILDKITE_BRANCH:-}"
 
 export FLEET_PACKAGE_REGISTRY_PORT=6104
 export TEST_CORS_SERVER_PORT=6105

.buildkite/scripts/packer_cache.sh

@@ -2,6 +2,7 @@
 
 set -euo pipefail
 
+source .buildkite/scripts/common/util.sh
 source .buildkite/scripts/common/env.sh
 source .buildkite/scripts/common/setup_node.sh
 

docs/user/dashboard/tsvb.asciidoc

@@ -122,7 +122,7 @@ To change this behavior, click *Panel options*, then specify a URL in the *Item
 [[tsvb-markdown]]
 ===== Markdown
 
-The *Markdown* visualization supports Markdown with Handlebar (mustache) syntax to insert dynamic data, and supports custom CSS using the LESS syntax.
+The *Markdown* visualization supports Markdown with Handlebar (mustache) syntax to insert dynamic data, and supports custom CSS.
 
 [float]
 [[tsvb-function-reference]]

package.json

@@ -272,10 +272,8 @@
     "json-stable-stringify": "^1.0.1",
     "json-stringify-pretty-compact": "1.2.0",
     "json-stringify-safe": "5.0.1",
-    "jsonwebtoken": "^8.5.1",
     "jsts": "^1.6.2",
     "kea": "^2.4.2",
-    "less": "npm:@elastic/less@2.7.3-kibana",
     "load-json-file": "^6.2.0",
     "loader-utils": "^1.2.3",
     "lodash": "^4.17.21",
@@ -555,7 +553,6 @@
     "@types/jsdom": "^16.2.3",
     "@types/json-stable-stringify": "^1.0.32",
     "@types/json5": "^0.0.30",
-    "@types/jsonwebtoken": "^8.5.5",
     "@types/license-checker": "15.0.0",
     "@types/listr": "^0.14.0",
     "@types/loader-utils": "^1.1.3",
@@ -663,7 +660,6 @@
     "babel-plugin-styled-components": "^1.13.2",
     "babel-plugin-transform-react-remove-prop-types": "^0.4.24",
     "backport": "^5.6.6",
-    "base64url": "^3.0.1",
     "callsites": "^3.1.0",
     "chai": "3.5.0",
     "chance": "1.0.18",

packages/kbn-cli-dev-mode/src/cli_dev_mode.ts

@@ -289,8 +289,8 @@ export class CliDevMode {
             await reporter.timings({
               timings: [
                 {
-                  group: 'yarn start',
-                  id: 'started',
+                  group: 'scripts/kibana',
+                  id: 'dev server started',
                   ms: Date.now() - this.startTime!,
                   meta: { success },
                 },
@@ -313,7 +313,7 @@ export class CliDevMode {
             await reporter.timings({
               timings: [
                 {
-                  group: 'yarn start',
+                  group: 'scripts/kibana',
                   id: 'dev server restart',
                   ms,
                   meta: {

packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts

@@ -90,20 +90,24 @@ export class CiStatsReporter {
       this.log.debug(e.message);
     }
 
+    const isElasticCommitter = email && email.endsWith('@elastic.co') ? true : false;
+
     const defaultMetadata = {
-      osPlatform: Os.platform(),
-      osRelease: Os.release(),
-      osArch: Os.arch(),
+      committerHash: email
+        ? crypto.createHash('sha256').update(email).digest('hex').substring(0, 20)
+        : undefined,
       cpuCount: Os.cpus()?.length,
       cpuModel: Os.cpus()[0]?.model,
       cpuSpeed: Os.cpus()[0]?.speed,
+      email: isElasticCommitter ? email : undefined,
       freeMem: Os.freemem(),
-      totalMem: Os.totalmem(),
-      committerHash: email
-        ? crypto.createHash('sha256').update(email).digest('hex').substring(0, 20)
-        : undefined,
-      isElasticCommitter: email ? email.endsWith('@elastic.co') : undefined,
+      isElasticCommitter,
       kibanaUuid,
+      nestedTiming: process.env.CI_STATS_NESTED_TIMING ? true : false,
+      osArch: Os.arch(),
+      osPlatform: Os.platform(),
+      osRelease: Os.release(),
+      totalMem: Os.totalmem(),
     };
 
     this.log.debug('CIStatsReporter committerHash: %s', defaultMetadata.committerHash);

packages/kbn-dev-utils/src/run/metrics.ts

@@ -8,6 +8,7 @@
 
 import path from 'path';
 import { REPO_ROOT } from '@kbn/utils';
+import normalizePath from 'normalize-path';
 import { CiStatsReporter } from '../ci_stats_reporter';
 import { ToolingLog } from '../tooling_log';
 
@@ -23,39 +24,39 @@ export class Metrics {
     this.reporter = CiStatsReporter.fromEnv(log);
     this.meta = new Map();
     this.startTime = Date.now();
-    this.filePath = path.relative(REPO_ROOT, process.argv[1]).replace('.js', '');
+
+    // standardize to unix path
+    this.filePath = normalizePath(path.relative(REPO_ROOT, process.argv[1]).replace('.js', ''));
+  }
+
+  createTiming(meta: object, command?: string) {
+    return {
+      group: `${command ? `${this.filePath} ${command}` : this.filePath}`,
+      id: 'total',
+      ms: Date.now() - this.startTime,
+      meta: {
+        nestedTiming: process.env.CI_STATS_NESTED_TIMING,
+        ...Object.fromEntries(this.meta),
+        ...meta,
+      },
+    };
+  }
+
+  async reportCancelled(command?: string) {
+    return await this.reporter.timings({
+      timings: [this.createTiming({ cancelled: true }, command)],
+    });
   }
 
   async reportSuccess(command?: string) {
     return await this.reporter.timings({
-      timings: [
-        {
-          group: `${command ? `${this.filePath} ${command}` : this.filePath}`,
-          id: 'total',
-          ms: Date.now() - this.startTime,
-          meta: {
-            success: true,
-            ...Object.fromEntries(this.meta),
-          },
-        },
-      ],
+      timings: [this.createTiming({ success: true }, command)],
     });
   }
 
   async reportError(errorMessage?: string, command?: string) {
     return await this.reporter.timings({
-      timings: [
-        {
-          group: `${command ? `${this.filePath} ${command}` : this.filePath}`,
-          id: 'total',
-          ms: Date.now() - this.startTime,
-          meta: {
-            success: false,
-            errorMessage,
-            ...Object.fromEntries(this.meta),
-          },
-        },
-      ],
+      timings: [this.createTiming({ success: false, errorMessage }, command)],
     });
   }
 }

packages/kbn-optimizer/src/report_optimizer_timings.ts

@@ -46,7 +46,7 @@ export function reportOptimizerTimings(log: ToolingLog, config: OptimizerConfig)
         await reporter.timings({
           timings: [
             {
-              group: '@kbn/optimizer',
+              group: 'scripts/build_kibana_platform_plugins',
               id: 'total',
               ms: time,
               meta: {

packages/kbn-pm/dist/index.js

@@ -9024,18 +9024,21 @@ class CiStatsReporter {
       this.log.debug(e.message);
     }
 
+    const isElasticCommitter = email && email.endsWith('@elastic.co') ? true : false;
     const defaultMetadata = {
-      osPlatform: _os.default.platform(),
-      osRelease: _os.default.release(),
-      osArch: _os.default.arch(),
+      committerHash: email ? _crypto.default.createHash('sha256').update(email).digest('hex').substring(0, 20) : undefined,
       cpuCount: (_Os$cpus = _os.default.cpus()) === null || _Os$cpus === void 0 ? void 0 : _Os$cpus.length,
       cpuModel: (_Os$cpus$ = _os.default.cpus()[0]) === null || _Os$cpus$ === void 0 ? void 0 : _Os$cpus$.model,
       cpuSpeed: (_Os$cpus$2 = _os.default.cpus()[0]) === null || _Os$cpus$2 === void 0 ? void 0 : _Os$cpus$2.speed,
+      email: isElasticCommitter ? email : undefined,
       freeMem: _os.default.freemem(),
-      totalMem: _os.default.totalmem(),
-      committerHash: email ? _crypto.default.createHash('sha256').update(email).digest('hex').substring(0, 20) : undefined,
-      isElasticCommitter: email ? email.endsWith('@elastic.co') : undefined,
-      kibanaUuid
+      isElasticCommitter,
+      kibanaUuid,
+      nestedTiming: process.env.CI_STATS_NESTED_TIMING ? true : false,
+      osArch: _os.default.arch(),
+      osPlatform: _os.default.platform(),
+      osRelease: _os.default.release(),
+      totalMem: _os.default.totalmem()
     };
     this.log.debug('CIStatsReporter committerHash: %s', defaultMetadata.committerHash);
     return await this.req({
@@ -60653,6 +60656,7 @@ function _defineProperty(obj, key, value) { if (key in obj) { Object.definePrope
 
 
 
+process.env.CI_STATS_NESTED_TIMING = 'true';
 async function runCommand(command, config) {
   const runStartTime = Date.now();
   let kbn;

packages/kbn-pm/src/commands/bootstrap.ts

@@ -145,7 +145,10 @@ export const BootstrapCommand: ICommand = {
       upstreamBranch: kbn.kibanaProject.json.branch,
       // prevent loading @kbn/utils by passing null
       kibanaUuid: kbn.getUuid() || null,
-      timings: timings.map((t) => ({ group: 'scripts/kbn bootstrap', ...t })),
+      timings: timings.map((t) => ({
+        group: 'scripts/kbn bootstrap',
+        ...t,
+      })),
     });
   },
 };

packages/kbn-pm/src/run.ts

@@ -15,6 +15,8 @@ import { buildProjectGraph } from './utils/projects';
 import { renderProjectsTree } from './utils/projects_tree';
 import { Kibana } from './utils/kibana';
 
+process.env.CI_STATS_NESTED_TIMING = 'true';
+
 export async function runCommand(command: ICommand, config: Omit<ICommandConfig, 'kbn'>) {
   const runStartTime = Date.now();
   let kbn;

packages/kbn-test/src/jest/utils/get_url.ts

@@ -22,11 +22,6 @@ interface UrlParam {
   username?: string;
 }
 
-interface App {
-  pathname?: string;
-  hash?: string;
-}
-
 /**
  * Converts a config and a pathname to a url
  * @param {object} config A url config
@@ -46,11 +41,11 @@ interface App {
  * @return {string}
  */
 
-function getUrl(config: UrlParam, app: App) {
+function getUrl(config: UrlParam, app: UrlParam) {
   return url.format(_.assign({}, config, app));
 }
 
-getUrl.noAuth = function getUrlNoAuth(config: UrlParam, app: App) {
+getUrl.noAuth = function getUrlNoAuth(config: UrlParam, app: UrlParam) {
   config = _.pickBy(config, function (val, param) {
     return param !== 'auth';
   });

renovate.json5

@@ -86,10 +86,9 @@
       {
         groupName: 'platform security modules',
         packageNames: [
-          'broadcast-channel', 
-          'jsonwebtoken', '@types/jsonwebtoken', 
-          'node-forge', '@types/node-forge', 
-          'require-in-the-middle', 
+          'broadcast-channel',
+          'node-forge', '@types/node-forge',
+          'require-in-the-middle',
           'tough-cookie', '@types/tough-cookie',
           'xml-crypto', '@types/xml-crypto'
         ],

src/plugins/discover/public/application/apps/main/services/use_saved_search.ts

@@ -6,15 +6,14 @@
  * Side Public License, v 1.
  */
 import { useCallback, useEffect, useMemo, useRef } from 'react';
-import { BehaviorSubject, merge, Subject } from 'rxjs';
-import { debounceTime, filter, tap } from 'rxjs/operators';
+import { BehaviorSubject, Subject } from 'rxjs';
 import { DiscoverServices } from '../../../../build_services';
 import { DiscoverSearchSessionManager } from './discover_search_session';
 import { SearchSource } from '../../../../../../data/common';
 import { GetStateReturn } from './discover_state';
 import { ElasticSearchHit } from '../../../doc_views/doc_views_types';
 import { RequestAdapter } from '../../../../../../inspector/public';
-import { AutoRefreshDoneFn } from '../../../../../../data/public';
+import type { AutoRefreshDoneFn } from '../../../../../../data/public';
 import { validateTimeRange } from '../utils/validate_time_range';
 import { Chart } from '../components/chart/point_series';
 import { useSingleton } from '../utils/use_singleton';
@@ -23,6 +22,7 @@ import { FetchStatus } from '../../../types';
 import { fetchAll } from '../utils/fetch_all';
 import { useBehaviorSubject } from '../utils/use_behavior_subject';
 import { sendResetMsg } from './use_saved_search_messages';
+import { getFetch$ } from '../utils/get_fetch_observable';
 
 export interface SavedSearchData {
   main$: DataMain$;
@@ -134,6 +134,7 @@ export const useSavedSearch = ({
    */
   const refs = useRef<{
     abortController?: AbortController;
+    autoRefreshDone?: AutoRefreshDoneFn;
   }>({});
 
   /**
@@ -145,29 +146,17 @@ export const useSavedSearch = ({
      * handler emitted by `timefilter.getAutoRefreshFetch$()`
      * to notify when data completed loading and to start a new autorefresh loop
      */
-    let autoRefreshDoneCb: AutoRefreshDoneFn | undefined;
-    const fetch$ = merge(
+    const setAutoRefreshDone = (fn: AutoRefreshDoneFn | undefined) => {
+      refs.current.autoRefreshDone = fn;
+    };
+    const fetch$ = getFetch$({
+      setAutoRefreshDone,
+      data,
+      main$,
       refetch$,
-      filterManager.getFetches$(),
-      timefilter.getFetch$(),
-      timefilter.getAutoRefreshFetch$().pipe(
-        tap((done) => {
-          autoRefreshDoneCb = done;
-        }),
-        filter(() => {
-          /**
-           * filter to prevent auto-refresh triggered fetch when
-           * loading is still ongoing
-           */
-          const currentFetchStatus = main$.getValue().fetchStatus;
-          return (
-            currentFetchStatus !== FetchStatus.LOADING && currentFetchStatus !== FetchStatus.PARTIAL
-          );
-        })
-      ),
-      data.query.queryString.getUpdates$(),
-      searchSessionManager.newSearchSessionIdFromURL$.pipe(filter((sessionId) => !!sessionId))
-    ).pipe(debounceTime(100));
+      searchSessionManager,
+      searchSource,
+    });
 
     const subscription = fetch$.subscribe((val) => {
       if (!validateTimeRange(timefilter.getTime(), services.toastNotifications)) {
@@ -190,8 +179,8 @@ export const useSavedSearch = ({
         }).subscribe({
           complete: () => {
             // if this function was set and is executed, another refresh fetch can be triggered
-            autoRefreshDoneCb?.();
-            autoRefreshDoneCb = undefined;
+            refs.current.autoRefreshDone?.();
+            refs.current.autoRefreshDone = undefined;
           },
         });
       } catch (error) {