semgrep_metrics.atd

(* Semgrep metrics.
 *
 * The goal of this module is to specify the schema for the metrics
 * data sent by the Semgrep CLI to https://metrics.semgrep.dev.
 *
 * coupling: with semgrep/PRIVACY.md where we carefully document
 * all the metrics we send with a big table explaining each field.
 *
 * translated originally from semgrep/cli/.../metrics.py
 *
 * TODO:
 * - some optional arguments could be removed when `osemgrep` replace
 *   `pysemgrep`. Currently, we try to fit the tests and don't update the
 *   JSON output.
 * - specify the schema for the answer from https://metrics.semgrep.dev
 *   Here is an example of answer:
 *       { "errorType":"TypeError",
 *         "errorMessage":"Cannot read property 'map' of undefined",
 *          "trace":[
 *             "TypeError: Cannot read property 'map' of undefined",
 *             "    at createPerRuleObjects (/var/task/index.js:287:24)",
 *             "    at Runtime.exports.handler (/var/task/index.js:363:20)",
 *           ]
 *        }
*)

<python text="from dataclasses import field">

(*****************************************************************************)
(* Basic types *)
(*****************************************************************************)

type uuid = string wrap <ocaml module="ATD_string_wrap.Uuidm">

type sha256 = string wrap <ocaml module="ATD_string_wrap.Sha256">

(* RFC 3339 formatted datetime *)
type datetime = string wrap <ocaml module="ATD_string_wrap.Datetime">

type lang = string

(* Note that there is no 'fpath' type here. We don't send concrete filenames *)

(*****************************************************************************)
(* Entry point *)
(*****************************************************************************)

(* The use of 'mutable' below is a bit unusual, but many metrics fields are
 * adjusted in different modules and functions in Semgrep so it was simpler
 * to use a global in Metrics_.ml and mutable fields.
 * See the comments in Metrics_.ml about the global 'g' for more information.
 *)
type payload = {
    (* required event and timing *)
    event_id <ocaml mutable>: uuid;
    (* as stored in ~/.semgrep/settings.yml *)
    anonymous_user_id <python default="''"> <ocaml mutable>: string;
    started_at <python default="Datetime('')"> <ocaml mutable>: datetime;
    sent_at <python default="Datetime('')"> <ocaml mutable>: datetime;

    (* other fields *)
    environment: environment;
    performance: performance;
    extension: extension;
    errors: errors;
    (* TODO? why called value? *)
    value: misc;
    (* the string is a 'lang' below but we can't use the alias because this
     * confuses atd which is checking for a 'string' because of the json
     * repr annotation coming after.
     * TODO? should we move under performance instead of at the top here?
     *)
    ~parse_rate <ocaml mutable>: (string (* lang *) * parse_stat) list
                                 <json repr="object">;

    (* Metrics related to osemgrep migration. It's intentionally made optional
     * so that it's easier to remove the field without breaking backward
     * compatibility once the migration is complete.
     *)
    ?osemgrep <ocaml mutable>: osemgrep_metrics option;
}

(*****************************************************************************)
(* Environment *)
(*****************************************************************************)

type environment = {
    (* semgrep version *)
    version <ocaml mutable>: string;
    (* Since Semgrep 1.59 *)
    os: string; (* e.g. linux *)
    (* Since Semgrep 1.59 *)
    isTranspiledJS <ocaml mutable>: bool;
    projectHash <ocaml mutable>: sha256 nullable;
    configNamesHash <python default="Sha256hash('')"> <ocaml mutable>: sha256;
    ?rulesHash <ocaml mutable>: sha256 option;
    (* TODO: should be just boolean *)
    ci <ocaml mutable>: string nullable;
    (* indicates whether `--baseline-commit` option has been specified. When
       `baseline_commit` option is set, it implies that the scan is operating
       in differential scan mode. Since semgrep 1.46 *)
    ~isDiffScan <python default="False"> <ocaml mutable> : bool;
    ?integrationName <ocaml mutable>: string option;
    ~isAuthenticated <python default="False"> <ocaml mutable>: bool;
  }

(*****************************************************************************)
(* Performance *)
(*****************************************************************************)

type performance = {
    ?numRules <ocaml mutable>: int option;
    ?numTargets <ocaml mutable>: int option;
    ?totalBytesScanned <ocaml mutable>: int option;
    ?fileStats <ocaml mutable>: file_stats list option;
    ?ruleStats <ocaml mutable>: rule_stats list option;
    ?profilingTimes <ocaml mutable>: (string * float) list <json repr="object"> option;
    ?maxMemoryBytes <ocaml mutable>: int option;
  }

type file_stats = {
    size <ocaml mutable>: int;
    numTimesScanned <ocaml mutable>: int;
    (* LATER: factorize with semgrep_output_v1.atd? *)
    ?parseTime <ocaml mutable>: float option;
    ?matchTime <ocaml mutable>: float option;
    ?runTime <ocaml mutable>: float option;
  }

type rule_stats = {
    ruleHash <ocaml mutable>: string;
    bytesScanned <ocaml mutable>: int;
    ?matchTime <ocaml mutable>: float option;
}

(*****************************************************************************)
(* Parsing stats *)
(*****************************************************************************)

type parse_stat = {
    targets_parsed <ocaml mutable>: int;
    num_targets <ocaml mutable>: int;
    bytes_parsed <ocaml mutable>: int;
    num_bytes <ocaml mutable>: int;
}

(*****************************************************************************)
(* Errors *)
(*****************************************************************************)

type errors = {
    ?returnCode <ocaml mutable>: int option;
    ?errors <ocaml mutable>: error list option;
}

(* This is mostly semgrep_output_v1.error_type but in a string form
 * thanks to Error.string_of_error_type.
 * TODO: use directly semgrep_output_v1.error_type but need ATD modules.
 *)
type error = string

(*****************************************************************************)
(* Extension *)
(*****************************************************************************)

type extension = {
    ?machineId <ocaml mutable>: string option;
    ?isNewAppInstall <ocaml mutable>: bool option;
    ?sessionId <ocaml mutable>: string option;
    ?version <ocaml mutable>: string option;
    ?ty <ocaml mutable>: string option;
}

(*****************************************************************************)
(* Misc *)
(*****************************************************************************)

type pro_features = {
    (* The value is determined by the `--diff-depth` option, and it's utilized
       for inter-file differential scanning. Since semgrep 1.46 *)
    ?diffDepth <ocaml mutable>: int option;
    (* The number of scanned files per language for inter-file diff scan mode.
       This number represents the count of changed files and their limited
       dependencies. Since semgrep 1.70 *)
    ?numInterfileDiffScanned <ocaml mutable>: (string (* lang *) * int) list
                                              <json repr="object"> option;
}

type analysis_type <ocaml attr="deriving show"> = [ 
    | Intraprocedural 
    | Interprocedural 
    | Interfile
]

(* Ideally this and supply_chain_config would be unit or empty records,  but
 * atd doesn't generate correct code for python when unit is aliased
 * (to_json/from_json are wrong and try to call methods on a fake Unit class).
 * In the alternative, {} doesn't work either since atd just passes it through
 * instead of converting to unit and it is not valid OCaml syntax.
 *
 * So we have a "reserved for future use" (RFU) field just to act as a
 * placeholder.
 *)
type code_config  <ocaml attr="deriving show"> = { ?_rfu: int option; }

type secrets_origin <ocaml attr="deriving show"> = [ Any | Semgrep | NoCommunity ]
type secrets_config 
     <ocaml attr="deriving show"> = {
  permitted_origins: secrets_origin;
}

type supply_chain_config  <ocaml attr="deriving show"> = { ?_rfu: int option; }

(* Since v1.55.0 *)
type engine_config
     <ocaml attr="deriving show"> = {
  analysis_type: analysis_type;
  pro_langs: bool;
  (* `Some c` where `c` is the config if the product was run.
   * `None` if it was not run.
   *)
  ?code_config: code_config option;
  ?secrets_config: secrets_config option;
  ?supply_chain_config: supply_chain_config option;
}

type misc = {
    (* coupling: features is commented a lot in semgrep/PRIVACY.md *)
    features <ocaml mutable>: string list;
    (* Since semgrep 1.46 *)
    ?proFeatures <ocaml mutable>: pro_features option;
    ?numFindings <ocaml mutable>: int option;
    (* Since Semgrep 1.56.0, see PA-3312 *)
    ?numFindingsByProduct <ocaml mutable>: (string * int) list <json repr="object"> option;
    ?numIgnored <ocaml mutable>: int option;
    ?ruleHashesWithFindings <ocaml mutable>: (string * int) list <json repr="object"> option;
    (* TODO: should be OSS | Pro, see semgrep_output_v1.atd engine_kind type *)
    ~engineRequested <python default="'OSS'"> <ocaml mutable>: string;
    (* Since Semgrep 1.54.0 *)
    ?engineConfig <ocaml mutable>: engine_config option;
    (* Since Semgrep 1.49.0 *)
    ?interfileLanguagesUsed: string list option;
  }

(*****************************************************************************)
(* Osemgrep specific metrics *)
(*****************************************************************************)

type osemgrep_format_output = {
  (* Whether the RPC succeeded. *)
  ?succeeded: bool option;
  (* Name of the output format. *)
  ?format: string option;
  (* Time in seconds to call osemgrep through RPC and get its result back. *)
  ?osemgrep_rpc_response_time_seconds: float option;
  (* Time in seconds for osemgrep to format the output, excluding RPC. *)
  ?osemgrep_format_time_seconds: float option;

  (* We also validate whether osemgrep and pysemgrep return the same output or not.
   * The fields below help us keep track of this information.
   *)

  (* Time in seconds for pysemgrep to format the output. *)
  ?pysemgrep_format_time_seconds: float option;
  (* Time in seconds to validate whether the output from osemgrep matches pysemgrep. *)
  ?validation_time_seconds: float option;
  (* Whether the osemgrep output matched the pysemgrep output. *)
  ?is_match: bool option;
}

type osemgrep_metrics = {
  ?format_output: osemgrep_format_output option;
}

(*****************************************************************************)
(* TODO Response by metrics.semgrep.dev *)
(*****************************************************************************)