KQL Search

id: 6f1a2b3c-4d5e-4f10-9301-aaaaaaaaaaa1
name: Foundry - Human-in-the-loop / consent bypass pattern
description: |
  Detects two patterns the Microsoft AI Red Team v2.0 taxonomy flags as
  the most consistently exploited failure mode: consent fatigue (a user
  approving many sensitive tool calls in a short window without time to
  read them) and compound action chains (several sensitive tools fired
  in seconds inside one conversation, where no individual step clearly
  warranted review but the compound outcome did).

  The rule probes both explicit consent telemetry
  (gen_ai.tool.consent.required / gen_ai.tool.user_approved) and a
  fallback heuristic over sensitive tool types (code interpreter, shell,
  http, email, sql, file write, resource create / delete) so it still
  fires when the agent platform has not yet wired consent fields.

  Thresholds are conservative to minimise false positives:
    - ConsentFatigue: >= 5 consent-required tool calls approved by the
      same agent inside a 5-minute bucket.
    - CompoundActionWithoutPause: >= 3 sensitive tool calls AND >= 2
      distinct sensitive tools inside a 2-minute bucket per conversation.
  The rule only emits when total approvals across signals reach >= 5,
  which suppresses single-step legitimate use.
severity: High
requiredDataConnectors:
- connectorId: ApplicationInsights
  dataTypes:
  - AppDependencies
queryFrequency: PT1H
queryPeriod: PT2H
triggerOperator: gt
triggerThreshold: 0
enabled: true
tactics:
- Execution
- PrivilegeEscalation
relevantTechniques:
- T1059
- T1098
query: |
  let sensitiveTools = dynamic([
      "code_interpreter","python","shell","bash","powershell","exec","run_code",
      "http_request","web_request","fetch","invoke_url","curl",
      "send_email","send_message","post_message","slack_post",
      "execute_sql","query_database","run_query",
      "file_write","write_file","upload","copy_file",
      "create_resource","delete_resource","deploy","provision","azure_write"
  ]);
  let toolEvents =
      AppDependencies
      | where TimeGenerated > ago(2h)
      | where isnotempty(Properties["gen_ai.tool.name"])
          and isnotempty(Properties["gen_ai.conversation.id"])
      | extend
          Agent     = tostring(Properties["gen_ai.agent.name"]),
          ConvId    = tostring(Properties["gen_ai.conversation.id"]),
          ProjectId = tostring(Properties["microsoft.foundry.project.id"]),
          ToolName  = tolower(tostring(Properties["gen_ai.tool.name"])),
          ToolType  = tolower(tostring(Properties["gen_ai.tool.type"])),
          Approved  = tobool(coalesce(
                         Properties["gen_ai.tool.user_approved"],
                         Properties["gen_ai.tool.consent.granted"],
                         Properties["microsoft.agent.tool.user_approved"])),
          Required  = tobool(coalesce(
                         Properties["gen_ai.tool.consent.required"],
                         Properties["gen_ai.tool.requires_consent"],
                         Properties["microsoft.agent.tool.requires_consent"]))
      | extend IsSensitive = ToolName has_any (sensitiveTools)
                          or ToolType has_any (sensitiveTools);
  let consentFatigue =
      toolEvents
      | where Required == true and Approved == true
      | summarize Approvals = count(),
                  Tools     = make_set(ToolName, 16),
                  AnyConv   = take_any(ConvId),
                  FirstSeen = min(TimeGenerated),
                  LastSeen  = max(TimeGenerated)
          by Agent, Bucket = bin(TimeGenerated, 5m)
      | where Approvals >= 5
      | extend Signal = "ConsentFatigue";
  let compoundAction =
      toolEvents
      | where IsSensitive
      | summarize Approvals = count(),
                  Tools         = make_set(ToolName, 16),
                  DistinctTools = dcount(ToolName),
                  AnyAgent      = take_any(Agent),
                  AnyConv       = take_any(ConvId),
                  FirstSeen     = min(TimeGenerated),
                  LastSeen      = max(TimeGenerated)
          by ConvId, Bucket = bin(TimeGenerated, 2m)
      | where Approvals >= 3 and DistinctTools >= 2
      | project Agent = AnyAgent, Bucket, FirstSeen, LastSeen, Tools, AnyConv,
                Approvals, Signal = "CompoundActionWithoutPause";
  union consentFatigue, compoundAction
  | summarize Signals        = make_set(Signal, 4),
              ApprovalsTotal = sum(Approvals),
              ToolsAll       = make_set(Tools, 32),
              SampleConv     = take_any(AnyConv),
              FirstSeen      = min(FirstSeen),
              LastSeen       = max(LastSeen)
          by Agent
  | where ApprovalsTotal >= 5
  | extend AccountName = iff(isempty(Agent), "unknown-agent", Agent)
  | project LastSeen, AccountName, Agent, Signals, ApprovalsTotal, ToolsAll, SampleConv, FirstSeen
  | order by ApprovalsTotal desc
entityMappings:
- entityType: Account
  fieldMappings:
  - identifier: Name
    columnName: AccountName
eventGroupingSettings:
  aggregationKind: SingleAlert
incidentConfiguration:
  createIncident: true
  groupingConfiguration:
    enabled: true
    reopenClosedIncident: false
    lookbackDuration: PT12H
    matchingMethod: Selected
    groupByEntities:
    - Account
    groupByAlertDetails: []
    groupByCustomDetails: []
version: 1.0.0
kind: Scheduled
tags:
- Sentinel-As-Code
- Custom
- Foundry
- AI
- HitL
- ConsentBypass
- AIRT-v2