KQL Search

let sensitiveTools = dynamic([
    "code_interpreter","python","shell","bash","powershell","exec","run_code",
    "http_request","web_request","fetch","invoke_url","curl",
    "send_email","send_message","post_message","slack_post",
    "execute_sql","query_database","run_query",
    "file_write","write_file","upload","copy_file",
    "create_resource","delete_resource","deploy","provision","azure_write"
]);
let toolEvents =
    AppDependencies
    | where TimeGenerated > ago(2h)
    | where isnotempty(Properties["gen_ai.tool.name"])
        and isnotempty(Properties["gen_ai.conversation.id"])
    | extend
        Agent     = tostring(Properties["gen_ai.agent.name"]),
        ConvId    = tostring(Properties["gen_ai.conversation.id"]),
        ProjectId = tostring(Properties["microsoft.foundry.project.id"]),
        ToolName  = tolower(tostring(Properties["gen_ai.tool.name"])),
        ToolType  = tolower(tostring(Properties["gen_ai.tool.type"])),
        Approved  = tobool(coalesce(
                       Properties["gen_ai.tool.user_approved"],
                       Properties["gen_ai.tool.consent.granted"],
                       Properties["microsoft.agent.tool.user_approved"])),
        Required  = tobool(coalesce(
                       Properties["gen_ai.tool.consent.required"],
                       Properties["gen_ai.tool.requires_consent"],
                       Properties["microsoft.agent.tool.requires_consent"]))
    | extend IsSensitive = ToolName has_any (sensitiveTools)
                        or ToolType has_any (sensitiveTools);
let consentFatigue =
    toolEvents
    | where Required == true and Approved == true
    | summarize Approvals = count(),
                Tools     = make_set(ToolName, 16),
                AnyConv   = take_any(ConvId),
                FirstSeen = min(TimeGenerated),
                LastSeen  = max(TimeGenerated)
        by Agent, Bucket = bin(TimeGenerated, 5m)
    | where Approvals >= 5
    | extend Signal = "ConsentFatigue";
let compoundAction =
    toolEvents
    | where IsSensitive
    | summarize Approvals = count(),
                Tools         = make_set(ToolName, 16),
                DistinctTools = dcount(ToolName),
                AnyAgent      = take_any(Agent),
                AnyConv       = take_any(ConvId),
                FirstSeen     = min(TimeGenerated),
                LastSeen      = max(TimeGenerated)
        by ConvId, Bucket = bin(TimeGenerated, 2m)
    | where Approvals >= 3 and DistinctTools >= 2
    | project Agent = AnyAgent, Bucket, FirstSeen, LastSeen, Tools, AnyConv,
              Approvals, Signal = "CompoundActionWithoutPause";
union consentFatigue, compoundAction
| summarize Signals        = make_set(Signal, 4),
            ApprovalsTotal = sum(Approvals),
            ToolsAll       = make_set(Tools, 32),
            SampleConv     = take_any(AnyConv),
            FirstSeen      = min(FirstSeen),
            LastSeen       = max(LastSeen)
        by Agent
| where ApprovalsTotal >= 5
| extend AccountName = iff(isempty(Agent), "unknown-agent", Agent)
| project LastSeen, AccountName, Agent, Signals, ApprovalsTotal, ToolsAll, SampleConv, FirstSeen
| order by ApprovalsTotal desc