Multiple IP Entity Device Network Events

Query

// This query assumes a feed of threat indicators is ingested/synchronized periodically, and each synchronization ingests new indicators and only old indicators that have been modified.
// Active threat indicators in Sentinel are renovated as ThreatIntelligenceIndicator events every ~12 days.
let query_frequency = 1h;
let query_period = 14d;
let query_wait = 0h;
let table_query_lookback = 14d;
let _TIBenignProperty =
    _GetWatchlist('ID-TIBenignProperty')
    | where Notes has_any ("[DestinationIPAddress]")
    | project IndicatorId, BenignProperty
;
let _TIExcludedSources = toscalar(
    _GetWatchlist('Activity-ExpectedSignificantActivity')
    | where Activity == "ThreatIndicatorSource"
    | summarize make_list(Auxiliar)
    );
let _TITableMatch = (table_start: datetime, table_end: datetime, only_new_ti: boolean, ti_start: datetime = datetime(null)) {
    // Scheduled Analytics rules have a query period limit of 14d
    let _Indicators =// materialize(
        ThreatIntelligenceIndicator
        | where TimeGenerated > ago(query_period)
        // Take the earliest TimeGenerated and the latest column info
        | summarize hint.strategy=shuffle
            minTimeGenerated = min(TimeGenerated),
            arg_max(TimeGenerated, Active, Description, ActivityGroupNames, IndicatorId, ThreatType, DomainName, Url, ExpirationDateTime, ConfidenceScore, SourceSystem, Tags, AdditionalInformation, ExternalIndicatorId, NetworkIP, NetworkSourceIP, NetworkDestinationIP, EmailSourceIpAddress)
            by IndicatorId
        // Remove inactive or expired indicators
        | where not(not(Active) or ExpirationDateTime < now())
        // Pick indicators that contain the desired entity type
        | mv-expand IPAddress = pack_array(NetworkIP, NetworkSourceIP, NetworkDestinationIP, EmailSourceIpAddress) to typeof(string)
        | where isnotempty(IPAddress)
        | extend TI_IPAddress = IPAddress
        // Remove indicators from specific sources
        | where not(AdditionalInformation has_any (_TIExcludedSources) or Description has_any (_TIExcludedSources))
        // Remove excluded indicators with benign properties
        | join kind=leftanti _TIBenignProperty on IndicatorId, $left.IPAddress == $right.BenignProperty
        // Deduplicate indicators by IPAddress column, equivalent to using join kind=innerunique afterwards
        | summarize hint.strategy=shuffle
            minTimeGenerated = min(minTimeGenerated),
            take_any(*)
            by IPAddress
        // If we want only new indicators, remove indicators received previously
        | where not(only_new_ti and minTimeGenerated < ti_start)
    //)
    ;
    //let _IndicatorsLength = toscalar(_Indicators | summarize count());
    //let _IndicatorsPrefilter = toscalar(
    //    _Indicators
    //    | extend AuxiliarField = tostring(extract(@"([0-9A-Za-f]+)[\.\:]", 1, IPAddress))
    //    | summarize make_set_if(AuxiliarField, isnotempty(AuxiliarField), 10000)
    //);
    //let _IndicatorsPrefilterLength = array_length(_IndicatorsPrefilter);
    let _TableEvents =
        DeviceNetworkEvents
        | where ingestion_time() between (table_start .. table_end)
        // Filter events that may contain indicators
        | where isnotempty(RemoteIP)
            and not(ActionType == "NetworkSignatureInspected")
            and not(RemoteIPType in ("Private", "Loopback", "Teredo", "Broadcast"))//Reserved)
            or (isnotempty(AdditionalFields["answers"]) and AdditionalFields["rcode"] == 0 and (AdditionalFields["qtype_name"] in ("", "A", "AAAA")))
        //| where not(_IndicatorsPrefilterLength < 10000 and not(RemoteIP has_any (_IndicatorsPrefilter))) // "has_any" limit 10000
        | mv-expand IPAddress = case(
                            isnotempty(RemoteIP),
                            pack_array(iff(RemoteIPType == "FourToSixMapping", trim_start("::ffff:", RemoteIP), RemoteIP)),
                            AdditionalFields["answers"]
                        ) to typeof(string)
        | where isnotempty(parse_ipv4(IPAddress)) or isnotempty(parse_ipv6(IPAddress))
        | where not(isnotempty(parse_ipv4(IPAddress)) and (ipv4_is_private(IPAddress) or ipv4_is_in_any_range(IPAddress, "0.0.0.0/8", "127.0.0.0/8")))
        //| where not(_IndicatorsLength < 1000000 and not(IPAddress in (toscalar(_Indicators | summarize make_list(TI_IPAddress))))) // "in" limit 1.000.000
        | project-rename DeviceNetworkEvents_TimeGenerated = TimeGenerated
    ;
    _Indicators
    | join kind=inner hint.strategy=shuffle _TableEvents on IPAddress
    // Take only a single event by key columns
    //| summarize hint.strategy=shuffle take_any(*) by IPAddress, DeviceName
    //| where not(todynamic(AdditionalInformation)["value_type"] == "domain" and not(DomainName has RemoteUrl or RemoteUrl has DomainName))
    //| where not(todynamic(AdditionalInformation)["value_type"] == "url" and not(trim_end(@"\/", Url) endswith trim_end(@"\/", RemoteUrl)))
    //| where not(todynamic(AdditionalInformation)["value_type"] in ("domain", "url") and isempty(RemoteUrl))
    | project
        DeviceNetworkEvents_TimeGenerated,
        Description,
        ActivityGroupNames,
        IndicatorId,
        ThreatType,
        DomainName,
        Url,
        ExpirationDateTime,
        ConfidenceScore,
        SourceSystem,
        Tags,
        AdditionalInformation,
        TI_IPAddress,
        NetworkIP,
        NetworkSourceIP,
        NetworkDestinationIP,
        EmailSourceIpAddress,
        DeviceName,
        LocalIP,
        ActionType,
        RemoteUrl,
        RemoteIP,
        RemotePort,
        Protocol,
        LocalPort,
        AdditionalFields,
        InitiatingProcessCommandLine,
        InitiatingProcessFolderPath,
        InitiatingProcessAccountName,
        InitiatingProcessAccountUpn
};
union// isfuzzy=true
    // Match      current table events                                all indicators available
    _TITableMatch(ago(query_frequency + query_wait), ago(query_wait), false),
    // Match      past table events                                                          new indicators since last query execution
    _TITableMatch(ago(table_query_lookback + query_wait), ago(query_frequency + query_wait), true, ago(query_frequency))
| summarize arg_max(DeviceNetworkEvents_TimeGenerated, *) by IndicatorId, DeviceName
| extend
    timestamp = DeviceNetworkEvents_TimeGenerated,
    HostCustomEntity = DeviceName,
    AccountCustomEntity = InitiatingProcessAccountUpn,
    IPCustomEntity = TI_IPAddress

Explanation

This query is designed to identify and correlate network events with threat intelligence indicators in Microsoft Sentinel. Here's a simplified breakdown of what it does:

Setup and Parameters:
- The query is set to run every hour (query_frequency = 1h) and looks back over a 14-day period (query_period = 14d).
- It uses watchlists to filter out benign properties and exclude certain sources of threat indicators.
Threat Intelligence Indicators:
- It retrieves active threat indicators from the ThreatIntelligenceIndicator table that have been updated or are new within the last 14 days.
- It filters out inactive or expired indicators and those from excluded sources.
- It further refines the list by removing indicators with benign properties.
Network Events:
- It gathers network events from the DeviceNetworkEvents table within the specified time range.
- It filters these events to exclude certain types of IP addresses (e.g., private, loopback) and specific actions.
Correlation:
- The query correlates the filtered threat indicators with the network events by matching IP addresses.
- It ensures that only relevant events are considered by joining the two datasets.
Output:
- The result is a list of network events that match the threat indicators, providing details like the device name, IP addresses, threat description, and other relevant information.
- It also includes custom entities for further analysis in Sentinel.

In essence, this query helps security analysts identify potential threats by correlating network activity with known threat indicators, while filtering out noise from benign or expected sources.

Details

Jose Sebastián Canós

Released: April 1, 2024

Tables

ThreatIntelligenceIndicatorDeviceNetworkEvents

Keywords

ThreatIntelligenceIndicatorDeviceNetworkEvents

Operators

lethas_anyprojecttoscalarwheresummarizemake_listarg_maxmv-expandisnotemptyextendjoinleftantiproject-renamebetweenorcaseifftrim_startparse_ipv4parse_ipv6ipv4_is_privateipv4_is_in_any_rangeunionago

Actions

GitHub

KQL Search