description: Pipeline for parsing MISP Threat Intel
processors:

####################
# Event ECS fields #
####################
- set:
    field: event.ingested
    value: '{{_ingest.timestamp}}'
- set:
    field: event.kind
    value: enrichment
- set:
    field: event.category
    value: threat
- set:
    field: event.type
    value: indicator

######################
# General ECS fields #
######################
- rename:
    field: json.Event
    target_field: threatintel.misp
    ignore_missing: true
- set:
    field: threatintel.indicator.provider
    value: misp
    if: ctx?.threatintel?.misp?.Orgc?.local != 'false'
- set:
    field: threatintel.indicator.provider
    value: '{{misp.Orgc.name}}'
    if: ctx?.threatintel?.misp?.Orgc?.local == 'false'
    ignore_empty_value: true

# Removing fields not needed anymore, either because its copied somewhere else, or is not relevant to this event
- remove:
    field:
    - threatintel.misp.ShadowAttribute
    - threatintel.misp.RelatedEvent
    - threatintel.misp.Galaxy
    - threatintel.misp.Attribute.Galaxy
    - threatintel.misp.Attribute.ShadowAttribute
    - threatintel.misp.Object
    - threatintel.misp.EventReport
    - message
    ignore_missing: true
- date:
    field: threatintel.misp.timestamp
    formats:
      - UNIX
    ignore_failure: true
- rename:
    field: threatintel.misp.Attribute
    target_field: threatintel.misp.attribute
    ignore_missing: true
- rename:
    field: threatintel.misp.Orgc
    target_field: threatintel.misp.orgc
    ignore_missing: true
- rename:
    field: threatintel.misp.Org
    target_field: threatintel.misp.org
    ignore_missing: true
- rename:
    field: threatintel.misp.Attribute
    target_field: threatintel.misp.attribute
    ignore_missing: true
- rename:
    field: threatintel.misp.Tag
    target_field: threatintel.misp.tag
    ignore_missing: true

#####################
# Threat ECS Fields #
#####################
- rename:
    field: threatintel.misp.attribute.first_seen
    target_field: threatintel.indicator.first_seen
    ignore_missing: true
- rename:
    field: threatintel.misp.attribute.last_seen
    target_field: threatintel.indicator.last_seen
    ignore_missing: true
- convert:
    field: threatintel.misp.analysis
    type: long
    target_field: threatintel.indicator.scanner_stats
    ignore_missing: true
- convert:
    field: threatintel.misp.threat_level_id
    type: long
    ignore_missing: true

## File/Hash indicator operations
- set:
    field: threatintel.indicator.type
    value: file
    if: "['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.threatintel?.misp?.attribute?.type) || ctx.threatintel?.misp?.attribute?.type.startsWith('filename')"
- rename:
    field: threatintel.misp.attribute.value
    target_field: "threatintel.indicator.file.hash.{{threatintel.misp.attribute.type}}"
    ignore_missing: true
    if: "ctx?.threatintel?.indicator?.type == 'file' && ctx?.threatintel?.misp?.attribute?.type != null && !ctx?.threatintel?.misp?.attribute?.type.startsWith('filename')"
- rename:
    field: threatintel.misp.attribute.value
    target_field: threatintel.indicator.file.name
    ignore_missing: true
    if: "ctx?.threatintel?.indicator?.type == 'file' && ctx?.threatintel?.misp?.attribute?.type == 'filename'"
- grok:
    field: threatintel.misp.attribute.type
    patterns: 
      - "%{WORD}\\|%{WORD:_tmp.hashtype}"
    ignore_missing: true
    if: ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|')
- grok:
    field: threatintel.misp.attribute.value
    patterns: 
      - "%{DATA:threatintel.indicator.file.name}\\|%{GREEDYDATA:_tmp.hashvalue}"
    ignore_missing: true
    if: ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|')
- set:
    field: threatintel.indicator.file.hash.{{_tmp.hashtype}}
    value: '{{_tmp.hashvalue}}'
    if: "ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|') && ctx?._tmp?.hashvalue != null && ctx?._tmp?.hashtype != null"

## URL/URI indicator operations
- set:
    field: threatintel.indicator.type
    value: url
    if: "ctx?.threatintel?.misp?.attribute?.type != null && ['url', 'link', 'uri'].contains(ctx?.threatintel?.misp?.attribute?.type)"
- uri_parts:
    field: threatintel.misp.attribute.value
    target_field: threatintel.indicator.url
    keep_original: true
    remove_if_successful: true
    if: ctx?.threatintel?.indicator?.type == 'url' && ctx?.threatintel?.misp?.attribute?.type != 'uri'
- rename:
    field: threatintel.misp.attribute.value
    target_field: threatintel.indicator.url.full
    ignore_missing: true
    if: "ctx?.threatintel?.indicator?.type == 'url' && ctx?.threatintel?.indicator?.url?.original == null && ctx?.threatintel?.misp?.attribute?.type != 'uri'"

## Regkey indicator operations
- set:
    field: threatintel.indicator.type
    value: windows-registry-key
    if: "ctx?.threatintel?.misp?.attribute?.type != null && ctx?.threatintel?.misp?.attribute?.type.startsWith('regkey')"
- rename:
    field: threatintel.misp.attribute.value
    target_field: threatintel.indicator.registry.key
    ignore_missing: true
    if: "ctx?.threatintel?.indicator?.type == 'windows-registry-key' && ctx?.threatintel?.misp?.attribute?.type == 'regkey'"
- grok:
    field: threatintel.misp.attribute.value
    patterns: 
      - "%{DATA:threatintel.indicator.registry.key}\\|%{DATA:threatintel.indicator.registry.value}"
    ignore_missing: true
    if: "ctx?.threatintel?.misp?.attribute?.type == 'regkey|value'"

## AS indicator operations
- set:
    field: threatintel.indicator.type
    value: autonomous-system
    if: "ctx?.threatintel?.misp?.attribute?.type != null && ctx?.threatintel?.misp?.attribute?.type == 'AS'"
- convert:
    field: threatintel.misp.attribute.value
    type: long
    target_field: threatintel.indicator.as.number
    ignore_missing: true
    if: ctx?.threatintel?.indicator?.type == 'autonomous-system'

## Domain/IP/Port indicator operations
- set:
    field: threatintel.indicator.type
    value: domain-name
    if: "ctx?.threatintel?.misp?.attribute?.type != null && (ctx?.threatintel?.misp?.attribute?.type == 'hostname' || ctx?.threatintel?.misp?.attribute?.type.startsWith('domain'))"
- set:
    field: threatintel.indicator.type
    value: ipv4-addr
    if: "ctx?.threatintel?.misp?.attribute?.type != null && ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx?.threatintel?.misp?.attribute?.type)"
- rename:
    field: threatintel.misp.attribute.value
    target_field: threatintel.indicator.domain
    ignore_missing: true
    if: "ctx?.threatintel?.indicator?.type == 'domain-name' && ctx?.threatintel?.misp?.attribute?.type != 'domain|ip'"
- rename:
    field: threatintel.misp.attribute.value
    target_field: threatintel.indicator.ip
    ignore_missing: true
    if: "ctx?.threatintel?.indicator?.type == 'ipv4-addr' && ctx?.threatintel?.misp?.attribute?.type != 'domain|ip' && !['ip-src|port', 'ip-dst|port'].contains(ctx?.threatintel?.misp?.attribute?.type)"
- grok:
    field: threatintel.misp.attribute.value
    patterns: 
      - "%{DATA:threatintel.indicator.domain}\\|%{IP:threatintel.indicator.ip}"
    ignore_missing: true
    if: ctx.threatintel?.misp?.attribute?.type == 'domain|ip'
- grok:
    field: threatintel.misp.attribute.value
    patterns: 
      - "%{IP:threatintel.indicator.ip}\\|%{NUMBER:threatintel.indicator.port}"
    ignore_missing: true
    if: "['ip-src|port', 'ip-dst|port'].contains(ctx.threatintel?.misp?.attribute?.type)"

## Email indicator operations
# Currently this ignores email-message, except setting the type it will leave the rest of the fields under misp.
- set:
    field: threatintel.indicator.type
    value: email-addr
    if: "ctx?.threatintel?.misp?.attribute?.type != null && ['email-dst', 'email-src'].contains(ctx.threatintel?.misp?.attribute?.type)"
- set:
    field: threatintel.indicator.type
    value: email-message
    if: "ctx?.threatintel?.misp?.attribute?.type != null && ctx.threatintel?.misp?.attribute?.type.startsWith('email') && !['email-dst', 'email-src'].contains(ctx.threatintel?.misp?.attribute?.type)"
- rename:
    field: threatintel.misp.attribute.value
    target_field: threatintel.indicator.email.address
    ignore_missing: true
    if: ctx?.threatintel?.indicator?.type == 'email-addr'

## MAC Address indicator operations
- set:
    field: threatintel.indicator.type
    value: mac-addr
    if: "ctx?.threatintel?.misp?.attribute?.type != null && ['mac-address', 'mac-eui-64'].contains(ctx.threatintel?.misp?.attribute?.type)"
- rename:
    field: threatintel.misp.attribute.value
    target_field: threatintel.indicator.mac
    ignore_missing: true
    if: ctx?.threatintel?.indicator?.type == 'mac-addr'

###################
# Tags ECS fields #
###################
# Stripping special characters from tags
- script:
      lang: painless
      if: ctx?.threatintel?.misp?.tag != null
      source: |
        def tags = ctx.threatintel.misp.tag.stream()
           .map(t -> t.name.replace('\\', '').replace('"', ''))
           .collect(Collectors.toList());
        def tlpTags = tags.stream()
           .filter(t -> t.startsWith('tlp:'))
           .map(t -> t.replace('tlp:', ''))
           .collect(Collectors.toList());
        
        ctx.tags = tags;
        ctx.threatintel.indicator.marking = [ 'tlp': tlpTags ];

# Setting indicator type to unknown if it does not match anything
- set:
    field: threatintel.indicator.type
    value: unknown
    if: ctx?.threatintel?.indicator?.type == null

######################
# Cleanup processors #
######################
- script:
    lang: painless
    if: ctx?.threatintel != null
    source: |
      void handleMap(Map map) {
        for (def x : map.values()) {
          if (x instanceof Map) {
              handleMap(x);
          } else if (x instanceof List) {
              handleList(x);
          }
        }
      map.values().removeIf(v -> v == null);
      }
      void handleList(List list) {
        for (def x : list) {
            if (x instanceof Map) {
                handleMap(x);
            } else if (x instanceof List) {
                handleList(x);
            }
        }
      }
      handleMap(ctx);
# Removing fields not needed anymore, either because its copied somewhere else, or is not relevant to this event
- remove:
    field:
    - threatintel.misp.attribute.value
    ignore_missing: true
    if: ctx?.threatintel?.indicator?.type != 'unknown'
- remove:
    field:
    - threatintel.misp.Attribute.timestamp
    - threatintel.misp.timestamp
    - threatintel.misp.tag
    - threatintel.misp.org
    - threatintel.misp.analysis
    - _tmp
    - json
    ignore_missing: true

on_failure:
- set:
    field: error.message
    value: '{{ _ingest.on_failure_message }}'
