Skip to content

Commit 9f9675d

Browse files
nazargesyknazargesyk
authored
Merge pull request #8 from UncoderIO/iocs_mapping_hash_fixes
IOC`s fix mapping and hash types
2 parents c6cbbe7 + 35587fa commit 9f9675d

File tree

6 files changed

+46
-58
lines changed

6 files changed

+46
-58
lines changed

‎siem-converter/app/converter/backends/graylog/mappings/graylog_cti.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
DEFAULT_GRAYLOG_MAPPING = {
2-
"SourceIP": "sourceAddress",
3-
"DestinationIP": "destinationAddress",
4-
"Domain": "destinationDnsDomain",
5-
"URL": "requestUrl",
6-
"HashMd5": "fileHash",
7-
"HashSha1": "fileHash",
8-
"HashSha256": "fileHash",
9-
"HashSha512": "fileHash",
2+
"SourceIP": "source.ip",
3+
"DestinationIP": "destination.ip",
4+
"Domain": "destination.domain",
5+
"URL": "url.original",
6+
"HashMd5": "file.hash.md5",
7+
"HashSha1": "file.hash.sha1",
8+
"HashSha256": "file.hash.sha256",
9+
"HashSha512": "file.hash.sha512",
1010
"Emails": "emails",
1111
"Files": "filePath"
1212
}

‎siem-converter/app/converter/backends/microsoft/mappings/microsoft_sentinel_cti.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
DEFAULT_MICROSOFT_SENTINEL_MAPPING = {
2-
"DestinationIP": "DestinationIP",
2+
"DestinationIP": "DestinationIp",
33
"SourceIP": "SourceIp",
44
"HashSha512": "FileHashSha512",
55
"HashSha256": "FileHashSha256",
66
"HashMd5": "FileHashMd5",
77
"Emails": "SenderFromAddress",
88
"Domain": "DestinationHostname",
99
"HashSha1": "FileHashSha1",
10-
"Files": "FileName",
10+
"Files": "TargetFileName",
1111
"URL": "URL"
1212
}

‎siem-converter/app/converter/core/parser_cti.py‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,23 +5,29 @@
55
from pydantic import BaseModel
66

77
from app.converter.core.exceptions.iocs import IocsLimitExceededException, EmptyIOCSException
8-
from app.converter.tools.const import IP_IOC_REGEXP_PATTERN, DOMAIN_IOC_REGEXP_PATTERN, URL_IOC_REGEXP_PATTERN,\
9-
hash_regexes, IOCType, HashType, IocParsingRule
8+
from app.converter.tools.const import IP_IOC_REGEXP_PATTERN, DOMAIN_IOC_REGEXP_PATTERN, URL_IOC_REGEXP_PATTERN, \
9+
hash_regexes, IOCType, HashType, IocParsingRule, HASH_MAP
1010

1111

1212
class Iocs(BaseModel):
1313
ip: list[str] = []
1414
url: list[str] = []
1515
domain: list[str] = []
16-
hash: list[str] = []
16+
hash_dict: dict = {}
1717

1818
def get_total_count(self) -> int:
19-
return len(self.ip) + len(self.url) + len(self.domain) + len(self.hash)
19+
hash_len = 0
20+
for value in self.hash_dict.values():
21+
hash_len += len(value)
22+
return len(self.ip) + len(self.url) + len(self.domain) + hash_len
2023

2124
def return_iocs(self) -> dict:
22-
if all(not value for value in [self.ip, self.url, self.domain, self.hash]):
25+
if all(not value for value in [self.ip, self.url, self.domain, self.hash_dict]):
2326
raise EmptyIOCSException()
24-
return {"ip": self.ip, "url": self.url, "domain": self.domain, "hash": self.hash}
27+
result = {"DestinationIP": self.ip, "URL": self.url, "Domain": self.domain}
28+
for key, value in self.hash_dict.items():
29+
result[HASH_MAP[key]] = value
30+
return result
2531

2632

2733
class CTIParser:
@@ -47,7 +53,7 @@ def get_iocs_from_string(
4753
if not include_hash_types:
4854
include_hash_types = list(hash_regexes.keys())
4955
for hash_type in include_hash_types:
50-
iocs.hash.extend(self._find_all_str_by_regex(string, hash_regexes[hash_type]))
56+
iocs.hash_dict[hash_type] = self._find_all_str_by_regex(string, hash_regexes[hash_type])
5157
iocs = self.remove_duplicates(iocs)
5258
iocs = self.remove_exceptions(iocs, exceptions)
5359
if ioc_parsing_rules is None or "remove_private_and_reserved_ips" in ioc_parsing_rules:
@@ -69,14 +75,16 @@ def remove_duplicates(self, iocs):
6975
iocs.ip = self._remove_duplicates_from_list(iocs.ip)
7076
iocs.domain = self._remove_duplicates_from_list(iocs.domain)
7177
iocs.url = self._remove_duplicates_from_list(iocs.url)
72-
iocs.hash = self._remove_duplicates_from_list(iocs.hash)
78+
for key, value in iocs.hash_dict.items():
79+
iocs.hash_dict[key] = self._remove_duplicates_from_list(value)
7380
return iocs
7481

7582
def remove_exceptions(self, iocs, exceptions=None):
7683
iocs.ip = self._remove_exceptions(iocs.ip, exceptions)
7784
iocs.domain = self._remove_exceptions(iocs.domain, exceptions)
7885
iocs.url = self._remove_exceptions(iocs.url, exceptions)
79-
iocs.hash = self._remove_exceptions(iocs.hash, exceptions)
86+
for key, value in iocs.hash_dict.items():
87+
iocs.hash_dict[key] = self._remove_exceptions(value, exceptions)
8088
return iocs
8189

8290
@staticmethod

‎siem-converter/app/converter/core/render_cti.py‎

Lines changed: 4 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,6 @@ class RenderCTI:
3434
final_result_for_one: str = "union * | where {result}\n"
3535
default_mapping = None
3636

37-
def get_default_mapping(self, include_source_ip=False):
38-
return self.prepare_mapping(self.default_mapping, include_source_ip)
39-
4037
def create_field_value(self, field: str, value: str, generic_field: str):
4138
return self.data_map.format(key=field, value=value)
4239

@@ -53,35 +50,17 @@ def render(self, data: List[List[IocsChunkValue]]) -> list[str]:
5350
def collect_data_values(self, chunk):
5451
data_values = []
5552
key_chunk = []
56-
processing_key = chunk[0].generic_field
53+
processing_key = chunk[0].platform_field
5754
for value in chunk:
58-
if processing_key != value.generic_field:
55+
if processing_key != value.platform_field:
5956
data_values.append(self.or_group.format(or_group=self.or_operator.join(key_chunk),
6057
processing_key=processing_key))
6158
key_chunk = []
62-
processing_key = value.generic_field
63-
key_chunk.append(self.create_field_value(field=value.generic_field,
59+
processing_key = value.platform_field
60+
key_chunk.append(self.create_field_value(field=value.platform_field,
6461
value=value.value,
6562
generic_field=value.generic_field))
6663
if key_chunk:
6764
data_values.append(
6865
self.or_group.format(or_group=self.or_operator.join(key_chunk), processing_key=processing_key))
6966
return data_values
70-
71-
def prepare_mapping(self, mapping: dict, include_source_ip: bool = False) -> dict:
72-
m = {
73-
"DestinationIP": "ip",
74-
"Domain": "domain",
75-
"URL": "url",
76-
"HashMd5": "hash",
77-
"HashSha1": "hash",
78-
"HashSha256": "hash",
79-
"HashSha512": "hash",
80-
}
81-
if include_source_ip:
82-
m["SourceIP"] = "ip"
83-
res = {}
84-
for key, new_field_name in mapping.items():
85-
if key in m:
86-
res[new_field_name] = m[key]
87-
return res

‎siem-converter/app/converter/cti_converter.py‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,8 @@ def __init__(self):
1717
self.logger = logging.getLogger("cti_converter")
1818
self.parser = CTIParser()
1919

20-
def _reverse_mapping(self, platform: CTIPlatform, data: Dict[str, List[str]],
21-
include_source_ip: bool = False) -> Dict[str, str]:
22-
mapping: Dict = self.renders.get(platform.name).get_default_mapping(include_source_ip)
23-
reverse_mapping = {}
24-
for platform_field, generic_field in mapping.items():
25-
if data.get(generic_field):
26-
reverse_mapping.update({generic_field: platform_field})
27-
return reverse_mapping
20+
def _get_render_mapping(self, platform: CTIPlatform, include_source_ip: bool = False) -> Dict[str, str]:
21+
return self.renders.get(platform.name).default_mapping
2822

2923
@handle_translation_exceptions
3024
def __parse_iocs_from_string(self, text: str, include_ioc_types: list = None, include_hash_types: list = None,
@@ -39,11 +33,10 @@ def __parse_iocs_from_string(self, text: str, include_ioc_types: list = None, in
3933
@handle_translation_exceptions
4034
def __render_translation(self, parsed_data: dict, platform_data: CTIPlatform, iocs_per_query: int,
4135
include_source_ip: bool = False) -> List[str]:
42-
reverse_mapping = self._reverse_mapping(data=parsed_data,
43-
include_source_ip=include_source_ip, platform=platform_data)
36+
mapping = self._get_render_mapping(platform=platform_data, include_source_ip=include_source_ip)
4437
platform = self.renders.get(platform_data.name)
4538
platform_generation = self.generate(data=parsed_data, platform=platform, iocs_per_query=iocs_per_query,
46-
mapping=reverse_mapping)
39+
mapping=mapping)
4740
return platform_generation
4841

4942
def convert(self, text: str,
@@ -73,9 +66,10 @@ def _get_iocs_chunk(chunks_size: int, data: Dict[str, List[str]],
7366
result = []
7467
for generic_field, iocs_list in data.items():
7568
for ioc in iocs_list:
76-
result.append(IocsChunkValue(generic_field=generic_field,
77-
platform_field=mapping.get(generic_field, generic_field),
78-
value=ioc))
69+
if mapping.get(generic_field):
70+
result.append(IocsChunkValue(generic_field=generic_field,
71+
platform_field=mapping[generic_field],
72+
value=ioc))
7973
return [result[i:i + chunks_size] for i in range(0, len(result), chunks_size)]
8074

8175
def generate(self, platform: RenderCTI, iocs_per_query, data: Dict[str, List[str]],

‎siem-converter/app/converter/tools/const.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,13 @@
1010
"replace_dots", "remove_private_and_reserved_ips", "replace_hxxp"
1111
]
1212

13+
HASH_MAP = {
14+
"md5": "HashMd5",
15+
"sha1": "HashSha1",
16+
"sha256": "HashSha256",
17+
"sha512": "HashSha512"
18+
}
19+
1320
hash_regexes = {
1421
"md5": r"(?:^|[\s\/\[(\"',;{>|])([A-Fa-f0-9]{32})(?=[\s)\]\"',;\n<|]|$)",
1522
"sha1": r"(?:^|[\s\/\[(\"',;{>|])([A-Fa-f0-9]{40})(?=[\s)\]\"',;\n<|]|$)",

0 commit comments

Comments
 (0)