Skip to content

Commit fc418e0

Browse files
nazargesyknazargesyk
authored
Merge branch 'main' into gis-9137
2 parents 6a98de3 + 0e5e0ca commit fc418e0

File tree

73 files changed

+454
-192
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

73 files changed

+454
-192
lines changed

‎uncoder-core/app/routers/meta_info.py‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
from dataclasses import asdict
2+
3+
from fastapi import APIRouter, Body, HTTPException
4+
5+
from app.models.meta_info import (
6+
MetaInfo,
7+
MetaInfoResponse,
8+
MitreInfoContainer,
9+
MitreTacticContainer,
10+
MitreTechniqueContainer,
11+
ParsedLogSources,
12+
RawMetaInfo,
13+
)
14+
from app.translator.core.exceptions.core import UnsupportedPlatform
15+
from app.translator.translator import app_translator
16+
17+
meta_info_router = APIRouter()
18+
19+
20+
@meta_info_router.post("/get_meta_info/", tags=["meta_info"], description="Get Rule MetaInfo")
21+
@meta_info_router.post("/get_meta_info/", include_in_schema=False)
22+
def get_meta_info_data(
23+
source_platform_id: str = Body(..., embed=True), text: str = Body(..., embed=True)
24+
) -> MetaInfoResponse:
25+
try:
26+
logsources, raw_query_container = app_translator.parse_meta_info(text=text, source=source_platform_id)
27+
except UnsupportedPlatform as exc:
28+
raise HTTPException(status_code=400, detail="Unsuported platform") from exc
29+
except Exception as exc:
30+
raise HTTPException(status_code=400, detail="Unexpected error.") from exc
31+
if not raw_query_container:
32+
raise HTTPException(status_code=400, detail="Can't parse metadata")
33+
most_frequent_product = max(logsources.get("product"), key=logsources.get("product").get, default=None)
34+
most_frequent_service = max(logsources.get("service"), key=logsources.get("service").get, default=None)
35+
most_frequent_category = max(logsources.get("category"), key=logsources.get("category").get, default=None)
36+
37+
logsources.get("product", {}).pop(most_frequent_product, None)
38+
logsources.get("service", {}).pop(most_frequent_service, None)
39+
logsources.get("category", {}).pop(most_frequent_category, None)
40+
41+
parsed_logsources = ParsedLogSources(
42+
most_frequent_product=most_frequent_product,
43+
most_frequent_service=most_frequent_service,
44+
most_frequent_category=most_frequent_category,
45+
least_frequent_products=list(logsources.get("product", {}).keys()),
46+
least_frequent_services=list(logsources.get("service", {}).keys()),
47+
least_frequent_categories=list(logsources.get("category", {}).keys()),
48+
)
49+
return MetaInfoResponse(
50+
query=raw_query_container.query,
51+
language=raw_query_container.language,
52+
meta_info=MetaInfo(
53+
id_=raw_query_container.meta_info.id,
54+
title=raw_query_container.meta_info.title,
55+
description=raw_query_container.meta_info.description,
56+
author=raw_query_container.meta_info.author,
57+
date=raw_query_container.meta_info.date,
58+
false_positives=raw_query_container.meta_info.false_positives,
59+
license_=raw_query_container.meta_info.license,
60+
mitre_attack=MitreInfoContainer(
61+
tactics=[
62+
MitreTacticContainer(**asdict(tactic_container))
63+
for tactic_container in raw_query_container.meta_info.mitre_attack.tactics
64+
],
65+
techniques=[
66+
MitreTechniqueContainer(**asdict(tactic_container))
67+
for tactic_container in raw_query_container.meta_info.mitre_attack.techniques
68+
],
69+
),
70+
output_table_fields=raw_query_container.meta_info.output_table_fields,
71+
parsed_log_sources=parsed_logsources,
72+
query_fields=raw_query_container.meta_info.query_fields,
73+
query_period=raw_query_container.meta_info.query_period,
74+
raw_metainfo_container=RawMetaInfo(
75+
trigger_operator=raw_query_container.meta_info.raw_metainfo_container.trigger_operator,
76+
trigger_threshold=raw_query_container.meta_info.raw_metainfo_container.trigger_threshold,
77+
query_frequency=raw_query_container.meta_info.raw_metainfo_container.query_frequency,
78+
query_period=raw_query_container.meta_info.raw_metainfo_container.query_period,
79+
),
80+
raw_mitre_attack=raw_query_container.meta_info.raw_mitre_attack,
81+
references=raw_query_container.meta_info.references,
82+
severity=raw_query_container.meta_info.severity,
83+
source_mapping_ids=raw_query_container.meta_info.source_mapping_ids,
84+
status=raw_query_container.meta_info.status,
85+
tags=raw_query_container.meta_info.tags,
86+
timeframe=raw_query_container.meta_info.timeframe,
87+
),
88+
)

‎uncoder-core/app/translator/core/functions.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,10 @@ def order_to_render(self) -> dict[str, int]:
164164

165165
return {}
166166

167+
@property
168+
def supported_render_names(self) -> set[str]:
169+
return set(self._renders_map)
170+
167171

168172
class PlatformFunctions:
169173
dir_path: str = None

‎uncoder-core/app/translator/core/mapping.py‎

Lines changed: 43 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@ class LogSourceSignature(ABC):
2222
def is_suitable(self, **kwargs) -> bool:
2323
raise NotImplementedError("Abstract method")
2424

25+
def is_probably_suitable(self, **kwargs) -> bool:
26+
"""
27+
Performs check with more options, but the result is less accurate than the "is_suitable" method
28+
"""
29+
raise NotImplementedError("Abstract method")
30+
2531
@staticmethod
2632
def _check_conditions(conditions: list[Union[bool, None]]) -> bool:
2733
conditions = [condition for condition in conditions if condition is not None]
@@ -88,11 +94,13 @@ def __init__(
8894
log_source_signature: _LogSourceSignatureType = None,
8995
fields_mapping: Optional[FieldsMapping] = None,
9096
raw_log_fields: Optional[dict] = None,
97+
conditions: Optional[dict] = None,
9198
):
9299
self.source_id = source_id
93100
self.log_source_signature = log_source_signature
94101
self.fields_mapping = fields_mapping or FieldsMapping([])
95102
self.raw_log_fields = raw_log_fields
103+
self.conditions = conditions
96104

97105

98106
class BasePlatformMappings:
@@ -123,6 +131,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]:
123131

124132
field_mappings_dict = mapping_dict.get("field_mapping", {})
125133
raw_log_fields = mapping_dict.get("raw_log_fields", {})
134+
conditions = mapping_dict.get("conditions", {})
126135
field_mappings_dict.update({field: field for field in raw_log_fields})
127136
fields_mapping = self.prepare_fields_mapping(field_mapping=field_mappings_dict)
128137
self.update_default_source_mapping(default_mapping=default_mapping, fields_mapping=fields_mapping)
@@ -131,6 +140,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]:
131140
log_source_signature=log_source_signature,
132141
fields_mapping=fields_mapping,
133142
raw_log_fields=raw_log_fields,
143+
conditions=conditions,
134144
)
135145

136146
if self.skip_load_default_mappings:
@@ -170,31 +180,47 @@ def get_source_mappings_by_fields_and_log_sources(
170180

171181
return by_log_sources_and_fields or by_fields or [self._source_mappings[DEFAULT_MAPPING_NAME]]
172182

173-
def get_source_mappings_by_ids(self, source_mapping_ids: list[str]) -> list[SourceMapping]:
183+
def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
184+
return self._source_mappings.get(source_id)
185+
186+
def get_source_mappings_by_ids(
187+
self, source_mapping_ids: list[str], return_default: bool = True
188+
) -> list[SourceMapping]:
174189
source_mappings = []
175190
for source_mapping_id in source_mapping_ids:
191+
if source_mapping_id == DEFAULT_MAPPING_NAME:
192+
continue
176193
if source_mapping := self.get_source_mapping(source_mapping_id):
177194
source_mappings.append(source_mapping)
178195

179-
if not source_mappings:
196+
if not source_mappings and return_default:
180197
source_mappings = [self.get_source_mapping(DEFAULT_MAPPING_NAME)]
181198

182199
return source_mappings
183200

184-
def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
185-
return self._source_mappings.get(source_id)
201+
def get_source_mappings_by_log_sources(self, log_sources: dict) -> Optional[list[str]]:
202+
raise NotImplementedError("Abstract method")
186203

187204
@property
188205
def default_mapping(self) -> SourceMapping:
189206
return self._source_mappings[DEFAULT_MAPPING_NAME]
190207

191-
def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[str]:
208+
def check_fields_mapping_existence(
209+
self,
210+
query_field_tokens: list[Field],
211+
function_field_tokens_map: dict[str, list[Field]],
212+
supported_func_render_names: set[str],
213+
source_mapping: SourceMapping,
214+
) -> list[str]:
192215
unmapped = []
193-
for field in field_tokens:
194-
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
195-
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
196-
if not mapped_field and field.source_name not in unmapped:
197-
unmapped.append(field.source_name)
216+
217+
for field in query_field_tokens:
218+
self._check_field_mapping_existence(field, source_mapping, unmapped)
219+
220+
for func_name, function_field_tokens in function_field_tokens_map.items():
221+
if func_name in supported_func_render_names:
222+
for field in function_field_tokens:
223+
self._check_field_mapping_existence(field, source_mapping, unmapped)
198224

199225
if self.is_strict_mapping and unmapped:
200226
raise StrictPlatformException(
@@ -203,6 +229,13 @@ def check_fields_mapping_existence(self, field_tokens: list[Field], source_mappi
203229

204230
return unmapped
205231

232+
@staticmethod
233+
def _check_field_mapping_existence(field: Field, source_mapping: SourceMapping, unmapped: list[str]) -> None:
234+
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
235+
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
236+
if not mapped_field and field.source_name not in unmapped:
237+
unmapped.append(field.source_name)
238+
206239
@staticmethod
207240
def map_field(field: Field, source_mapping: SourceMapping) -> list[str]:
208241
generic_field_name = field.get_generic_field_name(source_mapping.source_id)

‎uncoder-core/app/translator/core/mixins/rule.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ def parse_mitre_attack(self, tags: list[str]) -> MitreInfoContainer:
4242
tag = tag.lower()
4343
if tag.startswith("attack."):
4444
tag = tag[7::]
45-
if tag.startswith("t"):
45+
if tag[-1].isdigit():
4646
parsed_techniques.append(tag)
4747
else:
4848
parsed_tactics.append(tag)

‎uncoder-core/app/translator/core/models/query_container.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ def __init__(
6565
date: Optional[str] = None,
6666
output_table_fields: Optional[list[Field]] = None,
6767
query_fields: Optional[list[Field]] = None,
68+
function_fields: Optional[list[Field]] = None,
69+
function_fields_map: Optional[dict[str, list[Field]]] = None,
6870
license_: Optional[str] = None,
6971
severity: Optional[str] = None,
7072
references: Optional[list[str]] = None,
@@ -76,7 +78,7 @@ def __init__(
7678
parsed_logsources: Optional[dict] = None,
7779
timeframe: Optional[timedelta] = None,
7880
query_period: Optional[timedelta] = None,
79-
mitre_attack: MitreInfoContainer = MitreInfoContainer(),
81+
mitre_attack: Optional[MitreInfoContainer] = None,
8082
raw_metainfo_container: Optional[RawMetaInfoContainer] = None,
8183
) -> None:
8284
self.id = id_ or str(uuid.uuid4())
@@ -86,23 +88,25 @@ def __init__(
8688
self.risk_score = risk_score
8789
self.type_ = type_ or ""
8890
self.description = description or ""
89-
self.author = [v.strip() for v in author] if author else []
91+
self.author = [v.strip() for v in author] if author and author != [None] else []
9092
self.date = date or datetime.now().date().strftime("%Y-%m-%d")
9193
self.output_table_fields = output_table_fields or []
9294
self.query_fields = query_fields or []
95+
self.function_fields = function_fields or []
96+
self.function_fields_map = function_fields_map or {}
9397
self.license = license_ or "DRL 1.1"
9498
self.severity = severity or SeverityType.low
9599
self.references = references or []
96100
self.tags = tags or []
97-
self.mitre_attack = mitre_attack or None
101+
self.mitre_attack = mitre_attack or MitreInfoContainer()
98102
self.raw_mitre_attack = raw_mitre_attack or []
99103
self.status = status or "stable"
100104
self.false_positives = false_positives or []
101105
self._source_mapping_ids = source_mapping_ids or [DEFAULT_MAPPING_NAME]
102106
self.parsed_logsources = parsed_logsources or {}
103107
self.timeframe = timeframe
104108
self.query_period = query_period
105-
self.raw_metainfo_container = raw_metainfo_container
109+
self.raw_metainfo_container = raw_metainfo_container or RawMetaInfoContainer()
106110

107111
@property
108112
def author_str(self) -> str:

‎uncoder-core/app/translator/core/parser.py‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,16 +65,19 @@ def get_query_tokens(self, query: str) -> list[QUERY_TOKEN_TYPE]:
6565
@staticmethod
6666
def get_field_tokens(
6767
query_tokens: list[QUERY_TOKEN_TYPE], functions: Optional[list[Function]] = None
68-
) -> list[Field]:
69-
field_tokens = []
68+
) -> tuple[list[Field], list[Field], dict[str, list[Field]]]:
69+
query_field_tokens = []
70+
function_field_tokens = []
71+
function_field_tokens_map = {}
7072
for token in query_tokens:
7173
if isinstance(token, (FieldField, FieldValue, FunctionValue)):
72-
field_tokens.extend(token.fields)
74+
query_field_tokens.extend(token.fields)
7375

74-
if functions:
75-
field_tokens.extend([field for func in functions for field in func.fields])
76+
for func in functions or []:
77+
function_field_tokens.extend(func.fields)
78+
function_field_tokens_map[func.name] = func.fields
7679

77-
return field_tokens
80+
return query_field_tokens, function_field_tokens, function_field_tokens_map
7881

7982
def get_source_mappings(
8083
self, field_tokens: list[Field], log_sources: dict[str, list[Union[int, str]]]

‎uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_bits_client.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
platform: ElasticSearch
22
source: windows_bits_client
33

4+
conditions:
5+
winlog.channel: 'Microsoft-Windows-Bits-Client/Operational'
46

57
log_source:
68
index: [winlogbeat-*, logs-*]

‎uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_image_load.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
platform: ElasticSearch
22
source: windows_image_load
33

4+
conditions:
5+
event.action:
6+
- 'Image loaded (rule: ImageLoad)'
7+
- 'load'
48

59
log_source:
610
index: [winlogbeat-*, logs-*]

‎uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_ldap_debug.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
platform: ElasticSearch
22
source: windows_ldap_debug
33

4+
conditions:
5+
winlog.channel: 'Microsoft-Windows-LDAP-Client/Debug'
46

57
log_source:
68
index: [winlogbeat-*, logs-*]

‎uncoder-core/app/translator/mappings/platforms/elasticsearch/windows_network_connection.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
platform: ElasticSearch
22
source: windows_network_connection
33

4+
conditions:
5+
event.category: 'network'
46

57
log_source:
68
index: [winlogbeat-*, logs-*]

0 commit comments

Comments
 (0)