Skip to content

Commit a1a1341

Browse files
nazargesyknazargesyk
authored
Merge branch 'main' into gis-9197
2 parents e74ecd3 + d5639ce commit a1a1341

File tree

140 files changed

+1816
-221
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

140 files changed

+1816
-221
lines changed

‎uncoder-core/app/translator/core/functions.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,10 @@ def order_to_render(self) -> dict[str, int]:
164164

165165
return {}
166166

167+
@property
168+
def supported_render_names(self) -> set[str]:
169+
return set(self._renders_map)
170+
167171

168172
class PlatformFunctions:
169173
dir_path: str = None

‎uncoder-core/app/translator/core/mapping.py‎

Lines changed: 43 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,12 @@ class LogSourceSignature(ABC):
2222
def is_suitable(self, **kwargs) -> bool:
2323
raise NotImplementedError("Abstract method")
2424

25+
def is_probably_suitable(self, **kwargs) -> bool:
26+
"""
27+
Performs check with more options, but the result is less accurate than the "is_suitable" method
28+
"""
29+
raise NotImplementedError("Abstract method")
30+
2531
@staticmethod
2632
def _check_conditions(conditions: list[Union[bool, None]]) -> bool:
2733
conditions = [condition for condition in conditions if condition is not None]
@@ -88,11 +94,13 @@ def __init__(
8894
log_source_signature: _LogSourceSignatureType = None,
8995
fields_mapping: Optional[FieldsMapping] = None,
9096
raw_log_fields: Optional[dict] = None,
97+
conditions: Optional[dict] = None,
9198
):
9299
self.source_id = source_id
93100
self.log_source_signature = log_source_signature
94101
self.fields_mapping = fields_mapping or FieldsMapping([])
95102
self.raw_log_fields = raw_log_fields
103+
self.conditions = conditions
96104

97105

98106
class BasePlatformMappings:
@@ -123,6 +131,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]:
123131

124132
field_mappings_dict = mapping_dict.get("field_mapping", {})
125133
raw_log_fields = mapping_dict.get("raw_log_fields", {})
134+
conditions = mapping_dict.get("conditions", {})
126135
field_mappings_dict.update({field: field for field in raw_log_fields})
127136
fields_mapping = self.prepare_fields_mapping(field_mapping=field_mappings_dict)
128137
self.update_default_source_mapping(default_mapping=default_mapping, fields_mapping=fields_mapping)
@@ -131,6 +140,7 @@ def prepare_mapping(self) -> dict[str, SourceMapping]:
131140
log_source_signature=log_source_signature,
132141
fields_mapping=fields_mapping,
133142
raw_log_fields=raw_log_fields,
143+
conditions=conditions,
134144
)
135145

136146
if self.skip_load_default_mappings:
@@ -170,31 +180,47 @@ def get_source_mappings_by_fields_and_log_sources(
170180

171181
return by_log_sources_and_fields or by_fields or [self._source_mappings[DEFAULT_MAPPING_NAME]]
172182

173-
def get_source_mappings_by_ids(self, source_mapping_ids: list[str]) -> list[SourceMapping]:
183+
def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
184+
return self._source_mappings.get(source_id)
185+
186+
def get_source_mappings_by_ids(
187+
self, source_mapping_ids: list[str], return_default: bool = True
188+
) -> list[SourceMapping]:
174189
source_mappings = []
175190
for source_mapping_id in source_mapping_ids:
191+
if source_mapping_id == DEFAULT_MAPPING_NAME:
192+
continue
176193
if source_mapping := self.get_source_mapping(source_mapping_id):
177194
source_mappings.append(source_mapping)
178195

179-
if not source_mappings:
196+
if not source_mappings and return_default:
180197
source_mappings = [self.get_source_mapping(DEFAULT_MAPPING_NAME)]
181198

182199
return source_mappings
183200

184-
def get_source_mapping(self, source_id: str) -> Optional[SourceMapping]:
185-
return self._source_mappings.get(source_id)
201+
def get_source_mappings_by_log_sources(self, log_sources: dict) -> Optional[list[str]]:
202+
raise NotImplementedError("Abstract method")
186203

187204
@property
188205
def default_mapping(self) -> SourceMapping:
189206
return self._source_mappings[DEFAULT_MAPPING_NAME]
190207

191-
def check_fields_mapping_existence(self, field_tokens: list[Field], source_mapping: SourceMapping) -> list[str]:
208+
def check_fields_mapping_existence(
209+
self,
210+
query_field_tokens: list[Field],
211+
function_field_tokens_map: dict[str, list[Field]],
212+
supported_func_render_names: set[str],
213+
source_mapping: SourceMapping,
214+
) -> list[str]:
192215
unmapped = []
193-
for field in field_tokens:
194-
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
195-
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
196-
if not mapped_field and field.source_name not in unmapped:
197-
unmapped.append(field.source_name)
216+
217+
for field in query_field_tokens:
218+
self._check_field_mapping_existence(field, source_mapping, unmapped)
219+
220+
for func_name, function_field_tokens in function_field_tokens_map.items():
221+
if func_name in supported_func_render_names:
222+
for field in function_field_tokens:
223+
self._check_field_mapping_existence(field, source_mapping, unmapped)
198224

199225
if self.is_strict_mapping and unmapped:
200226
raise StrictPlatformException(
@@ -203,6 +229,13 @@ def check_fields_mapping_existence(self, field_tokens: list[Field], source_mappi
203229

204230
return unmapped
205231

232+
@staticmethod
233+
def _check_field_mapping_existence(field: Field, source_mapping: SourceMapping, unmapped: list[str]) -> None:
234+
generic_field_name = field.get_generic_field_name(source_mapping.source_id)
235+
mapped_field = source_mapping.fields_mapping.get_platform_field_name(generic_field_name=generic_field_name)
236+
if not mapped_field and field.source_name not in unmapped:
237+
unmapped.append(field.source_name)
238+
206239
@staticmethod
207240
def map_field(field: Field, source_mapping: SourceMapping) -> list[str]:
208241
generic_field_name = field.get_generic_field_name(source_mapping.source_id)

‎uncoder-core/app/translator/core/mixins/rule.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ def parse_mitre_attack(self, tags: list[str]) -> MitreInfoContainer:
4242
tag = tag.lower()
4343
if tag.startswith("attack."):
4444
tag = tag[7::]
45-
if tag.startswith("t"):
45+
if tag[-1].isdigit():
4646
parsed_techniques.append(tag)
4747
else:
4848
parsed_tactics.append(tag)

‎uncoder-core/app/translator/core/mixins/tokens.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
from typing import Union
2+
3+
from app.translator.core.const import QUERY_TOKEN_TYPE
4+
from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType
5+
from app.translator.core.mapping import SourceMapping
6+
from app.translator.core.models.query_tokens.field_value import FieldValue
7+
from app.translator.core.models.query_tokens.identifier import Identifier
8+
9+
10+
class ExtraConditionMixin:
11+
def generate_extra_conditions(self, source_mapping: SourceMapping) -> list[QUERY_TOKEN_TYPE]:
12+
extra_tokens = []
13+
for field, value in source_mapping.conditions.items():
14+
extra_tokens.extend(
15+
[
16+
FieldValue(source_name=field, operator=Identifier(token_type=OperatorType.EQ), value=value),
17+
Identifier(token_type=LogicalOperatorType.AND),
18+
]
19+
)
20+
return extra_tokens

‎uncoder-core/app/translator/core/models/query_container.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ def __init__(
6565
date: Optional[str] = None,
6666
output_table_fields: Optional[list[Field]] = None,
6767
query_fields: Optional[list[Field]] = None,
68+
function_fields: Optional[list[Field]] = None,
69+
function_fields_map: Optional[dict[str, list[Field]]] = None,
6870
license_: Optional[str] = None,
6971
severity: Optional[str] = None,
7072
references: Optional[list[str]] = None,
@@ -76,7 +78,7 @@ def __init__(
7678
parsed_logsources: Optional[dict] = None,
7779
timeframe: Optional[timedelta] = None,
7880
query_period: Optional[timedelta] = None,
79-
mitre_attack: MitreInfoContainer = MitreInfoContainer(),
81+
mitre_attack: Optional[MitreInfoContainer] = None,
8082
raw_metainfo_container: Optional[RawMetaInfoContainer] = None,
8183
) -> None:
8284
self.id = id_ or str(uuid.uuid4())
@@ -86,23 +88,25 @@ def __init__(
8688
self.risk_score = risk_score
8789
self.type_ = type_ or ""
8890
self.description = description or ""
89-
self.author = [v.strip() for v in author] if author else []
91+
self.author = [v.strip() for v in author] if author and author != [None] else []
9092
self.date = date or datetime.now().date().strftime("%Y-%m-%d")
9193
self.output_table_fields = output_table_fields or []
9294
self.query_fields = query_fields or []
95+
self.function_fields = function_fields or []
96+
self.function_fields_map = function_fields_map or {}
9397
self.license = license_ or "DRL 1.1"
9498
self.severity = severity or SeverityType.low
9599
self.references = references or []
96100
self.tags = tags or []
97-
self.mitre_attack = mitre_attack or None
101+
self.mitre_attack = mitre_attack or MitreInfoContainer()
98102
self.raw_mitre_attack = raw_mitre_attack or []
99103
self.status = status or "stable"
100104
self.false_positives = false_positives or []
101105
self._source_mapping_ids = source_mapping_ids or [DEFAULT_MAPPING_NAME]
102106
self.parsed_logsources = parsed_logsources or {}
103107
self.timeframe = timeframe
104108
self.query_period = query_period
105-
self.raw_metainfo_container = raw_metainfo_container
109+
self.raw_metainfo_container = raw_metainfo_container or RawMetaInfoContainer()
106110

107111
@property
108112
def author_str(self) -> str:

‎uncoder-core/app/translator/core/parser.py‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -65,16 +65,19 @@ def get_query_tokens(self, query: str) -> list[QUERY_TOKEN_TYPE]:
6565
@staticmethod
6666
def get_field_tokens(
6767
query_tokens: list[QUERY_TOKEN_TYPE], functions: Optional[list[Function]] = None
68-
) -> list[Field]:
69-
field_tokens = []
68+
) -> tuple[list[Field], list[Field], dict[str, list[Field]]]:
69+
query_field_tokens = []
70+
function_field_tokens = []
71+
function_field_tokens_map = {}
7072
for token in query_tokens:
7173
if isinstance(token, (FieldField, FieldValue, FunctionValue)):
72-
field_tokens.extend(token.fields)
74+
query_field_tokens.extend(token.fields)
7375

74-
if functions:
75-
field_tokens.extend([field for func in functions for field in func.fields])
76+
for func in functions or []:
77+
function_field_tokens.extend(func.fields)
78+
function_field_tokens_map[func.name] = func.fields
7679

77-
return field_tokens
80+
return query_field_tokens, function_field_tokens, function_field_tokens_map
7881

7982
def get_source_mappings(
8083
self, field_tokens: list[Field], log_sources: dict[str, list[Union[int, str]]]

‎uncoder-core/app/translator/core/render.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -403,6 +403,9 @@ def process_raw_log_field_prefix(self, field: str, source_mapping: SourceMapping
403403
if raw_log_field_type := source_mapping.raw_log_fields.get(field):
404404
return [self.process_raw_log_field(field=field, field_type=raw_log_field_type)]
405405

406+
def generate_extra_conditions(self, source_mapping: SourceMapping) -> list[QUERY_TOKEN_TYPE]: # noqa: ARG002
407+
return []
408+
406409
def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMapping) -> str:
407410
if not self.raw_log_field_patterns_map:
408411
return ""
@@ -428,16 +431,23 @@ def _generate_from_tokenized_query_container_by_source_mapping(
428431
self, query_container: TokenizedQueryContainer, source_mapping: SourceMapping
429432
) -> str:
430433
unmapped_fields = self.mappings.check_fields_mapping_existence(
431-
query_container.meta_info.query_fields, source_mapping
434+
query_container.meta_info.query_fields,
435+
query_container.meta_info.function_fields_map,
436+
self.platform_functions.manager.supported_render_names,
437+
source_mapping,
432438
)
433439
rendered_functions = self.generate_functions(query_container.functions.functions, source_mapping)
434440
prefix = self.generate_prefix(source_mapping.log_source_signature, rendered_functions.rendered_prefix)
435441

436442
if source_mapping.raw_log_fields:
437443
defined_raw_log_fields = self.generate_raw_log_fields(
438-
fields=query_container.meta_info.query_fields, source_mapping=source_mapping
444+
fields=query_container.meta_info.query_fields + query_container.meta_info.function_fields,
445+
source_mapping=source_mapping,
439446
)
440447
prefix += f"\n{defined_raw_log_fields}"
448+
if source_mapping.conditions:
449+
extra_tokens = self.generate_extra_conditions(source_mapping=source_mapping)
450+
query_container.tokens = [*extra_tokens, *query_container.tokens]
441451
query = self.generate_query(tokens=query_container.tokens, source_mapping=source_mapping)
442452
not_supported_functions = query_container.functions.not_supported + rendered_functions.not_supported
443453
return self.finalize_query(

uncoder-core/app/translator/mappings/platforms/anomali/common.yml renamed to uncoder-core/app/translator/mappings/platforms/anomali/proxy.yml

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,19 @@
11
platform: Anomali
2-
description: Common field mapping
2+
source: proxy
33

44
field_mapping:
55
c-uri-query: url
66
c-useragent: user_agent
7+
c-uri: url
8+
cs-method: http_method
9+
cs-bytes: bytes_out
10+
cs-referrer: http_referrer
11+
sc-status: return_code
12+
13+
dns-query: query
14+
dns-answer: answer
15+
dns-record: record_type
16+
717
CommandLine: command_line
818
DestinationHostname: dest
919
DestinationIp: dest_ip

‎uncoder-core/app/translator/mappings/platforms/anomali/webserver.yml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
platform: Anomali
2+
source: webserver
3+
4+
field_mapping:
5+
c-uri-query: url
6+
c-useragent: user_agent
7+
c-uri: url
8+
cs-method: http_method
9+
cs-bytes: bytes_out
10+
cs-referrer: http_referrer
11+
sc-status: return_code
12+
13+
dns-query: query
14+
dns-answer: answer
15+
dns-record: record_type
16+
17+
CommandLine: command_line
18+
DestinationHostname: dest
19+
DestinationIp: dest_ip
20+
DestinationPort: dest_port
21+
Details: reg_value_data
22+
dst_ip: dest_ip
23+
dst_port: dest_port
24+
EventID: event_id
25+
EventName: event_name
26+
FileName: file_name
27+
FilePath: file_path
28+
Image: image
29+
NewProcessName: image
30+
OriginalFileName: original_file_name
31+
ParentCommandLine: parent_command_line
32+
ParentImage: parent_image
33+
ParentProcessID: parent_process_id
34+
Platform: platform
35+
ProcessCommandLine: command_line
36+
ProcessID: process_id
37+
SourceImage: parent_image
38+
SourcePort: src_port
39+
TargetFilename: file_name
40+
TargetObject: reg_key
41+
UserAgent: user_agent

‎uncoder-core/app/translator/mappings/platforms/arcsight/default.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
platform: ArcSight
2+
source: default
3+
4+
5+
default_log_source: {}

0 commit comments

Comments
 (0)