From 1af9f3a473b789d194b8d7edf0073f6843df5b0b Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Tue, 4 Feb 2025 10:04:51 -0800 Subject: [PATCH 01/11] define serialization functions for rba and risk bits in line with normal pydantic workflow --- .../detection_abstract.py | 26 ++++---- contentctl/objects/rba.py | 60 +++++++++++++++++-- 2 files changed, 67 insertions(+), 19 deletions(-) diff --git a/contentctl/objects/abstract_security_content_objects/detection_abstract.py b/contentctl/objects/abstract_security_content_objects/detection_abstract.py index 1231548c..fc0505ce 100644 --- a/contentctl/objects/abstract_security_content_objects/detection_abstract.py +++ b/contentctl/objects/abstract_security_content_objects/detection_abstract.py @@ -383,21 +383,17 @@ def providing_technologies(self) -> List[ProvidingTechnology]: @computed_field @property def risk(self) -> list[dict[str, Any]]: - risk_objects: list[dict[str, str | int]] = [] - - for entity in self.rba.risk_objects: - risk_object: dict[str, str | int] = dict() - risk_object["risk_object_type"] = entity.type - risk_object["risk_object_field"] = entity.field - risk_object["risk_score"] = entity.score - risk_objects.append(risk_object) - - for entity in self.rba.threat_objects: - threat_object: dict[str, str] = dict() - threat_object["threat_object_field"] = entity.field - threat_object["threat_object_type"] = entity.type - risk_objects.append(threat_object) - return risk_objects + if self.rba is None: + raise Exception( + f"Attempting to serialize rba section of [{self.name}], however RBA section is None" + ) + """ + action.risk.param._risk + of the conf file only contains a list of dicts. We do not eant to + include the message here, so we do not return it. + """ + rba_dict = self.rba.model_dump() + return rba_dict["risk_objects"] + rba_dict["threat_objects"] @computed_field @property diff --git a/contentctl/objects/rba.py b/contentctl/objects/rba.py index d33da47c..2efffca1 100644 --- a/contentctl/objects/rba.py +++ b/contentctl/objects/rba.py @@ -1,9 +1,12 @@ -from enum import Enum -from pydantic import BaseModel, computed_field, Field +from __future__ import annotations + from abc import ABC -from typing import Set, Annotated -from contentctl.objects.enums import RiskSeverity +from enum import Enum +from typing import Annotated, Any, Set +from pydantic import BaseModel, Field, computed_field, model_serializer + +from contentctl.objects.enums import RiskSeverity RiskScoreValue_Type = Annotated[int, Field(ge=1, le=100)] @@ -42,6 +45,7 @@ class ThreatObjectType(str, Enum): TLS_HASH = "tls_hash" URL = "url" +#[{"risk_object_field": "dest", "risk_object_type": "system", "risk_score": 64}, {"threat_object_field": "src_ip", "threat_object_type": "ip_address"}] class RiskObject(BaseModel): field: str @@ -51,6 +55,28 @@ class RiskObject(BaseModel): def __hash__(self): return hash((self.field, self.type, self.score)) + def __lt__(self, other: RiskObject) -> bool: + if ( + f"{self.field}{self.type}{self.score}" + < f"{other.field}{other.type}{other.score}" + ): + return True + return False + + @model_serializer + def serialize_risk_object(self) -> dict[str, str | int]: + ''' + We define this explicitly for two reasons, even though the automatic + serialization works correctly. First we want to enforce a specific + field order for reasons of readability. Second, some of the fields + actually have different names than they do in the object. + ''' + return{ + "risk_object_field": self.field, + "risk_object_type": self.type, + "risk_score": self.score + } + class ThreatObject(BaseModel): field: str @@ -59,6 +85,24 @@ class ThreatObject(BaseModel): def __hash__(self): return hash((self.field, self.type)) + def __lt__(self, other: ThreatObject) -> bool: + if f"{self.field}{self.type}" < f"{other.field}{other.type}": + return True + return False + + @model_serializer + def serialize_threat_object(self) -> dict[str, str]: + ''' + We define this explicitly for two reasons, even though the automatic + serialization works correctly. First we want to enforce a specific + field order for reasons of readability. Second, some of the fields + actually have different names than they do in the object. + ''' + return{ + "threat_object_field": self.field, + "threat_object_type": self.type, + } + class RBAObject(BaseModel, ABC): message: str @@ -94,3 +138,11 @@ def severity(self) -> RiskSeverity: raise Exception( f"Error getting severity - risk_score must be between 0-100, but was actually {self.risk_score}" ) + + @model_serializer + def serialize_rba(self) -> dict[str, str | list[dict[str, str | int]]]: + return { + "message": self.message, + "risk_objects": [obj.model_dump() for obj in sorted(self.risk_objects)], + "threat_objects": [obj.model_dump() for obj in sorted(self.threat_objects)], + } From bc9d2ab278d8e23f094197b0de2950e4a3d7be8e Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Tue, 4 Feb 2025 11:47:14 -0800 Subject: [PATCH 02/11] linting cleanup and comment removal --- contentctl/objects/rba.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/contentctl/objects/rba.py b/contentctl/objects/rba.py index 2efffca1..a63c043e 100644 --- a/contentctl/objects/rba.py +++ b/contentctl/objects/rba.py @@ -2,7 +2,7 @@ from abc import ABC from enum import Enum -from typing import Annotated, Any, Set +from typing import Annotated, Set from pydantic import BaseModel, Field, computed_field, model_serializer @@ -45,7 +45,6 @@ class ThreatObjectType(str, Enum): TLS_HASH = "tls_hash" URL = "url" -#[{"risk_object_field": "dest", "risk_object_type": "system", "risk_score": 64}, {"threat_object_field": "src_ip", "threat_object_type": "ip_address"}] class RiskObject(BaseModel): field: str @@ -62,19 +61,19 @@ def __lt__(self, other: RiskObject) -> bool: ): return True return False - + @model_serializer def serialize_risk_object(self) -> dict[str, str | int]: - ''' - We define this explicitly for two reasons, even though the automatic - serialization works correctly. First we want to enforce a specific - field order for reasons of readability. Second, some of the fields + """ + We define this explicitly for two reasons, even though the automatic + serialization works correctly. First we want to enforce a specific + field order for reasons of readability. Second, some of the fields actually have different names than they do in the object. - ''' - return{ + """ + return { "risk_object_field": self.field, "risk_object_type": self.type, - "risk_score": self.score + "risk_score": self.score, } @@ -89,16 +88,16 @@ def __lt__(self, other: ThreatObject) -> bool: if f"{self.field}{self.type}" < f"{other.field}{other.type}": return True return False - + @model_serializer def serialize_threat_object(self) -> dict[str, str]: - ''' - We define this explicitly for two reasons, even though the automatic - serialization works correctly. First we want to enforce a specific - field order for reasons of readability. Second, some of the fields + """ + We define this explicitly for two reasons, even though the automatic + serialization works correctly. First we want to enforce a specific + field order for reasons of readability. Second, some of the fields actually have different names than they do in the object. - ''' - return{ + """ + return { "threat_object_field": self.field, "threat_object_type": self.type, } From 030ce2fc85995ce6360aead8c48852e265732189 Mon Sep 17 00:00:00 2001 From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com> Date: Wed, 5 Feb 2025 09:46:30 -0800 Subject: [PATCH 03/11] bump version to 5.0.1 in prep for patch release --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3c478963..6c1cc8d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "contentctl" -version = "5.0.0" +version = "5.0.1" description = "Splunk Content Control Tool" authors = ["STRT "] From cfc7bfd04e95ec3c635b835f0a16cb0c6a1efcca Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 5 Feb 2025 16:38:17 -0800 Subject: [PATCH 04/11] some cleanup for the inspect workflow. Be more explicit in passing the tags we want to enforce. Now we always validate using all tags, not just a specific tag based on the stack_type. We are also a bit more conformant with how the appinspect api docs say to use the api --- contentctl/actions/inspect.py | 98 ++++++++++++++++++++++++----------- 1 file changed, 69 insertions(+), 29 deletions(-) diff --git a/contentctl/actions/inspect.py b/contentctl/actions/inspect.py index e5ebcb06..4a59b135 100644 --- a/contentctl/actions/inspect.py +++ b/contentctl/actions/inspect.py @@ -1,23 +1,45 @@ -import sys -from dataclasses import dataclass -import pathlib -import json import datetime -import timeit +import json +import pathlib +import sys import time +import timeit +from dataclasses import dataclass +from io import BufferedReader -from requests import Session, post, get +from requests import Session, get, post from requests.auth import HTTPBasicAuth from contentctl.objects.config import inspect -from contentctl.objects.savedsearches_conf import SavedsearchesConf from contentctl.objects.errors import ( - MetadataValidationError, DetectionIDError, DetectionMissingError, - VersionDecrementedError, + MetadataValidationError, VersionBumpingError, + VersionDecrementedError, ) +from contentctl.objects.savedsearches_conf import SavedsearchesConf + +""" +The following list includes all appinspect tags available from: +https://dev.splunk.com/enterprise/reference/appinspect/appinspecttagreference/ + +This allows contentctl to be as forward-leaning as possible in catching +any potential issues on the widest variety of stacks. +""" +INCLUDED_TAGS_LIST = [ + "aarch64_compatibility", + "ast", + "cloud", + "future", + "manual", + "packaging_standards", + "private_app", + "private_classic", + "private_victoria", + "splunk_appinspect", +] +INCLUDED_TAGS_STRING = ",".join(INCLUDED_TAGS_LIST) @dataclass(frozen=True) @@ -28,7 +50,6 @@ class InspectInputDto: class Inspect: def execute(self, config: inspect) -> str: if config.build_app or config.build_api: - self.inspectAppCLI(config) appinspect_token = self.inspectAppAPI(config) if config.enable_metadata_validation: @@ -49,10 +70,6 @@ def inspectAppAPI(self, config: inspect) -> str: session.auth = HTTPBasicAuth( config.splunk_api_username, config.splunk_api_password ) - if config.stack_type not in ["victoria", "classic"]: - raise Exception( - f"stack_type MUST be either 'classic' or 'victoria', NOT '{config.stack_type}'" - ) APPINSPECT_API_LOGIN = "https://api.splunk.com/2.0/rest/login/splunk" @@ -64,10 +81,6 @@ def inspectAppAPI(self, config: inspect) -> str: APPINSPECT_API_VALIDATION_REQUEST = ( "https://appinspect.splunk.com/v1/app/validate" ) - headers = { - "Authorization": f"bearer {authorization_bearer}", - "Cache-Control": "no-cache", - } package_path = config.getPackageFilePath(include_version=False) if not package_path.is_file(): @@ -77,18 +90,43 @@ def inspectAppAPI(self, config: inspect) -> str: "trying to 'contentctl deploy_acs' the package BEFORE running 'contentctl build'?" ) - files = { + """ + Some documentation on "files" argument for requests.post exists here: + https://docs.python-requests.org/en/latest/api/ + The type (None, INCLUDED_TAGS_STRING) is intentional, and the None is important. + In curl syntax, the request we make below is equivalent to + curl -X POST \ + -H "Authorization: bearer " \ + -H "Cache-Control: no-cache" \ + -F "app_package=@" \ + -F "included_tags=cloud" \ + --url "https://appinspect.splunk.com/v1/app/validate" + + This is confirmed by the great resource: + https://curlconverter.com/ + """ + data: dict[str, tuple[None, str] | BufferedReader] = { "app_package": open(package_path, "rb"), - "included_tags": (None, "cloud"), + "included_tags": ( + None, + INCLUDED_TAGS_STRING, + ), # tuple with None is intentional here } - res = post(APPINSPECT_API_VALIDATION_REQUEST, headers=headers, files=files) + headers = { + "Authorization": f"bearer {authorization_bearer}", + "Cache-Control": "no-cache", + } + + res = post(APPINSPECT_API_VALIDATION_REQUEST, files=data, headers=headers) res.raise_for_status() request_id = res.json().get("request_id", None) - APPINSPECT_API_VALIDATION_STATUS = f"https://appinspect.splunk.com/v1/app/validate/status/{request_id}?included_tags=private_{config.stack_type}" - headers = headers = {"Authorization": f"bearer {authorization_bearer}"} + APPINSPECT_API_VALIDATION_STATUS = ( + f"https://appinspect.splunk.com/v1/app/validate/status/{request_id}" + ) + startTime = timeit.default_timer() # the first time, wait for 40 seconds. subsequent times, wait for less. # this is because appinspect takes some time to return, so there is no sense @@ -114,7 +152,9 @@ def inspectAppAPI(self, config: inspect) -> str: raise Exception(f"Error - Unknown Appinspect API status '{status}'") # We have finished running appinspect, so get the report - APPINSPECT_API_REPORT = f"https://appinspect.splunk.com/v1/app/report/{request_id}?included_tags=private_{config.stack_type}" + APPINSPECT_API_REPORT = ( + f"https://appinspect.splunk.com/v1/app/report/{request_id}" + ) # Get human-readable HTML report headers = headers = { "Authorization": f"bearer {authorization_bearer}", @@ -159,14 +199,14 @@ def inspectAppCLI(self, config: inspect) -> None: "\t - https://dev.splunk.com/enterprise/docs/developapps/testvalidate/appinspect/useappinspectclitool/" ) from splunk_appinspect.main import ( - validate, - MODE_OPTION, APP_PACKAGE_ARGUMENT, - OUTPUT_FILE_OPTION, - LOG_FILE_OPTION, - INCLUDED_TAGS_OPTION, EXCLUDED_TAGS_OPTION, + INCLUDED_TAGS_OPTION, + LOG_FILE_OPTION, + MODE_OPTION, + OUTPUT_FILE_OPTION, TEST_MODE, + validate, ) except Exception as e: print(e) From dc56e5abb269a7846253d46c64dc8ea6dee924ba Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Wed, 5 Feb 2025 16:39:41 -0800 Subject: [PATCH 05/11] given that stack_type is now only used in the deploy workflow, move it from inspect to deploy --- contentctl/objects/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index e72a60ec..ac6cef78 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -425,7 +425,6 @@ class inspect(build): "enforcement (defaults to the latest release of the app published on Splunkbase)." ), ) - stack_type: StackType = Field(description="The type of your Splunk Cloud Stack") @field_validator("enrichments", mode="after") @classmethod @@ -496,6 +495,9 @@ class new(Config_Base): class deploy_acs(inspect): model_config = ConfigDict(validate_default=False, arbitrary_types_allowed=True) + + stack_type: StackType = Field(description="The type of your Splunk Cloud Stack") + # ignore linter error splunk_cloud_jwt_token: str = Field( exclude=True, From f48b9f220b6fddd8a44497c8e4e215d0a0982243 Mon Sep 17 00:00:00 2001 From: Jose Hernandez Date: Thu, 6 Feb 2025 12:25:23 -0500 Subject: [PATCH 06/11] because we care --- contentctl/contentctl.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/contentctl/contentctl.py b/contentctl/contentctl.py index d5864eb7..98ad8ed9 100644 --- a/contentctl/contentctl.py +++ b/contentctl/contentctl.py @@ -1,4 +1,5 @@ import pathlib +import random import sys import traceback import warnings @@ -155,6 +156,24 @@ def test_common_func(config: test_common): """ +def get_random_compliment(): + compliments = [ + "Your code is as elegant as a perfectly balanced binary tree! 🌳", + "You're the human equivalent of well-documented code! ⭐", + "Bug-free code? Must be your work! 🚀", + "You make debugging look like an art form! 🎨", + "Your commits are poetry in motion! 📝", + "You're the exception handler of excellence! 🛡️", + "Your code reviews are legendary! 👑", + "You're the MVP of the repository! 🏆", + ] + return random.choice(compliments) + + +def recognize_func(): + print(get_random_compliment()) + + def main(): print(CONTENTCTL_5_WARNING) try: @@ -210,6 +229,7 @@ def main(): "test_servers": test_servers.model_construct(**t.__dict__), "release_notes": release_notes.model_construct(**config_obj), "deploy_acs": deploy_acs.model_construct(**t.__dict__), + "recognize": tyro.conf.Subcommand(), } ) @@ -240,6 +260,8 @@ def main(): deploy_acs_func(updated_config) elif type(config) is test or type(config) is test_servers: test_common_func(config) + elif type(config) is tyro.conf.Subcommand: + recognize_func() else: raise Exception(f"Unknown command line type '{type(config).__name__}'") except FileNotFoundError as e: From f81e40ae3221d27efe068cc71804997428561ad5 Mon Sep 17 00:00:00 2001 From: Jose Hernandez Date: Thu, 6 Feb 2025 12:26:04 -0500 Subject: [PATCH 07/11] because we care --- contentctl/contentctl.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/contentctl/contentctl.py b/contentctl/contentctl.py index 98ad8ed9..1a7822f9 100644 --- a/contentctl/contentctl.py +++ b/contentctl/contentctl.py @@ -158,14 +158,18 @@ def test_common_func(config: test_common): def get_random_compliment(): compliments = [ - "Your code is as elegant as a perfectly balanced binary tree! 🌳", - "You're the human equivalent of well-documented code! ⭐", - "Bug-free code? Must be your work! 🚀", - "You make debugging look like an art form! 🎨", - "Your commits are poetry in motion! 📝", - "You're the exception handler of excellence! 🛡️", - "Your code reviews are legendary! 👑", - "You're the MVP of the repository! 🏆", + "Your detection rules are like a zero-day shield! 🛡️", + "You catch threats like it's child's play! 🎯", + "Your correlation rules are pure genius! 🧠", + "Threat actors fear your detection engineering! ⚔️", + "You're the SOC's secret weapon! 🦾", + "Your false positive rate is impressively low! 📊", + "Malware trembles at your detection logic! 🦠", + "You're the threat hunter extraordinaire! 🔍", + "Your MITRE mappings are a work of art! 🎨", + "APTs have nightmares about your detections! 👻", + "Your content testing is bulletproof! 🎯", + "You're the detection engineering MVP! 🏆", ] return random.choice(compliments) From fc02ad3c5fae4c93f7c6834b688b218e00c511d9 Mon Sep 17 00:00:00 2001 From: Nasreddine Bencherchali Date: Thu, 6 Feb 2025 18:33:35 +0100 Subject: [PATCH 08/11] Update contentctl.py --- contentctl/contentctl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contentctl/contentctl.py b/contentctl/contentctl.py index 1a7822f9..12fe5465 100644 --- a/contentctl/contentctl.py +++ b/contentctl/contentctl.py @@ -233,7 +233,7 @@ def main(): "test_servers": test_servers.model_construct(**t.__dict__), "release_notes": release_notes.model_construct(**config_obj), "deploy_acs": deploy_acs.model_construct(**t.__dict__), - "recognize": tyro.conf.Subcommand(), + "recognize": tyro.conf.subcommand(), } ) @@ -264,7 +264,7 @@ def main(): deploy_acs_func(updated_config) elif type(config) is test or type(config) is test_servers: test_common_func(config) - elif type(config) is tyro.conf.Subcommand: + elif type(config) is tyro.conf.subcommand: recognize_func() else: raise Exception(f"Unknown command line type '{type(config).__name__}'") From 1daa8bc68a9efba2f71dc0d93255bad1f2f5f7f5 Mon Sep 17 00:00:00 2001 From: Jose Hernandez Date: Thu, 6 Feb 2025 12:34:51 -0500 Subject: [PATCH 09/11] fixing errors --- contentctl/contentctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentctl/contentctl.py b/contentctl/contentctl.py index 1a7822f9..13e2e1c6 100644 --- a/contentctl/contentctl.py +++ b/contentctl/contentctl.py @@ -233,7 +233,7 @@ def main(): "test_servers": test_servers.model_construct(**t.__dict__), "release_notes": release_notes.model_construct(**config_obj), "deploy_acs": deploy_acs.model_construct(**t.__dict__), - "recognize": tyro.conf.Subcommand(), + "recognize": tyro.conf.subcommand(), # type: ignore } ) From 9ff5e2d4b7f18aaf5f3dc9cb6b88cfbd27ebf680 Mon Sep 17 00:00:00 2001 From: Jose Hernandez Date: Thu, 6 Feb 2025 12:37:22 -0500 Subject: [PATCH 10/11] fixing more errors --- contentctl/contentctl.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/contentctl/contentctl.py b/contentctl/contentctl.py index 13e2e1c6..ff1e0628 100644 --- a/contentctl/contentctl.py +++ b/contentctl/contentctl.py @@ -3,6 +3,7 @@ import sys import traceback import warnings +from dataclasses import dataclass import tyro @@ -178,6 +179,13 @@ def recognize_func(): print(get_random_compliment()) +@dataclass +class RecognizeCommand: + """Dummy subcommand for 'recognize' that requires no parameters.""" + + pass + + def main(): print(CONTENTCTL_5_WARNING) try: @@ -233,7 +241,7 @@ def main(): "test_servers": test_servers.model_construct(**t.__dict__), "release_notes": release_notes.model_construct(**config_obj), "deploy_acs": deploy_acs.model_construct(**t.__dict__), - "recognize": tyro.conf.subcommand(), # type: ignore + "recognize": RecognizeCommand(), # Use the dummy subcommand } ) @@ -264,7 +272,7 @@ def main(): deploy_acs_func(updated_config) elif type(config) is test or type(config) is test_servers: test_common_func(config) - elif type(config) is tyro.conf.Subcommand: + elif isinstance(config, RecognizeCommand): recognize_func() else: raise Exception(f"Unknown command line type '{type(config).__name__}'") From 40e07bf3627b514891e008a4d8ecae92e7f4ad23 Mon Sep 17 00:00:00 2001 From: Jose Hernandez Date: Thu, 6 Feb 2025 12:41:53 -0500 Subject: [PATCH 11/11] tyro shake fist --- contentctl/contentctl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contentctl/contentctl.py b/contentctl/contentctl.py index d4a8bcbf..2541dd5d 100644 --- a/contentctl/contentctl.py +++ b/contentctl/contentctl.py @@ -181,7 +181,7 @@ def recognize_func(): @dataclass class RecognizeCommand: - """Dummy subcommand for 'recognize' that requires no parameters.""" + """Dummy subcommand for 'recognize' with no parameters.""" pass @@ -241,7 +241,7 @@ def main(): "test_servers": test_servers.model_construct(**t.__dict__), "release_notes": release_notes.model_construct(**config_obj), "deploy_acs": deploy_acs.model_construct(**t.__dict__), - "recognize": tyro.conf.subcommand(), + "recognize": RecognizeCommand(), } ) @@ -272,7 +272,7 @@ def main(): deploy_acs_func(updated_config) elif type(config) is test or type(config) is test_servers: test_common_func(config) - elif type(config) is tyro.conf.subcommand: + elif type(config) is RecognizeCommand: recognize_func() else: raise Exception(f"Unknown command line type '{type(config).__name__}'")