Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/teams traffic alerts #2070

Merged
merged 7 commits into from
Feb 12, 2025
Merged

Feature/teams traffic alerts #2070

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
35d1e64
initial feat
notshivansh Feb 7, 2025
c691539
complete traffic alerts for teams
notshivansh Feb 7, 2025
2b2bc90
fix unit tests
notshivansh Feb 7, 2025
6b5f33f
move unit test
notshivansh Feb 8, 2025
086e36f
Merge branch 'master' into feature/teams_traffic_alerts
notshivansh Feb 12, 2025
f49f1ab
fix logs
notshivansh Feb 12, 2025
f10b755
fix alert
notshivansh Feb 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 51 additions & 11 deletions apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@
import com.akto.dto.notifications.CustomWebhook.ActiveStatus;
import com.akto.dto.notifications.CustomWebhook.WebhookOptions;
import com.akto.dto.notifications.CustomWebhook.WebhookType;
import com.akto.dto.notifications.CustomWebhookResult;
import com.akto.dto.notifications.SlackWebhook;
import com.akto.dto.pii.PIISource;
import com.akto.dto.pii.PIIType;
Expand All @@ -72,6 +71,9 @@
import com.akto.log.LoggerMaker;
import com.akto.log.LoggerMaker.LogDb;
import com.akto.mixpanel.AktoMixpanel;
import com.akto.notifications.TrafficUpdates;
import com.akto.notifications.TrafficUpdates.AlertResult;
import com.akto.notifications.TrafficUpdates.AlertType;
import com.akto.notifications.slack.DailyUpdate;
import com.akto.notifications.slack.TestSummaryGenerator;
import com.akto.notifications.webhook.WebhookSender;
Expand All @@ -80,14 +82,12 @@
import com.akto.stigg.StiggReporterClient;
import com.akto.task.Cluster;
import com.akto.telemetry.TelemetryJob;
import com.akto.test_editor.TemplateSettingsUtil;
import com.akto.testing.ApiExecutor;
import com.akto.testing.HostDNSLookup;
import com.akto.testing.TemplateMapper;
import com.akto.usage.UsageMetricCalculator;
import com.akto.usage.UsageMetricHandler;
import com.akto.testing.workflow_node_executor.Utils;
import com.akto.util.enums.GlobalEnums;
import com.akto.util.filter.DictionaryFilter;
import com.akto.utils.jobs.JobUtils;
import com.akto.utils.jobs.MatchingJob;
Expand Down Expand Up @@ -115,7 +115,6 @@
import com.akto.utils.jobs.DeactivateCollections;
import com.akto.utils.billing.OrganizationUtils;
import com.akto.utils.crons.Crons;
import com.akto.utils.notifications.TrafficUpdates;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.akto.billing.UsageMetricUtils;
import com.google.gson.Gson;
Expand Down Expand Up @@ -238,19 +237,35 @@ public void accept(Account t) {
}
}


// look back period 6 days
loggerMaker.infoAndAddToDb("starting traffic alert scheduler", LoggerMaker.LogDb.DASHBOARD);
TrafficUpdates trafficUpdates = new TrafficUpdates(60*60*24*6);
trafficUpdates.populate(deactivatedHosts);

boolean slackWebhookFound = true;
List<SlackWebhook> listWebhooks = SlackWebhooksDao.instance.findAll(new BasicDBObject());
if (listWebhooks == null || listWebhooks.isEmpty()) {
loggerMaker.infoAndAddToDb("No slack webhooks found", LogDb.DASHBOARD);
slackWebhookFound = false;
}

boolean teamsWebhooksFound = true;
List<CustomWebhook> teamsWebhooks = CustomWebhooksDao.instance.findAll(
Filters.and(
Filters.eq(
CustomWebhook.WEBHOOK_TYPE,
CustomWebhook.WebhookType.MICROSOFT_TEAMS.name()),
Filters.in(CustomWebhook.SELECTED_WEBHOOK_OPTIONS,
CustomWebhook.WebhookOptions.TRAFFIC_ALERTS.name())));

if (teamsWebhooks == null || teamsWebhooks.isEmpty()) {
loggerMaker.infoAndAddToDb("No teams webhooks found", LogDb.DASHBOARD);
teamsWebhooksFound = false;
}

if (!(slackWebhookFound || teamsWebhooksFound)) {
return;
}
SlackWebhook webhook = listWebhooks.get(0);
loggerMaker.infoAndAddToDb("Slack Webhook found: " + webhook.getWebhook(), LogDb.DASHBOARD);

int thresholdSeconds = defaultTrafficAlertThresholdSeconds;
AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter());
Expand All @@ -262,7 +277,24 @@ public void accept(Account t) {
loggerMaker.infoAndAddToDb("threshold seconds: " + thresholdSeconds, LoggerMaker.LogDb.DASHBOARD);

if (thresholdSeconds > 0) {
trafficUpdates.sendAlerts(webhook.getWebhook(),webhook.getDashboardUrl()+"/dashboard/settings#Metrics", thresholdSeconds, deactivatedHosts);
Map<AlertType, AlertResult> alertMap = trafficUpdates.createAlerts(thresholdSeconds, deactivatedHosts);
if (slackWebhookFound && listWebhooks != null && !listWebhooks.isEmpty()) {
SlackWebhook webhook = listWebhooks.get(0);
loggerMaker.infoAndAddToDb("Slack Webhook found: " + webhook.getWebhook(),
LogDb.DASHBOARD);
trafficUpdates.sendSlackAlerts(webhook.getWebhook(),
getMetricsUrl(webhook.getDashboardUrl()), thresholdSeconds,
alertMap);
}

if (teamsWebhooksFound && teamsWebhooks != null && !teamsWebhooks.isEmpty()) {
for (CustomWebhook webhook : teamsWebhooks) {
trafficUpdates.sendTeamsAlerts(webhook,
getMetricsUrl(webhook.getDashboardUrl()),
thresholdSeconds, alertMap);
}
}
trafficUpdates.updateAlertSentTs(alertMap);
}
} catch (Exception e) {
loggerMaker.errorAndAddToDb(e,"Error while running traffic alerts: " + e.getMessage(), LogDb.DASHBOARD);
Expand All @@ -273,6 +305,10 @@ public void accept(Account t) {
}, 0, 4, TimeUnit.HOURS);
}

private static String getMetricsUrl(String ogUrl){
return ogUrl + "/dashboard/settings/metrics";
}

public void setUpAktoMixpanelEndpointsScheduler(){
scheduler.scheduleAtFixedRate(new Runnable() {
public void run() {
Expand Down Expand Up @@ -946,14 +982,18 @@ public static void webhookSenderUtil(CustomWebhook webhook) {
}

/*
* TESTING_RUN_RESULTS type webhooks are
* TESTING_RUN_RESULTS type webhooks are
* triggered only on test complete not periodically.
*
* TRAFFIC_ALERTS type webhooks have a separate cron.
*/

if (webhook.getSelectedWebhookOptions() != null &&
!webhook.getSelectedWebhookOptions().isEmpty()
&& webhook.getSelectedWebhookOptions()
.contains(CustomWebhook.WebhookOptions.TESTING_RUN_RESULTS)) {
&& (webhook.getSelectedWebhookOptions()
.contains(CustomWebhook.WebhookOptions.TESTING_RUN_RESULTS)
|| webhook.getSelectedWebhookOptions()
.contains(CustomWebhook.WebhookOptions.TRAFFIC_ALERTS))) {
return;
}

Expand Down
5 changes: 5 additions & 0 deletions ...aris_web/web/src/apps/dashboard/pages/settings/integrations/webhookLayout/WebhookCore.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ function WebhookCore(props) {
}, [])

let customWebhookOptions = [
{ "type":"TRAFFIC", "title": "Traffic alerts", "value": "TRAFFIC_ALERTS", "collectionSelection": false },
{ "type":"TRAFFIC", "title": "New endpoint", "value": "NEW_ENDPOINT", "collectionSelection": true, "collectionStateField": "newEndpointCollections" },
{ "type":"TRAFFIC", "title": "New endpoint count", "value": "NEW_ENDPOINT_COUNT", "collectionSelection": false },
{ "type":"TRAFFIC", "title": "New sensitive endpoint", "value": "NEW_SENSITIVE_ENDPOINT", "collectionSelection": true, "collectionStateField": "newSensitiveEndpointCollections" },
Expand Down Expand Up @@ -331,6 +332,8 @@ function WebhookCore(props) {
<LegacyCard title={CardTitle} key="options" actions={[{content: actionContent, onAction: toggleShowOptions}]}>
<LegacyCard.Section>
{CardComponent}
{(webhook.selectedWebhookOptions && !webhook.selectedWebhookOptions.includes("TRAFFIC_ALERTS")) ?
(<>
<Divider />
<div style={{ paddingTop: "10px" }}>
<Text variant="headingMd">Run every</Text>
Expand All @@ -346,6 +349,8 @@ function WebhookCore(props) {
</span>
))}
</div>
</>) : null
}
<br />
{(webhook.selectedWebhookOptions && webhook.selectedWebhookOptions.includes("API_THREAT_PAYLOADS")) ? (
<>
Expand Down
7 changes: 0 additions & 7 deletions libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,6 @@ private static void operationForCollectionId(List<TestingEndpoints> conditions,
}

public static void addToCollectionsForCollectionId(List<TestingEndpoints> conditions, int apiCollectionId) {
logger.info("Started adding apis for in addToCollectionsForCollectionId: " + apiCollectionId);
Bson update = Updates.addToSet(SingleTypeInfo._COLLECTION_IDS, apiCollectionId);
Bson matchFilter = Filters.nin(SingleTypeInfo._COLLECTION_IDS, apiCollectionId);
operationForCollectionId(conditions, apiCollectionId, update, matchFilter, false);
Expand Down Expand Up @@ -233,9 +232,6 @@ public void run() {
MCollection<?>[] collections = collectionsEntry.getValue();
Bson filter = filtersMap.get(type);

// printing filter size
long val = filter.toString().length();
logger.info("Size of the filter in updateCollectionsForCollectionId is: " + val + "Bytes");
updateCollections(collections, filter, update);
}

Expand All @@ -246,9 +242,6 @@ public void run() {
private static void updateCollections(MCollection<?>[] collections, Bson filter, Bson update) {
for (MCollection<?> collection : collections) {
UpdateResult res = collection.getMCollection().updateMany(filter, update);
logger.info(String.format(
"mongo update results for updating collection groups are: {matchedCount : %d} {modifiedCount : %d} {collection: %s}",
res.getMatchedCount(), res.getModifiedCount(), collection.getCollName()));
}
}

Expand Down
4 changes: 3 additions & 1 deletion libs/dao/src/main/java/com/akto/dto/notifications/CustomWebhook.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,9 @@ public enum WebhookOptions {
NEW_SENSITIVE_PARAMETER_COUNT("New Sensitive Parameter Count", "${AKTO.changes_info.newSensitiveParametersCount}"),
API_THREAT_PAYLOADS("API Threat payloads", "${AKTO.changes_info.apiThreatPayloads}"),
// optionReplaceString not being used for Testing Run results.
TESTING_RUN_RESULTS("Testing run results", "${AKTO.changes_info.apiTestingRunResults}");
TESTING_RUN_RESULTS("Testing run results", "${AKTO.changes_info.apiTestingRunResults}"),
// optionReplaceString not being used for Traffic alerts.
TRAFFIC_ALERTS("Traffic alerts", "${AKTO.changes_info.apiTrafficAlerts}");

final String optionName;
final String optionReplaceString;
Expand Down
65 changes: 43 additions & 22 deletions ...o/utils/notifications/TrafficUpdates.java → ...om/akto/notifications/TrafficUpdates.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,23 +1,21 @@
package com.akto.utils.notifications;
package com.akto.notifications;

import com.akto.calendar.DateUtils;
import com.akto.dao.ApiCollectionsDao;
import com.akto.dao.context.Context;
import com.akto.dao.traffic_metrics.TrafficMetricsAlertsDao;
import com.akto.dao.traffic_metrics.TrafficMetricsDao;
import com.akto.dto.ApiCollection;
import com.akto.dto.notifications.CustomWebhook;
import com.akto.dto.traffic_metrics.TrafficMetrics;
import com.akto.dto.traffic_metrics.TrafficMetricsAlert;
import com.akto.log.LoggerMaker;
import com.akto.log.LoggerMaker.LogDb;
import com.akto.notifications.slack.DailyUpdate;
import com.akto.runtime.Main;
import com.akto.usage.UsageMetricCalculator;
import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.model.*;
import com.slack.api.Slack;
import org.apache.commons.collections.ArrayStack;
import org.bson.Document;
import org.bson.conversions.Bson;

Expand All @@ -32,12 +30,12 @@ public TrafficUpdates(int lookBackPeriod) {
this.lookBackPeriod = lookBackPeriod;
}

enum AlertType {
public enum AlertType {
OUTGOING_REQUESTS_MIRRORING,
FILTERED_REQUESTS_RUNTIME
}

private static final LoggerMaker loggerMaker = new LoggerMaker(TrafficUpdates.class);
private static final LoggerMaker loggerMaker = new LoggerMaker(TrafficUpdates.class, LogDb.DASHBOARD);

public void populate(List<String> deactivatedHosts) {

Expand All @@ -50,19 +48,22 @@ public void populate(List<String> deactivatedHosts) {
loggerMaker.infoAndAddToDb("Finished populateTrafficDetails for " + AlertType.FILTERED_REQUESTS_RUNTIME, LoggerMaker.LogDb.DASHBOARD);
}

public void sendAlerts(String webhookUrl, String metricsUrl, int thresholdSeconds, List<String> deactivatedHosts) {
public Map<AlertType, AlertResult> createAlerts(int thresholdSeconds, List<String> deactivatedHosts) {
Bson filter = deactivatedHosts != null && !deactivatedHosts.isEmpty() ? Filters.nin(TrafficMetricsAlert.HOST, deactivatedHosts) : Filters.empty();
List<TrafficMetricsAlert> trafficMetricsAlertList = TrafficMetricsAlertsDao.instance.findAll(filter);
List<TrafficMetricsAlert> filteredTrafficMetricsAlertsList = filterTrafficMetricsAlertsList(trafficMetricsAlertList);
loggerMaker.infoAndAddToDb("filteredTrafficMetricsAlertsList: " + filteredTrafficMetricsAlertsList.size(), LoggerMaker.LogDb.DASHBOARD);

loggerMaker.infoAndAddToDb("Starting sendAlerts for " + AlertType.FILTERED_REQUESTS_RUNTIME, LoggerMaker.LogDb.DASHBOARD);
sendAlerts(thresholdSeconds,AlertType.OUTGOING_REQUESTS_MIRRORING, filteredTrafficMetricsAlertsList, webhookUrl, metricsUrl);
loggerMaker.infoAndAddToDb("Finished sendAlerts for " + AlertType.FILTERED_REQUESTS_RUNTIME, LoggerMaker.LogDb.DASHBOARD);
Map<AlertType, AlertResult> alertMap = new HashMap<>();
loggerMaker.infoAndAddToDb("Creating alerts for " + AlertType.FILTERED_REQUESTS_RUNTIME, LoggerMaker.LogDb.DASHBOARD);

loggerMaker.infoAndAddToDb("Starting sendAlerts for " + AlertType.FILTERED_REQUESTS_RUNTIME, LoggerMaker.LogDb.DASHBOARD);
sendAlerts(thresholdSeconds, AlertType.FILTERED_REQUESTS_RUNTIME, filteredTrafficMetricsAlertsList, webhookUrl, metricsUrl);
loggerMaker.infoAndAddToDb("Finished sendAlerts for " + AlertType.FILTERED_REQUESTS_RUNTIME, LoggerMaker.LogDb.DASHBOARD);
for (AlertType alertType : AlertType.values()) {
loggerMaker.infoAndAddToDb("Starting createAlerts for " + alertType, LoggerMaker.LogDb.DASHBOARD);
AlertResult alertResult = createAlert(thresholdSeconds, alertType, filteredTrafficMetricsAlertsList);
alertMap.put(alertType, alertResult);
loggerMaker.infoAndAddToDb("Finished createAlerts for " + alertType, LoggerMaker.LogDb.DASHBOARD);
}
return alertMap;
}

public List<TrafficMetricsAlert> filterTrafficMetricsAlertsList(List<TrafficMetricsAlert> trafficMetricsAlertList) {
Expand Down Expand Up @@ -173,8 +174,6 @@ public void sendRedAlert(Set<String> hosts, AlertType alertType, String webhookU
} catch (IOException e) {
e.printStackTrace();
}

updateTsFieldHostWise(hosts, alertType, Context.now(), true);
}

public static String generateRedAlertPayload(Set<String> hosts, AlertType alertType, String metricsUrl) {
Expand Down Expand Up @@ -227,8 +226,6 @@ public void sendGreenAlert(Set<String> hosts, AlertType alertType, String webhoo
} catch (IOException e) {
e.printStackTrace();
}

updateTsFieldHostWise(hosts, alertType, Context.now(), false);
}

public static String generateGreenAlertPayload(Set<String> hosts, AlertType alertType, String metricsUrl) {
Expand All @@ -245,13 +242,13 @@ public static String generateGreenAlertPayload(Set<String> hosts, AlertType aler
return null;
}


BasicDBList sectionsList = new BasicDBList();
sectionsList.add(DailyUpdate.createSimpleBlockText(text));
BasicDBObject ret = new BasicDBObject("blocks", sectionsList);
return ret.toJson();
}

// call this separately.
public static void updateTsFieldHostWise(Set<String> hosts, AlertType alertType, int ts, boolean isRed) {

String fieldName;
Expand Down Expand Up @@ -311,12 +308,36 @@ public static AlertResult generateAlertResult(int thresholdSeconds, AlertType al
return new AlertResult(redAlertHosts, greenAlertHosts);
}

public void sendAlerts(int thresholdSeconds, AlertType alertType, List<TrafficMetricsAlert> trafficMetricsAlertList,
String webhookUrl, String metricsUrl) {
AlertResult alertResult = generateAlertResult(thresholdSeconds, alertType, trafficMetricsAlertList);
public void sendSlackAlerts(String webhookUrl, String metricsUrl, int thresholdSeconds, Map<AlertType, AlertResult> alertMap) {
for (AlertType alertType : alertMap.keySet()) {
loggerMaker.infoAndAddToDb("Starting sendSlackAlerts for " + alertType, LoggerMaker.LogDb.DASHBOARD);
actuallySendSlackAlerts(alertType, webhookUrl, metricsUrl, alertMap.get(alertType));
loggerMaker.infoAndAddToDb("Finished sendSlackAlerts for " + alertType, LoggerMaker.LogDb.DASHBOARD);
}
}

public void sendTeamsAlerts(CustomWebhook webhook, String metricsUrl, int thresholdSeconds, Map<AlertType, AlertResult> alertMap) {
for (AlertType alertType : alertMap.keySet()) {
loggerMaker.infoAndAddToDb("Starting sendTeamsAlerts for " + alertType, LoggerMaker.LogDb.DASHBOARD);
TrafficUpdatesTeams.createAndSendTeamsTrafficAlerts(alertType, webhook, metricsUrl, alertMap.get(alertType));
loggerMaker.infoAndAddToDb("Finished sendTeamsAlerts for " + alertType, LoggerMaker.LogDb.DASHBOARD);
}
}

private void actuallySendSlackAlerts(AlertType alertType, String webhookUrl, String metricsUrl, AlertResult alertResult) {
if (!alertResult.redAlertHosts.isEmpty()) sendRedAlert(alertResult.redAlertHosts, alertType, webhookUrl, metricsUrl);
if (!alertResult.greenAlertHosts.isEmpty()) sendGreenAlert(alertResult.greenAlertHosts, alertType, webhookUrl, metricsUrl);
}

private AlertResult createAlert(int thresholdSeconds, AlertType alertType, List<TrafficMetricsAlert> trafficMetricsAlertList) {
return generateAlertResult(thresholdSeconds, alertType, trafficMetricsAlertList);
}

public void updateAlertSentTs(Map<AlertType, AlertResult> alertMap) {
for (AlertType alertType : alertMap.keySet()) {
updateTsFieldHostWise(alertMap.get(alertType).redAlertHosts, alertType, Context.now(), true);
updateTsFieldHostWise(alertMap.get(alertType).greenAlertHosts, alertType, Context.now(), false);
}
}

}
Loading
Loading