Merge pull request #2070 from akto-api-security/feature/teams_traffic…

…_alerts

Feature/teams traffic alerts

apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java

@@ -46,7 +46,6 @@
 import com.akto.dto.notifications.CustomWebhook.ActiveStatus;
 import com.akto.dto.notifications.CustomWebhook.WebhookOptions;
 import com.akto.dto.notifications.CustomWebhook.WebhookType;
-import com.akto.dto.notifications.CustomWebhookResult;
 import com.akto.dto.notifications.SlackWebhook;
 import com.akto.dto.pii.PIISource;
 import com.akto.dto.pii.PIIType;
@@ -72,6 +71,9 @@
 import com.akto.log.LoggerMaker;
 import com.akto.log.LoggerMaker.LogDb;
 import com.akto.mixpanel.AktoMixpanel;
+import com.akto.notifications.TrafficUpdates;
+import com.akto.notifications.TrafficUpdates.AlertResult;
+import com.akto.notifications.TrafficUpdates.AlertType;
 import com.akto.notifications.slack.DailyUpdate;
 import com.akto.notifications.slack.TestSummaryGenerator;
 import com.akto.notifications.webhook.WebhookSender;
@@ -80,14 +82,12 @@
 import com.akto.stigg.StiggReporterClient;
 import com.akto.task.Cluster;
 import com.akto.telemetry.TelemetryJob;
-import com.akto.test_editor.TemplateSettingsUtil;
 import com.akto.testing.ApiExecutor;
 import com.akto.testing.HostDNSLookup;
 import com.akto.testing.TemplateMapper;
 import com.akto.usage.UsageMetricCalculator;
 import com.akto.usage.UsageMetricHandler;
 import com.akto.testing.workflow_node_executor.Utils;
-import com.akto.util.enums.GlobalEnums;
 import com.akto.util.filter.DictionaryFilter;
 import com.akto.utils.jobs.JobUtils;
 import com.akto.utils.jobs.MatchingJob;
@@ -115,7 +115,6 @@
 import com.akto.utils.jobs.DeactivateCollections;
 import com.akto.utils.billing.OrganizationUtils;
 import com.akto.utils.crons.Crons;
-import com.akto.utils.notifications.TrafficUpdates;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.akto.billing.UsageMetricUtils;
 import com.google.gson.Gson;
@@ -238,19 +237,35 @@ public void accept(Account t) {
                                 }
                             }
 
-
                             // look back period 6 days
                             loggerMaker.infoAndAddToDb("starting traffic alert scheduler", LoggerMaker.LogDb.DASHBOARD);
                             TrafficUpdates trafficUpdates = new TrafficUpdates(60*60*24*6);
                             trafficUpdates.populate(deactivatedHosts);
 
+                            boolean slackWebhookFound = true;
                             List<SlackWebhook> listWebhooks = SlackWebhooksDao.instance.findAll(new BasicDBObject());
                             if (listWebhooks == null || listWebhooks.isEmpty()) {
                                 loggerMaker.infoAndAddToDb("No slack webhooks found", LogDb.DASHBOARD);
+                                slackWebhookFound = false;
+                            }
+
+                            boolean teamsWebhooksFound = true;
+                            List<CustomWebhook> teamsWebhooks = CustomWebhooksDao.instance.findAll(
+                                    Filters.and(
+                                            Filters.eq(
+                                                    CustomWebhook.WEBHOOK_TYPE,
+                                                    CustomWebhook.WebhookType.MICROSOFT_TEAMS.name()),
+                                            Filters.in(CustomWebhook.SELECTED_WEBHOOK_OPTIONS,
+                                                    CustomWebhook.WebhookOptions.TRAFFIC_ALERTS.name())));
+
+                            if (teamsWebhooks == null || teamsWebhooks.isEmpty()) {
+                                loggerMaker.infoAndAddToDb("No teams webhooks found", LogDb.DASHBOARD);
+                                teamsWebhooksFound = false;
+                            }
+
+                            if (!(slackWebhookFound || teamsWebhooksFound)) {
                                 return;
                             }
-                            SlackWebhook webhook = listWebhooks.get(0);
-                            loggerMaker.infoAndAddToDb("Slack Webhook found: " + webhook.getWebhook(), LogDb.DASHBOARD);
 
                             int thresholdSeconds = defaultTrafficAlertThresholdSeconds;
                             AccountSettings accountSettings = AccountSettingsDao.instance.findOne(AccountSettingsDao.generateFilter());
@@ -262,7 +277,24 @@ public void accept(Account t) {
                             loggerMaker.infoAndAddToDb("threshold seconds: " + thresholdSeconds, LoggerMaker.LogDb.DASHBOARD);
 
                             if (thresholdSeconds > 0) {
-                                trafficUpdates.sendAlerts(webhook.getWebhook(),webhook.getDashboardUrl()+"/dashboard/settings#Metrics", thresholdSeconds, deactivatedHosts);
+                                Map<AlertType, AlertResult> alertMap = trafficUpdates.createAlerts(thresholdSeconds, deactivatedHosts);
+                                if (slackWebhookFound && listWebhooks != null && !listWebhooks.isEmpty()) {
+                                    SlackWebhook webhook = listWebhooks.get(0);
+                                    loggerMaker.infoAndAddToDb("Slack Webhook found: " + webhook.getWebhook(),
+                                            LogDb.DASHBOARD);
+                                    trafficUpdates.sendSlackAlerts(webhook.getWebhook(),
+                                            getMetricsUrl(webhook.getDashboardUrl()), thresholdSeconds,
+                                            alertMap);
+                                }
+
+                                if (teamsWebhooksFound && teamsWebhooks != null && !teamsWebhooks.isEmpty()) {
+                                    for (CustomWebhook webhook : teamsWebhooks) {
+                                        trafficUpdates.sendTeamsAlerts(webhook,
+                                        getMetricsUrl(webhook.getDashboardUrl()),
+                                                thresholdSeconds, alertMap);
+                                    }
+                                }
+                                trafficUpdates.updateAlertSentTs(alertMap);
                             }
                         } catch (Exception e) {
                             loggerMaker.errorAndAddToDb(e,"Error while running traffic alerts: " + e.getMessage(), LogDb.DASHBOARD);
@@ -273,6 +305,10 @@ public void accept(Account t) {
         }, 0, 4, TimeUnit.HOURS);
     }
 
+    private static String getMetricsUrl(String ogUrl){
+        return ogUrl + "/dashboard/settings/metrics";
+    }
+
     public void setUpAktoMixpanelEndpointsScheduler(){
         scheduler.scheduleAtFixedRate(new Runnable() {
             public void run() {
@@ -946,14 +982,18 @@ public static void webhookSenderUtil(CustomWebhook webhook) {
         }
 
         /*
-         * TESTING_RUN_RESULTS type webhooks are 
+         * TESTING_RUN_RESULTS type webhooks are
          * triggered only on test complete not periodically.
+         * 
+         * TRAFFIC_ALERTS type webhooks have a separate cron.
          */
 
         if (webhook.getSelectedWebhookOptions() != null &&
                 !webhook.getSelectedWebhookOptions().isEmpty()
-                && webhook.getSelectedWebhookOptions()
-                        .contains(CustomWebhook.WebhookOptions.TESTING_RUN_RESULTS)) {
+                && (webhook.getSelectedWebhookOptions()
+                        .contains(CustomWebhook.WebhookOptions.TESTING_RUN_RESULTS)
+                        || webhook.getSelectedWebhookOptions()
+                                .contains(CustomWebhook.WebhookOptions.TRAFFIC_ALERTS))) {
             return;
         }
 

apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/integrations/webhookLayout/WebhookCore.jsx

@@ -113,6 +113,7 @@ function WebhookCore(props) {
     }, [])
 
     let customWebhookOptions = [
+        { "type":"TRAFFIC", "title": "Traffic alerts", "value": "TRAFFIC_ALERTS", "collectionSelection": false },
         { "type":"TRAFFIC", "title": "New endpoint", "value": "NEW_ENDPOINT", "collectionSelection": true, "collectionStateField": "newEndpointCollections" },
         { "type":"TRAFFIC", "title": "New endpoint count", "value": "NEW_ENDPOINT_COUNT", "collectionSelection": false },
         { "type":"TRAFFIC", "title": "New sensitive endpoint", "value": "NEW_SENSITIVE_ENDPOINT", "collectionSelection": true, "collectionStateField": "newSensitiveEndpointCollections" },
@@ -331,6 +332,8 @@ function WebhookCore(props) {
         <LegacyCard title={CardTitle} key="options" actions={[{content: actionContent, onAction: toggleShowOptions}]}>
             <LegacyCard.Section>
                 {CardComponent}
+                {(webhook.selectedWebhookOptions && !webhook.selectedWebhookOptions.includes("TRAFFIC_ALERTS")) ?
+                (<>
                 <Divider />
                 <div style={{ paddingTop: "10px" }}>
                     <Text variant="headingMd">Run every</Text>
@@ -346,6 +349,8 @@ function WebhookCore(props) {
                         </span>
                     ))}
                 </div>
+                </>) : null
+                }
                 <br />
                 {(webhook.selectedWebhookOptions && webhook.selectedWebhookOptions.includes("API_THREAT_PAYLOADS")) ? (
                     <>

libs/dao/src/main/java/com/akto/dto/ApiCollectionUsers.java

@@ -170,7 +170,6 @@ private static void operationForCollectionId(List<TestingEndpoints> conditions,
     }
 
     public static void addToCollectionsForCollectionId(List<TestingEndpoints> conditions, int apiCollectionId) {
-        logger.info("Started adding apis for in addToCollectionsForCollectionId: " + apiCollectionId);
         Bson update = Updates.addToSet(SingleTypeInfo._COLLECTION_IDS, apiCollectionId);
         Bson matchFilter = Filters.nin(SingleTypeInfo._COLLECTION_IDS, apiCollectionId);
         operationForCollectionId(conditions, apiCollectionId, update, matchFilter, false);
@@ -233,9 +232,6 @@ public void run() {
                     MCollection<?>[] collections = collectionsEntry.getValue();
                     Bson filter = filtersMap.get(type);
 
-                    // printing filter size
-                    long val = filter.toString().length();
-                    logger.info("Size of the filter in updateCollectionsForCollectionId is: " + val + "Bytes");
                     updateCollections(collections, filter, update);
                 }
 
@@ -246,9 +242,6 @@ public void run() {
     private static void updateCollections(MCollection<?>[] collections, Bson filter, Bson update) {
         for (MCollection<?> collection : collections) {
             UpdateResult res = collection.getMCollection().updateMany(filter, update);
-            logger.info(String.format(
-                "mongo update results for updating collection groups are: {matchedCount : %d} {modifiedCount : %d} {collection: %s}",
-                res.getMatchedCount(), res.getModifiedCount(), collection.getCollName()));
         }
     }
 

libs/dao/src/main/java/com/akto/dto/notifications/CustomWebhook.java

@@ -56,7 +56,9 @@ public enum WebhookOptions {
         NEW_SENSITIVE_PARAMETER_COUNT("New Sensitive Parameter Count", "${AKTO.changes_info.newSensitiveParametersCount}"),
         API_THREAT_PAYLOADS("API Threat payloads", "${AKTO.changes_info.apiThreatPayloads}"),
         // optionReplaceString not being used for Testing Run results.
-        TESTING_RUN_RESULTS("Testing run results", "${AKTO.changes_info.apiTestingRunResults}");
+        TESTING_RUN_RESULTS("Testing run results", "${AKTO.changes_info.apiTestingRunResults}"),
+        // optionReplaceString not being used for Traffic alerts.
+        TRAFFIC_ALERTS("Traffic alerts", "${AKTO.changes_info.apiTrafficAlerts}");
 
         final String optionName;
         final String optionReplaceString;

apps/dashboard/src/main/java/com/akto/utils/notifications/TrafficUpdates.java → libs/utils/src/main/java/com/akto/notifications/TrafficUpdates.java

@@ -1,23 +1,21 @@
-package com.akto.utils.notifications;
+package com.akto.notifications;
 
-import com.akto.calendar.DateUtils;
 import com.akto.dao.ApiCollectionsDao;
 import com.akto.dao.context.Context;
 import com.akto.dao.traffic_metrics.TrafficMetricsAlertsDao;
 import com.akto.dao.traffic_metrics.TrafficMetricsDao;
 import com.akto.dto.ApiCollection;
+import com.akto.dto.notifications.CustomWebhook;
 import com.akto.dto.traffic_metrics.TrafficMetrics;
 import com.akto.dto.traffic_metrics.TrafficMetricsAlert;
 import com.akto.log.LoggerMaker;
+import com.akto.log.LoggerMaker.LogDb;
 import com.akto.notifications.slack.DailyUpdate;
-import com.akto.runtime.Main;
-import com.akto.usage.UsageMetricCalculator;
 import com.mongodb.BasicDBList;
 import com.mongodb.BasicDBObject;
 import com.mongodb.client.MongoCursor;
 import com.mongodb.client.model.*;
 import com.slack.api.Slack;
-import org.apache.commons.collections.ArrayStack;
 import org.bson.Document;
 import org.bson.conversions.Bson;
 
@@ -32,12 +30,12 @@ public TrafficUpdates(int lookBackPeriod) {
         this.lookBackPeriod = lookBackPeriod;
     }
 
-    enum AlertType {
+    public enum AlertType {
         OUTGOING_REQUESTS_MIRRORING,
         FILTERED_REQUESTS_RUNTIME
     }
 
-    private static final LoggerMaker loggerMaker = new LoggerMaker(TrafficUpdates.class);
+    private static final LoggerMaker loggerMaker = new LoggerMaker(TrafficUpdates.class, LogDb.DASHBOARD);
 
     public void populate(List<String> deactivatedHosts) {
 
@@ -50,19 +48,22 @@ public void populate(List<String> deactivatedHosts) {
         loggerMaker.infoAndAddToDb("Finished populateTrafficDetails for " + AlertType.FILTERED_REQUESTS_RUNTIME, LoggerMaker.LogDb.DASHBOARD);
     }
 
-    public void sendAlerts(String webhookUrl, String metricsUrl, int thresholdSeconds, List<String> deactivatedHosts) {
+    public Map<AlertType, AlertResult> createAlerts(int thresholdSeconds, List<String> deactivatedHosts) {
         Bson filter = deactivatedHosts != null && !deactivatedHosts.isEmpty() ? Filters.nin(TrafficMetricsAlert.HOST, deactivatedHosts) : Filters.empty();
         List<TrafficMetricsAlert> trafficMetricsAlertList = TrafficMetricsAlertsDao.instance.findAll(filter);
         List<TrafficMetricsAlert> filteredTrafficMetricsAlertsList = filterTrafficMetricsAlertsList(trafficMetricsAlertList);
         loggerMaker.infoAndAddToDb("filteredTrafficMetricsAlertsList: " + filteredTrafficMetricsAlertsList.size(), LoggerMaker.LogDb.DASHBOARD);
 
-        loggerMaker.infoAndAddToDb("Starting sendAlerts for " + AlertType.FILTERED_REQUESTS_RUNTIME, LoggerMaker.LogDb.DASHBOARD);
-        sendAlerts(thresholdSeconds,AlertType.OUTGOING_REQUESTS_MIRRORING, filteredTrafficMetricsAlertsList, webhookUrl, metricsUrl);
-        loggerMaker.infoAndAddToDb("Finished sendAlerts for " + AlertType.FILTERED_REQUESTS_RUNTIME, LoggerMaker.LogDb.DASHBOARD);
+        Map<AlertType, AlertResult> alertMap = new HashMap<>();
+        loggerMaker.infoAndAddToDb("Creating alerts for " + AlertType.FILTERED_REQUESTS_RUNTIME, LoggerMaker.LogDb.DASHBOARD);
 
-        loggerMaker.infoAndAddToDb("Starting sendAlerts for " + AlertType.FILTERED_REQUESTS_RUNTIME, LoggerMaker.LogDb.DASHBOARD);
-        sendAlerts(thresholdSeconds, AlertType.FILTERED_REQUESTS_RUNTIME, filteredTrafficMetricsAlertsList, webhookUrl, metricsUrl);
-        loggerMaker.infoAndAddToDb("Finished sendAlerts for " + AlertType.FILTERED_REQUESTS_RUNTIME, LoggerMaker.LogDb.DASHBOARD);
+        for (AlertType alertType : AlertType.values()) {
+            loggerMaker.infoAndAddToDb("Starting createAlerts for " + alertType, LoggerMaker.LogDb.DASHBOARD);
+            AlertResult alertResult = createAlert(thresholdSeconds, alertType, filteredTrafficMetricsAlertsList);
+            alertMap.put(alertType, alertResult);
+            loggerMaker.infoAndAddToDb("Finished createAlerts for " + alertType, LoggerMaker.LogDb.DASHBOARD);
+        }
+        return alertMap;
     }
 
     public List<TrafficMetricsAlert> filterTrafficMetricsAlertsList(List<TrafficMetricsAlert> trafficMetricsAlertList) {
@@ -173,8 +174,6 @@ public void sendRedAlert(Set<String> hosts, AlertType alertType, String webhookU
         } catch (IOException e) {
             e.printStackTrace();
         }
-
-        updateTsFieldHostWise(hosts, alertType, Context.now(), true);
     }
 
     public static String generateRedAlertPayload(Set<String> hosts, AlertType alertType, String metricsUrl) {
@@ -227,8 +226,6 @@ public void sendGreenAlert(Set<String> hosts, AlertType alertType, String webhoo
         } catch (IOException e) {
             e.printStackTrace();
         }
-
-        updateTsFieldHostWise(hosts, alertType, Context.now(), false);
     }
 
     public static String generateGreenAlertPayload(Set<String> hosts, AlertType alertType, String metricsUrl) {
@@ -245,13 +242,13 @@ public static String generateGreenAlertPayload(Set<String> hosts, AlertType aler
                 return null;
         }
 
-
         BasicDBList sectionsList = new BasicDBList();
         sectionsList.add(DailyUpdate.createSimpleBlockText(text));
         BasicDBObject ret = new BasicDBObject("blocks", sectionsList);
         return ret.toJson();
     }
 
+    // call this separately.
     public static void updateTsFieldHostWise(Set<String> hosts, AlertType alertType, int ts, boolean isRed) {
 
         String fieldName;
@@ -311,12 +308,36 @@ public static AlertResult generateAlertResult(int thresholdSeconds, AlertType al
         return new AlertResult(redAlertHosts, greenAlertHosts);
     }
 
-    public void sendAlerts(int thresholdSeconds, AlertType alertType, List<TrafficMetricsAlert> trafficMetricsAlertList,
-                           String webhookUrl, String metricsUrl) {
-        AlertResult alertResult = generateAlertResult(thresholdSeconds, alertType, trafficMetricsAlertList);
+    public void sendSlackAlerts(String webhookUrl, String metricsUrl, int thresholdSeconds, Map<AlertType, AlertResult> alertMap) {
+        for (AlertType alertType : alertMap.keySet()) {
+            loggerMaker.infoAndAddToDb("Starting sendSlackAlerts for " + alertType, LoggerMaker.LogDb.DASHBOARD);
+            actuallySendSlackAlerts(alertType, webhookUrl, metricsUrl, alertMap.get(alertType));
+            loggerMaker.infoAndAddToDb("Finished sendSlackAlerts for " + alertType, LoggerMaker.LogDb.DASHBOARD);
+        }
+    }
+
+    public void sendTeamsAlerts(CustomWebhook webhook, String metricsUrl, int thresholdSeconds, Map<AlertType, AlertResult> alertMap) {
+        for (AlertType alertType : alertMap.keySet()) {
+            loggerMaker.infoAndAddToDb("Starting sendTeamsAlerts for " + alertType, LoggerMaker.LogDb.DASHBOARD);
+            TrafficUpdatesTeams.createAndSendTeamsTrafficAlerts(alertType, webhook, metricsUrl, alertMap.get(alertType));
+            loggerMaker.infoAndAddToDb("Finished sendTeamsAlerts for " + alertType, LoggerMaker.LogDb.DASHBOARD);
+        }
+    }
 
+    private void actuallySendSlackAlerts(AlertType alertType, String webhookUrl, String metricsUrl, AlertResult alertResult) {
         if (!alertResult.redAlertHosts.isEmpty()) sendRedAlert(alertResult.redAlertHosts, alertType, webhookUrl, metricsUrl);
         if (!alertResult.greenAlertHosts.isEmpty()) sendGreenAlert(alertResult.greenAlertHosts, alertType, webhookUrl,  metricsUrl);
     }
 
+    private AlertResult createAlert(int thresholdSeconds, AlertType alertType, List<TrafficMetricsAlert> trafficMetricsAlertList) {
+        return generateAlertResult(thresholdSeconds, alertType, trafficMetricsAlertList);
+    }
+
+    public void updateAlertSentTs(Map<AlertType, AlertResult> alertMap) {
+        for (AlertType alertType : alertMap.keySet()) {
+            updateTsFieldHostWise(alertMap.get(alertType).redAlertHosts, alertType, Context.now(), true);
+            updateTsFieldHostWise(alertMap.get(alertType).greenAlertHosts, alertType, Context.now(), false);
+        }
+    }
+
 }