Hackathon (Core) - Telemetry and analytics

dotCMS/src/main/java/com/dotcms/analytics/track/collectors/EventSource.java

@@ -9,7 +9,8 @@ public enum EventSource {
     DOT_CMS("DOT_CMS"),
     REST_API("REST_API"),
     WORKFLOW("WORKFLOW"),
-    RULE("RULE");
+    RULE("RULE"),
+    DOT_CLI("DOT_CLI");
 
     private final String name;
 

dotCMS/src/main/java/com/dotcms/analytics/track/collectors/EventType.java

@@ -5,12 +5,14 @@
  * @author jsanca
  */
 public enum EventType {
+    FUTURE_TIME_MACHINE_REQUEST("FUTURE_TIME_MACHINE_REQUEST"),
     VANITY_REQUEST("VANITY_REQUEST"),
     FILE_REQUEST("FILE_REQUEST"),
     PAGE_REQUEST("PAGE_REQUEST"),
     CUSTOM_USER_EVENT("CUSTOM_USER_EVENT"),
+    URL_MAP("URL_MAP"),
+    CMD_EXECUTED("CMD_EXECUTED");
 
-    URL_MAP("URL_MAP");
 
     private final String type;
     private EventType(String type) {

dotCMS/src/main/java/com/dotcms/rest/api/v1/analytics/content/ContentAnalyticsResource.java

@@ -5,27 +5,19 @@
 import com.dotcms.analytics.content.ReportResponse;
 import com.dotcms.analytics.model.ResultSetItem;
 import com.dotcms.analytics.track.collectors.Collector;
-import com.dotcms.analytics.track.collectors.EventSource;
-import com.dotcms.analytics.track.collectors.EventType;
-import com.dotcms.analytics.track.collectors.WebEventsCollectorServiceFactory;
-import com.dotcms.analytics.track.matchers.FilesRequestMatcher;
-import com.dotcms.analytics.track.matchers.PagesAndUrlMapsRequestMatcher;
-import com.dotcms.analytics.track.matchers.RequestMatcher;
-import com.dotcms.analytics.track.matchers.UserCustomDefinedRequestMatcher;
-import com.dotcms.analytics.track.matchers.VanitiesRequestMatcher;
 import com.dotcms.experiments.business.ConfigExperimentUtil;
 import com.dotcms.rest.AnonymousAccess;
 import com.dotcms.rest.InitDataObject;
 import com.dotcms.rest.ResponseEntityStringView;
 import com.dotcms.rest.WebResource;
 import com.dotcms.rest.annotation.NoCache;
+import com.dotcms.rest.api.v1.analytics.content.util.ContentAnalyticsUtil;
 import com.dotcms.util.JsonUtil;
 import com.dotmarketing.beans.Host;
 import com.dotmarketing.business.web.WebAPILocator;
 import com.dotmarketing.exception.DotSecurityException;
 import com.dotmarketing.util.Config;
 import com.dotmarketing.util.Logger;
-import com.dotmarketing.util.UUIDUtil;
 import com.google.common.annotations.VisibleForTesting;
 import com.liferay.portal.model.User;
 import io.swagger.v3.oas.annotations.Operation;
@@ -46,10 +38,7 @@
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import java.io.Serializable;
-import java.util.HashMap;
 import java.util.Map;
-import java.util.Objects;
-import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import static com.dotcms.util.DotPreconditions.checkArgument;
@@ -69,17 +58,8 @@
 @Tag(name = "Content Analytics",
         description = "This REST Endpoint exposes information related to how dotCMS content is accessed and interacted with by users.")
 public class ContentAnalyticsResource {
-
-    private static final UserCustomDefinedRequestMatcher USER_CUSTOM_DEFINED_REQUEST_MATCHER =  new UserCustomDefinedRequestMatcher();
     private final Lazy<Boolean> ANALYTICS_EVENTS_REQUIRE_AUTHENTICATION = Lazy.of(() -> Config.getBooleanProperty("ANALYTICS_EVENTS_REQUIRE_AUTHENTICATION", true));
 
-    private static final Map<String, Supplier<RequestMatcher>> MATCHER_MAP = Map.of(
-            EventType.FILE_REQUEST.getType(), FilesRequestMatcher::new,
-            EventType.PAGE_REQUEST.getType(), PagesAndUrlMapsRequestMatcher::new,
-            EventType.URL_MAP.getType(), PagesAndUrlMapsRequestMatcher::new,
-            EventType.VANITY_REQUEST.getType(), VanitiesRequestMatcher::new
-    );
-
     private final WebResource webResource;
     private final ContentAnalyticsAPI contentAnalyticsAPI;
 
@@ -359,12 +339,8 @@ public ResponseEntityStringView fireUserCustomEvent(@Context final HttpServletRe
         }
 
         Logger.debug(this,  ()->"Creating an user custom event with the payload: " + userEventPayload);
-        request.setAttribute("requestId", Objects.nonNull(request.getAttribute("requestId")) ? request.getAttribute("requestId") : UUIDUtil.uuid());
-        final Map<String, Serializable> userEventPayloadWithDefaults = new HashMap<>(userEventPayload);
-        userEventPayloadWithDefaults.put(Collector.EVENT_SOURCE, EventSource.REST_API.getName());
-        userEventPayloadWithDefaults.put(Collector.EVENT_TYPE,   userEventPayload.getOrDefault(Collector.EVENT_TYPE, EventType.CUSTOM_USER_EVENT.getType()));
-        WebEventsCollectorServiceFactory.getInstance().getWebEventsCollectorService().fireCollectorsAndEmitEvent(request, response,
-                loadRequestMatcher(userEventPayload), userEventPayloadWithDefaults, fromPayload(userEventPayload));
+
+        ContentAnalyticsUtil.registerContentAnalyticsRestEvent(request, response, userEventPayload);
 
         return new ResponseEntityStringView("User event created successfully");
     }
@@ -374,27 +350,4 @@ private boolean isNotValidKey(final Map<String, Serializable> userEventPayload,
 
         return !userEventPayload.containsKey("key") || !ConfigExperimentUtil.INSTANCE.getAnalyticsKey(site).equals(userEventPayload.get("key"));
     }
-
-    private Map<String, Object> fromPayload(final Map<String, Serializable> userEventPayload) {
-        final Map<String, Object> baseContextMap = new HashMap<>();
-
-        if (userEventPayload.containsKey("url")) {
-
-            baseContextMap.put("uri", userEventPayload.get("url"));
-        }
-
-        if (userEventPayload.containsKey("doc_path")) {
-
-            baseContextMap.put("uri", userEventPayload.get("doc_path"));
-        }
-
-        return baseContextMap;
-    }
-
-    private RequestMatcher loadRequestMatcher(final Map<String, Serializable> userEventPayload) {
-
-        String eventType = (String) userEventPayload.getOrDefault(Collector.EVENT_TYPE, EventType.CUSTOM_USER_EVENT.getType());
-        return MATCHER_MAP.getOrDefault(eventType, () -> USER_CUSTOM_DEFINED_REQUEST_MATCHER).get();
-    }
-
 }

dotCMS/src/main/java/com/dotcms/rest/api/v1/analytics/content/util/ContentAnalyticsUtil.java

@@ -0,0 +1,71 @@
+package com.dotcms.rest.api.v1.analytics.content.util;
+
+import com.dotcms.analytics.track.collectors.Collector;
+import com.dotcms.analytics.track.collectors.EventSource;
+import com.dotcms.analytics.track.collectors.EventType;
+import com.dotcms.analytics.track.collectors.WebEventsCollectorServiceFactory;
+import com.dotcms.analytics.track.matchers.FilesRequestMatcher;
+import com.dotcms.analytics.track.matchers.PagesAndUrlMapsRequestMatcher;
+import com.dotcms.analytics.track.matchers.RequestMatcher;
+import com.dotcms.analytics.track.matchers.UserCustomDefinedRequestMatcher;
+import com.dotcms.analytics.track.matchers.VanitiesRequestMatcher;
+import com.dotmarketing.util.UUIDUtil;
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.function.Supplier;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+public class ContentAnalyticsUtil {
+
+    private static final UserCustomDefinedRequestMatcher USER_CUSTOM_DEFINED_REQUEST_MATCHER =  new UserCustomDefinedRequestMatcher();
+
+    private static final Map<String, Supplier<RequestMatcher>> MATCHER_MAP = Map.of(
+            EventType.FILE_REQUEST.getType(), FilesRequestMatcher::new,
+            EventType.PAGE_REQUEST.getType(), PagesAndUrlMapsRequestMatcher::new,
+            EventType.URL_MAP.getType(), PagesAndUrlMapsRequestMatcher::new,
+            EventType.VANITY_REQUEST.getType(), VanitiesRequestMatcher::new
+    );
+
+    public static void registerContentAnalyticsRestEvent(HttpServletRequest request, HttpServletResponse response,
+            Map<String, Serializable> userEventPayload) {
+
+        request.setAttribute("requestId", Objects.nonNull(request.getAttribute("requestId")) ? request.getAttribute("requestId") : UUIDUtil.uuid());
+
+        final Map<String, Serializable> userEventPayloadWithDefaults = new HashMap<>(
+                userEventPayload);
+        userEventPayloadWithDefaults.put(Collector.EVENT_SOURCE, EventSource.REST_API.getName());
+        if (!userEventPayloadWithDefaults.containsKey(Collector.EVENT_TYPE)){
+            userEventPayloadWithDefaults.put(Collector.EVENT_TYPE,   userEventPayload.getOrDefault(Collector.EVENT_TYPE, EventType.CUSTOM_USER_EVENT.getType()));
+        }
+        WebEventsCollectorServiceFactory.getInstance().getWebEventsCollectorService().fireCollectorsAndEmitEvent(
+                request, response,
+                loadRequestMatcher(userEventPayload), userEventPayloadWithDefaults, fromPayload(
+                        userEventPayload));
+    }
+
+    private static Map<String, Object> fromPayload(final Map<String, Serializable> userEventPayload) {
+        final Map<String, Object> baseContextMap = new HashMap<>();
+
+        if (userEventPayload.containsKey("url")) {
+
+            baseContextMap.put("uri", userEventPayload.get("url"));
+        }
+
+        if (userEventPayload.containsKey("doc_path")) {
+
+            baseContextMap.put("uri", userEventPayload.get("doc_path"));
+        }
+
+        return baseContextMap;
+    }
+
+    private static RequestMatcher loadRequestMatcher(final Map<String, Serializable> userEventPayload) {
+
+        String eventType = (String) userEventPayload.getOrDefault(Collector.EVENT_TYPE, EventType.CUSTOM_USER_EVENT.getType());
+        return MATCHER_MAP.getOrDefault(eventType, () -> USER_CUSTOM_DEFINED_REQUEST_MATCHER).get();
+    }
+
+}

dotCMS/src/main/java/com/dotcms/rest/api/v1/page/PageResource.java

@@ -2,6 +2,8 @@
 
 import static com.dotcms.util.DotPreconditions.checkNotNull;
 
+import com.dotcms.analytics.track.collectors.Collector;
+import com.dotcms.analytics.track.collectors.EventType;
 import com.dotcms.api.web.HttpServletRequestThreadLocal;
 import com.dotcms.content.elasticsearch.business.ESSearchResults;
 import com.dotcms.contenttype.business.ContentTypeAPI;
@@ -17,6 +19,7 @@
 import com.dotcms.rest.ResponseEntityView;
 import com.dotcms.rest.WebResource;
 import com.dotcms.rest.annotation.NoCache;
+import com.dotcms.rest.api.v1.analytics.content.util.ContentAnalyticsUtil;
 import com.dotcms.rest.api.v1.page.ImmutablePageRenderParams.Builder;
 import com.dotcms.rest.api.v1.personalization.PersonalizationPersonaPageViewPaginator;
 import com.dotcms.rest.api.v1.workflow.WorkflowResource;
@@ -83,6 +86,8 @@
 import io.swagger.v3.oas.annotations.tags.Tag;
 import io.vavr.control.Try;
 import java.io.IOException;
+import java.io.Serializable;
+import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -143,6 +148,14 @@ public class PageResource {
     public static final String TM_HOST = "tm_host";
     public static final String DOT_CACHE = "dotcache";
 
+    private static final String URI = "uri";
+    private static final String MODE_PARAM = "modeParam";
+    private static final String LANGUAGE_ID = "languageId";
+    private static final String PERSONA_ID = "personaId";
+    private static final String DEVICE_INODE = "deviceInode";
+    private static final String TIME_MACHINE_DATE = "timeMachineDate";
+    public static final String USER = "user";
+
     private final PageResourceHelper pageResourceHelper;
     private final WebResource webResource;
     private final HTMLPageAssetRenderedAPI htmlPageAssetRenderedAPI;
@@ -245,11 +258,42 @@ public Response loadJson(@Context final HttpServletRequest originalRequest,
                 .user(user)
                 .uri(uri)
                 .asJson(true);
+
+        final Optional<Instant> timeMachineDateInstant = getTimeMachineDate(timeMachineDateAsISO8601);
+        //Logging analytics for FTM if the publishing date is older than five minutes now
+        if (timeMachineDateInstant.isPresent() && isOlderThanFiveMinutes(timeMachineDateInstant.get())) {
+            Map<String, Serializable> userEventPayload = new HashMap<>();
+            userEventPayload.put(USER, user);
+            userEventPayload.put(URI, uri);
+            userEventPayload.put(MODE_PARAM, modeParam);
+            userEventPayload.put(LANGUAGE_ID, languageId);
+            userEventPayload.put(PERSONA_ID, personaId);
+
+            if (null != deviceInode){
+                userEventPayload.put(DEVICE_INODE, deviceInode);
+            }
+
+            userEventPayload.put(TIME_MACHINE_DATE, timeMachineDateInstant.get());
+            userEventPayload.put(Collector.EVENT_TYPE, EventType.FUTURE_TIME_MACHINE_REQUEST.getType());
+
+            ContentAnalyticsUtil.registerContentAnalyticsRestEvent(originalRequest, response,
+                    userEventPayload);
+        }
         final PageRenderParams renderParams = optionalRenderParams(modeParam,
-                languageId, deviceInode, timeMachineDateAsISO8601, builder);
+                languageId, deviceInode, timeMachineDateInstant, builder);
         return getPageRender(renderParams);
     }
 
+
+    /**
+     * Verifies that a given date is older than five minutes now
+     * @param dateToCheck
+     * @return
+     */
+    private static boolean isOlderThanFiveMinutes(Instant dateToCheck) {
+        return Duration.between(Instant.now(), dateToCheck).toMinutes() >= 5;
+    }
+
     /**
      * Returns the metadata -- i.e.; the objects that make up an HTML Page, including the rendered
      * HTML code from the page and its containers -- in the form of a JSON object based on the
@@ -323,14 +367,36 @@ public Response render(@Context final HttpServletRequest originalRequest,
                         .rejectWhenNoUser(true)
                         .init();
         final User user = initData.getUser();
+
+        final Optional<Instant> timeMachineDateInstant = getTimeMachineDate(timeMachineDateAsISO8601);
         final Builder builder = ImmutablePageRenderParams.builder();
         builder.originalRequest(originalRequest)
                 .request(request)
                 .response(response)
                 .user(user)
                 .uri(uri);
         final PageRenderParams renderParams = optionalRenderParams(modeParam,
-                languageId, deviceInode, timeMachineDateAsISO8601, builder);
+                languageId, deviceInode, timeMachineDateInstant, builder);
+
+
+        //Logging analytics for FTM if the publishing date is older than five minutes now
+        if (timeMachineDateInstant.isPresent() && isOlderThanFiveMinutes(timeMachineDateInstant.get())) {
+            Map<String, Serializable> userEventPayload = new HashMap<>();
+            userEventPayload.put(USER, user);
+            userEventPayload.put(URI, uri);
+            userEventPayload.put(MODE_PARAM, modeParam);
+            userEventPayload.put(LANGUAGE_ID, languageId);
+            userEventPayload.put(PERSONA_ID, personaId);
+            if (null != deviceInode){
+                userEventPayload.put(DEVICE_INODE, deviceInode);
+            }
+
+            userEventPayload.put(TIME_MACHINE_DATE, timeMachineDateInstant.get());
+            userEventPayload.put(Collector.EVENT_TYPE, EventType.FUTURE_TIME_MACHINE_REQUEST.getType());
+
+            ContentAnalyticsUtil.registerContentAnalyticsRestEvent(originalRequest, response,
+                    userEventPayload);
+        }
         return getPageRender(renderParams);
     }
 
@@ -339,12 +405,12 @@ public Response render(@Context final HttpServletRequest originalRequest,
      * @param modeParam      The current {@link PageMode} used to render the page.
      * @param languageId    The {@link com.dotmarketing.portlets.languagesmanager.model.Language}'s
      * @param deviceInode  The {@link java.lang.String}'s inode to render the page.
-     * @param timeMachineDateAsISO8601 The date to set the Time Machine to.
+     * @param timeMachineDateInstant The date to set the Time Machine to.
      * @param builder The builder to use to create the {@link PageRenderParams}.
      * @return The {@link PageRenderParams} object.
      */
     private PageRenderParams optionalRenderParams(final String modeParam,
-            final String languageId, final String deviceInode, final String timeMachineDateAsISO8601,
+            final String languageId, final String deviceInode, final Optional<Instant> timeMachineDateInstant,
             Builder builder) {
         if (null != languageId){
             builder.languageId(languageId);
@@ -355,8 +421,15 @@ private PageRenderParams optionalRenderParams(final String modeParam,
         if (null != deviceInode){
             builder.deviceInode(deviceInode);
         }
+        if (timeMachineDateInstant.isPresent()) {
+            builder.timeMachineDate(timeMachineDateInstant.get());
+        }
+        return builder.build();
+    }
+
+    private static Optional<Instant> getTimeMachineDate(String timeMachineDateAsISO8601) {
+        final Date date;
         if (null != timeMachineDateAsISO8601) {
-            final Date date;
             try {
                 date = Try.of(() -> DateUtil.convertDate(timeMachineDateAsISO8601)).getOrElseThrow(
                         e -> new IllegalArgumentException(
@@ -365,10 +438,9 @@ private PageRenderParams optionalRenderParams(final String modeParam,
             } catch (IllegalArgumentException e) {
                 throw new RuntimeException(e);
             }
-            final Instant instant = date.toInstant();
-            builder.timeMachineDate(instant);
+            return Optional.of(date.toInstant());
         }
-        return builder.build();
+        return Optional.empty();
     }
 
     /**

dotCMS/src/main/java/com/dotcms/telemetry/MetricFeature.java

@@ -15,6 +15,7 @@ public enum MetricFeature {
     SITE_SEARCH,
     IMAGE_API,
     LAYOUT,
-    CONTENT_TYPE_FIELDS
+    CONTENT_TYPE_FIELDS,
+    AI
 
 }

dotCMS/src/main/java/com/dotcms/telemetry/collectors/MetricStatsCollector.java

@@ -6,6 +6,9 @@
 import com.dotcms.telemetry.MetricValue;
 import com.dotcms.telemetry.MetricsSnapshot;
 import com.dotcms.telemetry.business.MetricsAPI;
+import com.dotcms.telemetry.collectors.ai.TotalEmbeddingsIndexesMetricType;
+import com.dotcms.telemetry.collectors.ai.TotalSitesUsingDotaiMetricType;
+import com.dotcms.telemetry.collectors.ai.TotalSitesWithAutoIndexContentConfigMetricType;
 import com.dotcms.telemetry.collectors.api.ApiMetricAPI;
 import com.dotcms.telemetry.collectors.api.ApiMetricTypes;
 import com.dotcms.telemetry.collectors.container.TotalFileContainersInLivePageDatabaseMetricType;
@@ -223,6 +226,10 @@ private MetricStatsCollector() {
         metricStatsCollectors.add(new CountOfTimeFieldsMetricType());
         metricStatsCollectors.add(new CountOfWYSIWYGFieldsMetricType());
 
+        metricStatsCollectors.add(new TotalSitesUsingDotaiMetricType());
+        metricStatsCollectors.add(new TotalEmbeddingsIndexesMetricType());
+        metricStatsCollectors.add(new TotalSitesWithAutoIndexContentConfigMetricType());
+
         metricStatsCollectors.addAll(ApiMetricTypes.INSTANCE.get());
     }