Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat (Core): ImportUtil refactor to improve responses #31407

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import com.dotmarketing.util.FileUtil;
import com.dotmarketing.util.ImportUtil;
import com.dotmarketing.util.Logger;
import com.dotmarketing.util.importer.model.ImportResult;
import com.google.common.hash.Hashing;
import com.liferay.portal.model.User;
import com.liferay.portal.util.Constants;
Expand Down Expand Up @@ -158,7 +159,7 @@ public void process(final Job job) throws JobProcessingException {
// Handle the import operation based on the command, by default any command that is not
// "publish" is considered preview.
final boolean isPublish = CMD_PUBLISH.equals(command);
handleImport(!isPublish, job, fileToImport, user, progressCallback);
handleImport(!isPublish, job, fileToImport, totalLines, user, progressCallback);

if (!cancellationRequested.get()) {
// Ensure the progress is at 100% when the job is done
Expand Down Expand Up @@ -325,11 +326,12 @@ public Map<String, Object> getResultMetadata(Job job) {
* @param preview Flag indicating whether the operation is a preview or publish
* @param job The import job configuration
* @param fileToImport The CSV file to be imported
* @param fileTotalLines The total number of lines in the file being processed
* @param user The user performing the import
* @param progressCallback Callback for tracking import progress
*/
private void handleImport(final boolean preview, final Job job, final File fileToImport,
final User user, final LongConsumer progressCallback) {
final long fileTotalLines, final User user, final LongConsumer progressCallback) {

if (!preview) {
AdminLogger.log(
Expand All @@ -343,9 +345,10 @@ private void handleImport(final boolean preview, final Job job, final File fileT

CsvReader csvReader = createCsvReader(reader);

final var importResults = processImport(preview, job, user, csvReader,
progressCallback);
resultMetadata = new HashMap<>(importResults);
final var importResults = processImport(
preview, job, user, csvReader, fileTotalLines, progressCallback
);
resultMetadata = JobUtil.transformToMap(importResults);
} catch (Exception e) {

try {
Expand All @@ -372,14 +375,16 @@ private void handleImport(final boolean preview, final Job job, final File fileT
* @param job - The {@link Job} being processed.
* @param user - The {@link User} performing this action.
* @param csvReader - The actual data contained in the CSV file.
* @param fileTotalLines - The total number of lines in the file being processed
* @param progressCallback - The callback function to update the progress of the job.
* @return The status of the content import performed by dotCMS. This provides information
* regarding inconsistencies, errors, warnings and/or precautions to the user.
* @throws DotDataException An error occurred when importing the CSV file.
*/
private Map<String, List<String>> processImport(final boolean preview, final Job job,
final User user, final CsvReader csvReader, final LongConsumer progressCallback)
throws DotDataException, IOException, DotSecurityException {
private ImportResult processImport(final boolean preview, final Job job,
final User user, final CsvReader csvReader, final long fileTotalLines,
final LongConsumer progressCallback
) throws DotDataException, IOException, DotSecurityException {

final var currentSiteId = getSiteIdentifier(job);
final var currentSiteName = getSiteName(job);
Expand All @@ -397,10 +402,11 @@ private Map<String, List<String>> processImport(final boolean preview, final Job
preview ? "Preview" : "Process"));
Logger.info(this, String.format("-> Content Type: %s", contentType.variable()));

return ImportUtil.importFile(importId, currentSiteId, contentType.id(), fields, preview,
language == null, user, language == null ? -1 : language.getId(),
return ImportUtil.importFileResult(importId, currentSiteId, contentType.id(), fields,
preview, language == null, user, language == null ? -1 : language.getId(),
headerInfo.headers, csvReader, headerInfo.languageCodeColumn,
headerInfo.countryCodeColumn, workflowActionId, httpReq, progressCallback);
headerInfo.countryCodeColumn, workflowActionId, fileTotalLines, httpReq,
progressCallback);
}

/**
Expand Down
48 changes: 48 additions & 0 deletions dotCMS/src/main/java/com/dotcms/jobs/business/util/JobUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,15 @@
import com.dotmarketing.util.Logger;
import com.dotmarketing.util.UtilMethods;
import com.dotmarketing.util.WebKeys;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.guava.GuavaModule;
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.github.jonpeterson.jackson.module.versioning.VersioningModule;
import com.liferay.portal.model.User;
import io.vavr.Lazy;
import io.vavr.control.Try;
import java.math.BigDecimal;
import java.math.RoundingMode;
Expand All @@ -30,6 +38,20 @@
*/
public class JobUtil {

/**
* Jackson mapper configuration and lazy initialized instance.
*/
private static final Lazy<ObjectMapper> objectMapper = Lazy.of(() -> {
ObjectMapper mapper = new ObjectMapper();
mapper.enable(SerializationFeature.INDENT_OUTPUT);
mapper.registerModule(new Jdk8Module());
mapper.registerModule(new GuavaModule());
mapper.registerModule(new JavaTimeModule());
mapper.registerModule(new VersioningModule());
mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
return mapper;
});

private JobUtil() {
throw new IllegalStateException("Utility class");
}
Expand Down Expand Up @@ -160,4 +182,30 @@ public static <T extends JobEvent> void sendEvents(final T event) {
.onFailure(e -> Logger.error(JobUtil.class, e.getMessage()));
}

/**
* Converts the given object into a Map representation. This method uses the configured
* ObjectMapper to transform the provided object into a Map with keys as strings and values as
* objects.
*
* @param toTransform The object to be converted into a Map
* @return A Map containing the key-value pairs representing the structure of the input object
*/
public static Map<String, Object> transformToMap(final Object toTransform) {
return objectMapper.get().convertValue(toTransform, Map.class);
}

/**
* Converts the given object into its JSON string representation using the configured
* ObjectMapper. The ObjectMapper is configured with various modules (JDK8, Guava, JavaTime,
* Versioning) and has pretty printing enabled.
*
* @param toTransform The object to be transformed into a JSON string
* @return A JSON string representation of the input object
* @throws JsonProcessingException If the object cannot be serialized to JSON
*/
public static String transformToString(final Object toTransform)
throws JsonProcessingException {
return objectMapper.get().writeValueAsString(toTransform);
}

}
Loading
Loading