From 3ba82872dd26560fb7b24e114c039c61654bb94e Mon Sep 17 00:00:00 2001 From: sband Date: Fri, 12 May 2023 17:46:00 +0530 Subject: [PATCH] fix: Store the resources in S3 buckets (data file and revisions offset files) #582 --- .../org/sirix/io/cloud/CloudPlatform.java | 7 + .../cloud/CloudStorageConnectionFactory.java | 7 + .../org/sirix/io/cloud/ICloudStorage.java | 9 + .../io/cloud/amazon/AmazonS3Storage.java | 197 ++++++++++++++++++ .../cloud/amazon/AmazonS3StorageReader.java | 122 +++++++++++ .../cloud/amazon/AmazonS3StorageWriter.java | 5 + 6 files changed, 347 insertions(+) create mode 100644 bundles/sirix-core/src/main/java/org/sirix/io/cloud/CloudPlatform.java create mode 100644 bundles/sirix-core/src/main/java/org/sirix/io/cloud/CloudStorageConnectionFactory.java create mode 100644 bundles/sirix-core/src/main/java/org/sirix/io/cloud/ICloudStorage.java create mode 100644 bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3Storage.java create mode 100644 bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3StorageReader.java create mode 100644 bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3StorageWriter.java diff --git a/bundles/sirix-core/src/main/java/org/sirix/io/cloud/CloudPlatform.java b/bundles/sirix-core/src/main/java/org/sirix/io/cloud/CloudPlatform.java new file mode 100644 index 000000000..fdd4ae509 --- /dev/null +++ b/bundles/sirix-core/src/main/java/org/sirix/io/cloud/CloudPlatform.java @@ -0,0 +1,7 @@ +package org.sirix.io.cloud; + +public enum CloudPlatform { + + AWS, GCP, AZURE + +} diff --git a/bundles/sirix-core/src/main/java/org/sirix/io/cloud/CloudStorageConnectionFactory.java b/bundles/sirix-core/src/main/java/org/sirix/io/cloud/CloudStorageConnectionFactory.java new file mode 100644 index 000000000..3dbde9922 --- /dev/null +++ b/bundles/sirix-core/src/main/java/org/sirix/io/cloud/CloudStorageConnectionFactory.java @@ -0,0 +1,7 @@ +package org.sirix.io.cloud; + +public interface CloudStorageConnectionFactory { + + + +} diff --git a/bundles/sirix-core/src/main/java/org/sirix/io/cloud/ICloudStorage.java b/bundles/sirix-core/src/main/java/org/sirix/io/cloud/ICloudStorage.java new file mode 100644 index 000000000..95a3c5742 --- /dev/null +++ b/bundles/sirix-core/src/main/java/org/sirix/io/cloud/ICloudStorage.java @@ -0,0 +1,9 @@ +package org.sirix.io.cloud; + +import org.sirix.io.IOStorage; + +public interface ICloudStorage extends IOStorage { + + + +} diff --git a/bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3Storage.java b/bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3Storage.java new file mode 100644 index 000000000..4e69ab687 --- /dev/null +++ b/bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3Storage.java @@ -0,0 +1,197 @@ +package org.sirix.io.cloud.amazon; + +import java.nio.file.Path; + +import org.sirix.access.ResourceConfiguration; +import org.sirix.io.Reader; +import org.sirix.io.RevisionFileData; +import org.sirix.io.Writer; +import org.sirix.io.bytepipe.ByteHandler; +import org.sirix.io.bytepipe.ByteHandlerPipeline; +import org.sirix.io.cloud.ICloudStorage; +import org.sirix.utils.LogWrapper; +import org.slf4j.LoggerFactory; + +import com.github.benmanes.caffeine.cache.AsyncCache; + +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.core.waiters.WaiterResponse; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketResponse; +import software.amazon.awssdk.services.s3.model.NoSuchBucketException; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.waiters.S3Waiter; + +/** + * Factory to provide Amazon S3 as storage backend + * + * @Auther Sanket Band (@sband) + **/ + +public class AmazonS3Storage implements ICloudStorage { + + /** + * Data file name. + */ + private static final String FILENAME = "sirix.data"; + + /** + * Revisions file name. + */ + private static final String REVISIONS_FILENAME = "sirix.revisions"; + + /** + * Instance to local storage. + */ + private final Path file; + + /** + * S3 storage bucket name + * + */ + private String bucketName; + + private S3Client s3Client; + + /** Logger. */ + private static final LogWrapper LOGGER = new LogWrapper(LoggerFactory.getLogger(AmazonS3Storage.class)); + + /** + * Byte handler pipeline. + */ + private final ByteHandlerPipeline byteHandlerPipeline; + + /** + * Revision file data cache. + */ + private final AsyncCache cache; + + /** + * Support AWS authentication only with .aws credentials file with the required + * profile name from the creds file + */ + public AmazonS3Storage(String bucketName, String awsProfile, + String region, + boolean shouldCreateBucketIfNotExists, final ResourceConfiguration resourceConfig, + AsyncCache cache, + ByteHandlerPipeline byteHandlerPipeline) { + this.bucketName = bucketName; + this.s3Client = this.getS3Client(awsProfile,region); + /* + * If the bucket does not exist, should create a new bucket based on the boolean + */ + /* + * Exit the system if the cloud storage bucket cannot be created Alternatively, + * we could just set a flag that could be checked before creating a reader or + * writer. Return null if the bucket is not created OR does not exist But that + * would keep the user under false impression that the bucket is created OR + * exists already even if it does not exists + */ + if (!isBucketExists(bucketName, s3Client)) { + if (shouldCreateBucketIfNotExists) { + createBucket(bucketName, s3Client); + } else { + LOGGER.error(String.format("Bucket: %s, does not exists on Amazon S3 storage, exiting the system", + bucketName)); + System.exit(1); + } + } + this.cache = cache; + this.byteHandlerPipeline = byteHandlerPipeline; + this.file = resourceConfig.resourcePath; + } + + private void createBucket(String bucketName, S3Client s3Client) { + try { + S3Waiter s3Waiter = s3Client.waiter(); + CreateBucketRequest bucketRequest = CreateBucketRequest.builder().bucket(bucketName).build(); + + s3Client.createBucket(bucketRequest); + HeadBucketRequest bucketRequestWait = HeadBucketRequest.builder().bucket(bucketName).build(); + + WaiterResponse waiterResponse = s3Waiter.waitUntilBucketExists(bucketRequestWait); + if (waiterResponse.matched().response().isPresent()) { + LOGGER.info(String.format("S3 bucket: %s has been created.", bucketName)); + } + } catch (S3Exception e) { + LOGGER.error(e.awsErrorDetails().errorMessage()); + LOGGER.error(String.format("Bucket: %s could not be created. Will not consume S3 storage", bucketName)); + System.exit(1); + } + } + + private boolean isBucketExists(String bucketName, S3Client s3Client) { + HeadBucketRequest headBucketRequest = HeadBucketRequest.builder().bucket(bucketName).build(); + + try { + s3Client.headBucket(headBucketRequest); + return true; + } catch (NoSuchBucketException e) { + return false; + } + } + + private S3Client getS3Client(String awsProfile, String region) { + S3Client s3Client = null; + s3Client = S3Client.builder() + .region(Region.of(region)) + .credentialsProvider(ProfileCredentialsProvider.create(awsProfile)) + .build(); + return s3Client; + } + + @Override + public Writer createWriter() { + /* + * This would create a writer that connects to the + * remote storagee*/ + return null; + } + + @Override + public Reader createReader() { + /* + * This would create a reader that connects to the + * remote storage on cloud + * */ + return null; + } + + @Override + public void close() { + // TODO Auto-generated method stub + + } + + @Override + public boolean exists() { + // TODO Auto-generated method stub + return false; + } + + @Override + public ByteHandler getByteHandler() { + return this.byteHandlerPipeline; + } + + /** + * Getting path for data file. + * This path would be used on the local storage + * @return the path for this data file + */ + private Path getDataFilePath() { + return file.resolve(ResourceConfiguration.ResourcePaths.DATA.getPath()).resolve(FILENAME); + } + + /** + * Getting concrete storage for this file. + * + * @return the concrete storage for this database + */ + private Path getRevisionFilePath() { + return file.resolve(ResourceConfiguration.ResourcePaths.DATA.getPath()).resolve(REVISIONS_FILENAME); + } +} diff --git a/bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3StorageReader.java b/bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3StorageReader.java new file mode 100644 index 000000000..0409eaa00 --- /dev/null +++ b/bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3StorageReader.java @@ -0,0 +1,122 @@ +package org.sirix.io.cloud.amazon; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.nio.file.FileSystems; +import java.time.Instant; + +import org.checkerframework.checker.nullness.qual.Nullable; +import org.sirix.api.PageReadOnlyTrx; +import org.sirix.io.Reader; +import org.sirix.io.RevisionFileData; +import org.sirix.io.bytepipe.ByteHandler; +import org.sirix.io.file.FileReader; +import org.sirix.page.PagePersister; +import org.sirix.page.PageReference; +import org.sirix.page.RevisionRootPage; +import org.sirix.page.SerializationType; +import org.sirix.page.interfaces.Page; +import org.sirix.utils.LogWrapper; +import org.slf4j.LoggerFactory; + +import com.github.benmanes.caffeine.cache.Cache; + +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Exception; + +public class AmazonS3StorageReader implements Reader { + + /** + * S3 storage bucket name + * + */ + private String bucketName; + + private S3Client s3Client; + + /** Logger. */ + private static final LogWrapper LOGGER = new LogWrapper(LoggerFactory.getLogger(AmazonS3StorageReader.class)); + + + private FileReader reader; + + public AmazonS3StorageReader(String bucketName, S3Client s3Client, + final RandomAccessFile dataFile, + final RandomAccessFile revisionsOffsetFile, + final ByteHandler byteHandler, + final SerializationType serializationType, + final PagePersister pagePersister, + final Cache cache) { + this.bucketName = bucketName; + this.s3Client = s3Client; + this.reader = new FileReader(dataFile, + revisionsOffsetFile, + byteHandler, + serializationType, + pagePersister, + cache); + } + + private void readObjectDataFromS3(String keyName) { + + try { + GetObjectRequest objectRequest = GetObjectRequest + .builder() + .key(keyName) + .bucket(bucketName) + .build(); + + ResponseBytes objectBytes = s3Client.getObjectAsBytes(objectRequest); + byte[] data = objectBytes.asByteArray(); + String path = System.getProperty("java.io.tmpdir") + FileSystems.getDefault().getSeparator() + keyName; + // Write the data to a local file. + File myFile = new File(path); + OutputStream os = new FileOutputStream(myFile); + os.write(data); + os.close(); + } catch (IOException ex) { + ex.printStackTrace(); + } catch (S3Exception e) { + System.err.println(e.awsErrorDetails().errorMessage()); + System.exit(1); + } + } + + @Override + public PageReference readUberPageReference() { + return reader.readUberPageReference(); + } + + @Override + public Page read(PageReference key, @Nullable PageReadOnlyTrx pageReadTrx) { + return reader.read(key, pageReadTrx); + } + + @Override + public void close() { + s3Client.close(); + reader.close(); + } + + @Override + public RevisionRootPage readRevisionRootPage(int revision, PageReadOnlyTrx pageReadTrx) { + return reader.readRevisionRootPage(revision, pageReadTrx); + } + + @Override + public Instant readRevisionRootPageCommitTimestamp(int revision) { + return reader.readRevisionRootPageCommitTimestamp(revision); + } + + @Override + public RevisionFileData getRevisionFileData(int revision) { + return reader.getRevisionFileData(revision); + } + +} diff --git a/bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3StorageWriter.java b/bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3StorageWriter.java new file mode 100644 index 000000000..e2f6fd572 --- /dev/null +++ b/bundles/sirix-core/src/main/java/org/sirix/io/cloud/amazon/AmazonS3StorageWriter.java @@ -0,0 +1,5 @@ +package org.sirix.io.cloud.amazon; + +public class AmazonS3StorageWriter { + +}