S3AttachmentContentStore

celements-s3/pom.xml

@@ -0,0 +1,45 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"
+>
+  <parent>
+    <groupId>com.celements</groupId>
+    <artifactId>celements</artifactId>
+    <version>7.0-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>celements-s3</artifactId>
+  <version>7.0-SNAPSHOT</version>
+  <description>Celements S3 Integration</description>
+  <dependencies>
+    <dependency>
+      <groupId>com.celements</groupId>
+      <artifactId>celements-config-source</artifactId>
+      <version>7.0-SNAPSHOT</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.celements</groupId>
+      <artifactId>celements-servlet</artifactId>
+      <version>7.0-SNAPSHOT</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.celements</groupId>
+      <artifactId>celements-model</artifactId>
+      <version>7.0-SNAPSHOT</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>s3</artifactId>
+      <version>2.41.5</version>
+    </dependency>
+  </dependencies>
+  <scm>
+    <connection>scm:git:git@github.com:celements/celements-base.git</connection>
+    <developerConnection>scm:git:git@github.com:celements/celements-base.git</developerConnection>
+    <url>https://github.com/celements/celements-base/tree/dev/celements-s3</url>
+    <tag>HEAD</tag>
+  </scm>
+</project>

celements-s3/src/main/java/com/celements/store/s3/S3Config.java

@@ -0,0 +1,98 @@
+package com.celements.store.s3;
+
+import java.net.URI;
+import java.util.Optional;
+
+import javax.annotation.Nullable;
+import javax.inject.Inject;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.xwiki.configuration.ConfigurationSource;
+
+import com.celements.configuration.CelementsAllPropertiesConfigurationSource;
+
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
+import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+
+@Configuration
+public class S3Config {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(S3Config.class);
+
+  private final ConfigurationSource cfgSrc;
+
+  @Inject
+  public S3Config(CelementsAllPropertiesConfigurationSource cfgSrc) {
+    this.cfgSrc = cfgSrc;
+  }
+
+  @Bean(destroyMethod = "close")
+  @Nullable
+  public S3Client s3Client() {
+    var endpoint = cfgSrc.getProperty("celements.s3.endpoint", "").trim();
+    var region = cfgSrc.getProperty("celements.s3.region", "eu-central").trim();
+    if (endpoint.isEmpty()) {
+      LOGGER.info("S3 endpoint not configured");
+      return null;
+    }
+    var client = S3Client.builder()
+        .endpointOverride(URI.create(endpoint))
+        .region(Region.of(region))
+        .credentialsProvider(StaticCredentialsProvider.create(buildCredentials()))
+        .requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
+        .responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED)
+        .build();
+    testClient(client);
+    LOGGER.info("S3 configured: {}", endpoint);
+    return client;
+  }
+
+  private void testClient(S3Client client) {
+    try {
+      client.listBuckets();
+    } catch (Exception exc) {
+      client.close();
+      throw exc;
+    }
+  }
+
+  private AwsCredentials buildCredentials() {
+    var accessKey = cfgSrc.getProperty("celements.s3.accessKey", "").trim();
+    var secretKey = cfgSrc.getProperty("celements.s3.secretKey", "").trim();
+    if (accessKey.isEmpty() || secretKey.isEmpty()) {
+      throw new IllegalArgumentException("celements.s3.accessKey/secretKey missing");
+    }
+    return AwsBasicCredentials.builder()
+        .accessKeyId(accessKey)
+        .secretAccessKey(secretKey)
+        .build();
+  }
+
+  @Bean(name = "s3BucketFilebase")
+  @Nullable
+  public String s3BucketFilebase(Optional<S3Client> s3Client) {
+    var bucket = cfgSrc.getProperty("celements.s3.bucket.filebase", "").trim();
+    if (bucket.isEmpty()) {
+      LOGGER.info("S3 filebase bucket not configured");
+      return null;
+    }
+    testBucket(s3Client, bucket);
+    LOGGER.info("S3 filebase bucket configured: {}", bucket);
+    return bucket;
+  }
+
+  private void testBucket(Optional<S3Client> s3Client, String bucket) {
+    s3Client
+        .orElseThrow(() -> new IllegalStateException("S3 client not configured"))
+        .headBucket(builder -> builder.bucket(bucket));
+  }
+
+}

celements-s3/src/main/java/com/celements/store/s3/att/S3AttachmentContentStore.java

@@ -0,0 +1,182 @@
+package com.celements.store.s3.att;
+
+import java.util.List;
+import java.util.Optional;
+
+import javax.inject.Inject;
+import javax.inject.Named;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Component;
+
+import com.celements.servlet.NodeConfig.NodeIdentity;
+import com.xpn.xwiki.doc.XWikiAttachment;
+import com.xpn.xwiki.doc.XWikiAttachmentContent;
+import com.xpn.xwiki.store.AttachmentContentStore;
+
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
+import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+
+@Component
+@Named(S3AttachmentContentStore.STORE_NAME)
+@Lazy
+public class S3AttachmentContentStore implements AttachmentContentStore {
+
+  private static final Logger LOGGER = LoggerFactory
+      .getLogger(S3AttachmentContentStore.class);
+
+  public static final String STORE_NAME = "store.attachment.content.s3";
+
+  private final NodeIdentity nodeIdentity;
+  private final S3Client s3Client;
+  private final String s3BucketFilebase;
+
+  @Inject
+  public S3AttachmentContentStore(
+      NodeIdentity nodeIdentity,
+      Optional<S3Client> s3Client,
+      @Named("s3BucketFilebase") Optional<String> s3BucketFilebase) {
+    this.nodeIdentity = nodeIdentity;
+    this.s3Client = s3Client
+        .orElseThrow(() -> new IllegalStateException("S3Client missing"));
+    this.s3BucketFilebase = s3BucketFilebase
+        .orElseThrow(() -> new IllegalStateException("s3BucketFilebase missing"));
+  }
+
+  @Override
+  public String getStoreName() {
+    return STORE_NAME;
+  }
+
+  /**
+   * Builds the S3 key for the given attachment. The key structure is as follows:
+   * attachment/{appName}/{wikiName}/{docId}/{attachmentId}
+   */
+  public String buildS3AttachmentKey(XWikiAttachment attachment) {
+    var doc = attachment.getDoc();
+    var wiki = doc.getDocumentReference().getWikiReference();
+    return String.join("/",
+        nodeIdentity.clusterName(), // allow bucket multi-tenancy by cluster name
+        "attachments", // subbucket for attachments
+        wiki.getName(), // identify wiki
+        Long.toString(doc.getId()), // identify document
+        Long.toString(attachment.getId())); // identify attachment
+  }
+
+  /**
+   * Builds the S3 key for the given attachment. The key structure is as follows:
+   * attachment/{appName}/{wikiName}/{docId}/{attachmentId}/{version}
+   */
+  public String buildS3AttachmentVersionKey(XWikiAttachment attachment) {
+    return String.join("/",
+        buildS3AttachmentKey(attachment),
+        attachment.getVersion()); // identify attachment version
+  }
+
+  public boolean hasContent(XWikiAttachment attachment) throws AttachmentContentStoreException {
+    var s3Key = buildS3AttachmentVersionKey(attachment);
+    LOGGER.debug("hasContent - {} : {}", s3Key, attachment);
+    try {
+      s3Client.headObject(builder -> builder
+          .bucket(s3BucketFilebase)
+          .key(s3Key));
+      return true;
+    } catch (NoSuchKeyException e) {
+      return false;
+    } catch (S3Exception e) {
+      throw new AttachmentContentStoreException(buildS3ErrorMessage(s3Key, e), e);
+    } catch (Exception e) {
+      throw new AttachmentContentStoreException("Failed checking attachment", e);
+    }
+  }
+
+  @Override
+  public void saveContent(XWikiAttachmentContent content) throws AttachmentContentStoreException {
+    var s3Key = buildS3AttachmentVersionKey(content.getAttachment());
+    LOGGER.info("saveContent - {} : {}", s3Key, content.getAttachment());
+    try {
+      try (var data = content.getContentInputStream()) {
+        s3Client.putObject(builder -> builder
+            .bucket(s3BucketFilebase)
+            .key(s3Key)
+            .contentLength((long) content.getSize())
+            .contentType(content.getAttachment().getMimeType()),
+            RequestBody.fromInputStream(data, content.getSize()));
+      }
+    } catch (S3Exception e) {
+      throw new AttachmentContentStoreException(buildS3ErrorMessage(s3Key, e), e);
+    } catch (Exception e) {
+      throw new AttachmentContentStoreException("Failed saving attachment", e);
+    }
+  }
+
+  @Override
+  public void loadContent(XWikiAttachmentContent content) throws AttachmentContentStoreException {
+    var s3Key = buildS3AttachmentVersionKey(content.getAttachment());
+    try {
+      try (var data = s3Client.getObject(builder -> builder
+          .bucket(s3BucketFilebase)
+          .key(s3Key))) {
+        content.setContent(data);
+      }
+      LOGGER.debug("loadContent - {} : {}", s3Key, content.getAttachment());
+    } catch (NoSuchKeyException e) {
+      LOGGER.debug("loadContent - {} not found : {}", s3Key, content.getAttachment());
+      throw new AttachmentContentStoreException("Attachment content not found in S3: " + s3Key, e);
+    } catch (S3Exception e) {
+      throw new AttachmentContentStoreException(buildS3ErrorMessage(s3Key, e), e);
+    } catch (Exception e) {
+      throw new AttachmentContentStoreException("Failed loading attachment", e);
+    }
+  }
+
+  @Override
+  public void deleteContent(XWikiAttachment attachment) throws AttachmentContentStoreException {
+    var s3Prefix = buildS3AttachmentKey(attachment) + "/";
+    LOGGER.info("deleteContent - {} : {}", s3Prefix, attachment);
+    List<ObjectIdentifier> batch = s3Client.listObjectsV2(builder -> builder
+        .bucket(s3BucketFilebase)
+        .prefix(s3Prefix))
+        .contents()
+        .stream()
+        .map(s3Object -> ObjectIdentifier.builder().key(s3Object.key()).build())
+        .toList();
+    if (batch.isEmpty()) {
+      return;
+    } else if (batch.size() >= 1000) {
+      throw new AttachmentContentStoreException(
+          "Too many objects to delete in S3 for attachment: " + attachment, null);
+    }
+    s3Client.deleteObjects(builder -> builder
+        .bucket(s3BucketFilebase)
+        .delete(deleteBuilder -> deleteBuilder.objects(batch)));
+  }
+
+  @Override
+  public void deleteContent(XWikiAttachmentContent content) throws AttachmentContentStoreException {
+    var s3Key = buildS3AttachmentVersionKey(content.getAttachment());
+    LOGGER.info("deleteContent - {} : {}", s3Key, content.getAttachment());
+    try {
+      s3Client.deleteObject(builder -> builder
+          .bucket(s3BucketFilebase)
+          .key(s3Key));
+    } catch (S3Exception e) {
+      throw new AttachmentContentStoreException(buildS3ErrorMessage(s3Key, e), e);
+    } catch (Exception e) {
+      throw new AttachmentContentStoreException("Failed deleting attachment", e);
+    }
+  }
+
+  private static String buildS3ErrorMessage(String s3Key, S3Exception e) {
+    return String.format("S3 error for attachment (key=%s, status=%d, code=%s)",
+        s3Key,
+        e.statusCode(),
+        e.awsErrorDetails() != null ? e.awsErrorDetails().errorCode() : "n/a");
+  }
+
+}