diff --git a/server/skillhub-storage/pom.xml b/server/skillhub-storage/pom.xml index 19ea55a4..2aeb5cc6 100644 --- a/server/skillhub-storage/pom.xml +++ b/server/skillhub-storage/pom.xml @@ -10,4 +10,25 @@ 0.1.0-SNAPSHOT skillhub-storage + + + + org.springframework.boot + spring-boot-starter + + + org.yaml + snakeyaml + + + software.amazon.awssdk + s3 + 2.20.26 + + + org.springframework.boot + spring-boot-starter-test + test + + diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/LocalFileStorageService.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/LocalFileStorageService.java new file mode 100644 index 00000000..5fd9ee8d --- /dev/null +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/LocalFileStorageService.java @@ -0,0 +1,62 @@ +package com.iflytek.skillhub.storage; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import java.io.*; +import java.nio.file.*; +import java.nio.file.attribute.BasicFileAttributes; +import java.time.Instant; +import java.util.List; + +@Service +@ConditionalOnProperty(name = "skillhub.storage.provider", havingValue = "local", matchIfMissing = true) +public class LocalFileStorageService implements ObjectStorageService { + private final Path basePath; + + public LocalFileStorageService(StorageProperties properties) { + this.basePath = Paths.get(properties.getLocal().getBasePath()); + } + + @Override + public void putObject(String key, InputStream data, long size, String contentType) { + try { + Path target = resolve(key); + Files.createDirectories(target.getParent()); + Path tmp = target.resolveSibling(target.getFileName() + ".tmp"); + try (OutputStream out = Files.newOutputStream(tmp, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { + data.transferTo(out); + } + Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { throw new UncheckedIOException("Failed to put object: " + key, e); } + } + + @Override + public InputStream getObject(String key) { + try { return Files.newInputStream(resolve(key)); } + catch (IOException e) { throw new UncheckedIOException("Failed to get object: " + key, e); } + } + + @Override + public void deleteObject(String key) { + try { Files.deleteIfExists(resolve(key)); } + catch (IOException e) { throw new UncheckedIOException("Failed to delete object: " + key, e); } + } + + @Override + public void deleteObjects(List keys) { keys.forEach(this::deleteObject); } + + @Override + public boolean exists(String key) { return Files.exists(resolve(key)); } + + @Override + public ObjectMetadata getMetadata(String key) { + try { + Path path = resolve(key); + BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class); + return new ObjectMetadata(attrs.size(), Files.probeContentType(path), attrs.lastModifiedTime().toInstant()); + } catch (IOException e) { throw new UncheckedIOException("Failed to get metadata: " + key, e); } + } + + private Path resolve(String key) { return basePath.resolve(key); } +} diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/ObjectMetadata.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/ObjectMetadata.java new file mode 100644 index 00000000..8652dfba --- /dev/null +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/ObjectMetadata.java @@ -0,0 +1,5 @@ +package com.iflytek.skillhub.storage; + +import java.time.Instant; + +public record ObjectMetadata(long size, String contentType, Instant lastModified) {} diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/ObjectStorageService.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/ObjectStorageService.java new file mode 100644 index 00000000..d5bb0b24 --- /dev/null +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/ObjectStorageService.java @@ -0,0 +1,13 @@ +package com.iflytek.skillhub.storage; + +import java.io.InputStream; +import java.util.List; + +public interface ObjectStorageService { + void putObject(String key, InputStream data, long size, String contentType); + InputStream getObject(String key); + void deleteObject(String key); + void deleteObjects(List keys); + boolean exists(String key); + ObjectMetadata getMetadata(String key); +} diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageProperties.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageProperties.java new file mode 100644 index 00000000..05393e7d --- /dev/null +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageProperties.java @@ -0,0 +1,25 @@ +package com.iflytek.skillhub.storage; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationProperties(prefix = "skillhub.storage.s3") +public class S3StorageProperties { + private String endpoint; + private String bucket = "skillhub"; + private String accessKey; + private String secretKey; + private String region = "us-east-1"; + + public String getEndpoint() { return endpoint; } + public void setEndpoint(String endpoint) { this.endpoint = endpoint; } + public String getBucket() { return bucket; } + public void setBucket(String bucket) { this.bucket = bucket; } + public String getAccessKey() { return accessKey; } + public void setAccessKey(String accessKey) { this.accessKey = accessKey; } + public String getSecretKey() { return secretKey; } + public void setSecretKey(String secretKey) { this.secretKey = secretKey; } + public String getRegion() { return region; } + public void setRegion(String region) { this.region = region; } +} diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java new file mode 100644 index 00000000..433ef50e --- /dev/null +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java @@ -0,0 +1,77 @@ +package com.iflytek.skillhub.storage; + +import jakarta.annotation.PostConstruct; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; + +import java.io.InputStream; +import java.net.URI; +import java.util.List; + +@Service +@ConditionalOnProperty(name = "skillhub.storage.provider", havingValue = "s3") +public class S3StorageService implements ObjectStorageService { + private static final Logger log = LoggerFactory.getLogger(S3StorageService.class); + private final S3StorageProperties properties; + private S3Client s3Client; + + public S3StorageService(S3StorageProperties properties) { this.properties = properties; } + + @PostConstruct + void init() { + var builder = S3Client.builder() + .region(Region.of(properties.getRegion())) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create(properties.getAccessKey(), properties.getSecretKey()))) + .forcePathStyle(true); + if (properties.getEndpoint() != null && !properties.getEndpoint().isBlank()) { + builder.endpointOverride(URI.create(properties.getEndpoint())); + } + this.s3Client = builder.build(); + ensureBucketExists(); + } + + private void ensureBucketExists() { + try { s3Client.headBucket(HeadBucketRequest.builder().bucket(properties.getBucket()).build()); } + catch (NoSuchBucketException e) { + log.info("Bucket '{}' does not exist, creating...", properties.getBucket()); + s3Client.createBucket(CreateBucketRequest.builder().bucket(properties.getBucket()).build()); + } + } + + @Override public void putObject(String key, InputStream data, long size, String contentType) { + s3Client.putObject(PutObjectRequest.builder().bucket(properties.getBucket()).key(key).contentType(contentType).contentLength(size).build(), RequestBody.fromInputStream(data, size)); + } + + @Override public InputStream getObject(String key) { + return s3Client.getObject(GetObjectRequest.builder().bucket(properties.getBucket()).key(key).build()); + } + + @Override public void deleteObject(String key) { + s3Client.deleteObject(DeleteObjectRequest.builder().bucket(properties.getBucket()).key(key).build()); + } + + @Override public void deleteObjects(List keys) { + if (keys.isEmpty()) return; + List ids = keys.stream().map(k -> ObjectIdentifier.builder().key(k).build()).toList(); + s3Client.deleteObjects(DeleteObjectsRequest.builder().bucket(properties.getBucket()).delete(Delete.builder().objects(ids).build()).build()); + } + + @Override public boolean exists(String key) { + try { s3Client.headObject(HeadObjectRequest.builder().bucket(properties.getBucket()).key(key).build()); return true; } + catch (NoSuchKeyException e) { return false; } + } + + @Override public ObjectMetadata getMetadata(String key) { + HeadObjectResponse resp = s3Client.headObject(HeadObjectRequest.builder().bucket(properties.getBucket()).key(key).build()); + return new ObjectMetadata(resp.contentLength(), resp.contentType(), resp.lastModified()); + } +} diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/StorageProperties.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/StorageProperties.java new file mode 100644 index 00000000..0240ddd8 --- /dev/null +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/StorageProperties.java @@ -0,0 +1,22 @@ +package com.iflytek.skillhub.storage; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationProperties(prefix = "skillhub.storage") +public class StorageProperties { + private String provider = "local"; + private LocalProperties local = new LocalProperties(); + + public String getProvider() { return provider; } + public void setProvider(String provider) { this.provider = provider; } + public LocalProperties getLocal() { return local; } + public void setLocal(LocalProperties local) { this.local = local; } + + public static class LocalProperties { + private String basePath = "./data/storage"; + public String getBasePath() { return basePath; } + public void setBasePath(String basePath) { this.basePath = basePath; } + } +} diff --git a/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/LocalFileStorageServiceTest.java b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/LocalFileStorageServiceTest.java new file mode 100644 index 00000000..dfaf2b72 --- /dev/null +++ b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/LocalFileStorageServiceTest.java @@ -0,0 +1,64 @@ +package com.iflytek.skillhub.storage; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class LocalFileStorageServiceTest { + @TempDir Path tempDir; + private LocalFileStorageService storageService; + + @BeforeEach + void setUp() { + StorageProperties props = new StorageProperties(); + props.getLocal().setBasePath(tempDir.toString()); + storageService = new LocalFileStorageService(props); + } + + @Test void shouldPutAndGetObject() throws Exception { + String key = "skills/1/1/SKILL.md"; + byte[] content = "# Hello".getBytes(StandardCharsets.UTF_8); + storageService.putObject(key, new ByteArrayInputStream(content), content.length, "text/markdown"); + try (InputStream result = storageService.getObject(key)) { assertArrayEquals(content, result.readAllBytes()); } + } + + @Test void shouldCheckExistence() { + assertFalse(storageService.exists("test/exists.txt")); + byte[] content = "data".getBytes(StandardCharsets.UTF_8); + storageService.putObject("test/exists.txt", new ByteArrayInputStream(content), content.length, "text/plain"); + assertTrue(storageService.exists("test/exists.txt")); + } + + @Test void shouldDeleteObject() { + byte[] content = "data".getBytes(StandardCharsets.UTF_8); + storageService.putObject("test/delete.txt", new ByteArrayInputStream(content), content.length, "text/plain"); + assertTrue(storageService.exists("test/delete.txt")); + storageService.deleteObject("test/delete.txt"); + assertFalse(storageService.exists("test/delete.txt")); + } + + @Test void shouldDeleteMultipleObjects() { + byte[] content = "data".getBytes(StandardCharsets.UTF_8); + storageService.putObject("a/1.txt", new ByteArrayInputStream(content), content.length, "text/plain"); + storageService.putObject("a/2.txt", new ByteArrayInputStream(content), content.length, "text/plain"); + storageService.deleteObjects(List.of("a/1.txt", "a/2.txt")); + assertFalse(storageService.exists("a/1.txt")); + assertFalse(storageService.exists("a/2.txt")); + } + + @Test void shouldGetMetadata() { + byte[] content = "hello world".getBytes(StandardCharsets.UTF_8); + storageService.putObject("test/meta.txt", new ByteArrayInputStream(content), content.length, "text/plain"); + ObjectMetadata metadata = storageService.getMetadata("test/meta.txt"); + assertEquals(content.length, metadata.size()); + assertNotNull(metadata.lastModified()); + } +}