scriptoriumadmin/skillhub
1
0
Fork 0
mirror of https://github.com/iflytek/skillhub.git synced 2026-09-12 23:01:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

feat(storage): add object storage SPI with LocalFile and S3 implementations

Browse source
This commit is contained in:
vsxd 2026-03-12 01:56:27 +08:00
parent 351829e1a9
commit ca36304df9
8 changed files with 289 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
server/skillhub-storage/pom.xml
View file

@ -10,4 +10,25 @@
<version>0.1.0-SNAPSHOT</version>
</parent>
<artifactId>skillhub-storage</artifactId>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>s3</artifactId>
<version>2.20.26</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

62
server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/LocalFileStorageService.java Normal file
View file

@ -0,0 +1,62 @@
package com.iflytek.skillhub.storage;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.io.*;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.time.Instant;
import java.util.List;
@Service
@ConditionalOnProperty(name = "skillhub.storage.provider", havingValue = "local", matchIfMissing = true)
public class LocalFileStorageService implements ObjectStorageService {
private final Path basePath;
public LocalFileStorageService(StorageProperties properties) {
this.basePath = Paths.get(properties.getLocal().getBasePath());
}
@Override
public void putObject(String key, InputStream data, long size, String contentType) {
try {
Path target = resolve(key);
Files.createDirectories(target.getParent());
Path tmp = target.resolveSibling(target.getFileName() + ".tmp");
try (OutputStream out = Files.newOutputStream(tmp, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
data.transferTo(out);
}
Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
} catch (IOException e) { throw new UncheckedIOException("Failed to put object: " + key, e); }
}
@Override
public InputStream getObject(String key) {
try { return Files.newInputStream(resolve(key)); }
catch (IOException e) { throw new UncheckedIOException("Failed to get object: " + key, e); }
}
@Override
public void deleteObject(String key) {
try { Files.deleteIfExists(resolve(key)); }
catch (IOException e) { throw new UncheckedIOException("Failed to delete object: " + key, e); }
}
@Override
public void deleteObjects(List<String> keys) { keys.forEach(this::deleteObject); }
@Override
public boolean exists(String key) { return Files.exists(resolve(key)); }
@Override
public ObjectMetadata getMetadata(String key) {
try {
Path path = resolve(key);
BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class);
return new ObjectMetadata(attrs.size(), Files.probeContentType(path), attrs.lastModifiedTime().toInstant());
} catch (IOException e) { throw new UncheckedIOException("Failed to get metadata: " + key, e); }
}
private Path resolve(String key) { return basePath.resolve(key); }
}

5
server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/ObjectMetadata.java Normal file
View file

@ -0,0 +1,5 @@
package com.iflytek.skillhub.storage;
import java.time.Instant;
public record ObjectMetadata(long size, String contentType, Instant lastModified) {}

13
server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/ObjectStorageService.java Normal file
View file

@ -0,0 +1,13 @@
package com.iflytek.skillhub.storage;
import java.io.InputStream;
import java.util.List;
public interface ObjectStorageService {
void putObject(String key, InputStream data, long size, String contentType);
InputStream getObject(String key);
void deleteObject(String key);
void deleteObjects(List<String> keys);
boolean exists(String key);
ObjectMetadata getMetadata(String key);
}

25
server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageProperties.java Normal file
View file

@ -0,0 +1,25 @@
package com.iflytek.skillhub.storage;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@ConfigurationProperties(prefix = "skillhub.storage.s3")
public class S3StorageProperties {
private String endpoint;
private String bucket = "skillhub";
private String accessKey;
private String secretKey;
private String region = "us-east-1";
public String getEndpoint() { return endpoint; }
public void setEndpoint(String endpoint) { this.endpoint = endpoint; }
public String getBucket() { return bucket; }
public void setBucket(String bucket) { this.bucket = bucket; }
public String getAccessKey() { return accessKey; }
public void setAccessKey(String accessKey) { this.accessKey = accessKey; }
public String getSecretKey() { return secretKey; }
public void setSecretKey(String secretKey) { this.secretKey = secretKey; }
public String getRegion() { return region; }
public void setRegion(String region) { this.region = region; }
}

77
server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java Normal file
View file

@ -0,0 +1,77 @@
package com.iflytek.skillhub.storage;
import jakarta.annotation.PostConstruct;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.*;
import java.io.InputStream;
import java.net.URI;
import java.util.List;
@Service
@ConditionalOnProperty(name = "skillhub.storage.provider", havingValue = "s3")
public class S3StorageService implements ObjectStorageService {
private static final Logger log = LoggerFactory.getLogger(S3StorageService.class);
private final S3StorageProperties properties;
private S3Client s3Client;
public S3StorageService(S3StorageProperties properties) { this.properties = properties; }
@PostConstruct
void init() {
var builder = S3Client.builder()
.region(Region.of(properties.getRegion()))
.credentialsProvider(StaticCredentialsProvider.create(
AwsBasicCredentials.create(properties.getAccessKey(), properties.getSecretKey())))
.forcePathStyle(true);
if (properties.getEndpoint() != null && !properties.getEndpoint().isBlank()) {
builder.endpointOverride(URI.create(properties.getEndpoint()));
}
this.s3Client = builder.build();
ensureBucketExists();
}
private void ensureBucketExists() {
try { s3Client.headBucket(HeadBucketRequest.builder().bucket(properties.getBucket()).build()); }
catch (NoSuchBucketException e) {
log.info("Bucket '{}' does not exist, creating...", properties.getBucket());
s3Client.createBucket(CreateBucketRequest.builder().bucket(properties.getBucket()).build());
}
}
@Override public void putObject(String key, InputStream data, long size, String contentType) {
s3Client.putObject(PutObjectRequest.builder().bucket(properties.getBucket()).key(key).contentType(contentType).contentLength(size).build(), RequestBody.fromInputStream(data, size));
}
@Override public InputStream getObject(String key) {
return s3Client.getObject(GetObjectRequest.builder().bucket(properties.getBucket()).key(key).build());
}
@Override public void deleteObject(String key) {
s3Client.deleteObject(DeleteObjectRequest.builder().bucket(properties.getBucket()).key(key).build());
}
@Override public void deleteObjects(List<String> keys) {
if (keys.isEmpty()) return;
List<ObjectIdentifier> ids = keys.stream().map(k -> ObjectIdentifier.builder().key(k).build()).toList();
s3Client.deleteObjects(DeleteObjectsRequest.builder().bucket(properties.getBucket()).delete(Delete.builder().objects(ids).build()).build());
}
@Override public boolean exists(String key) {
try { s3Client.headObject(HeadObjectRequest.builder().bucket(properties.getBucket()).key(key).build()); return true; }
catch (NoSuchKeyException e) { return false; }
}
@Override public ObjectMetadata getMetadata(String key) {
HeadObjectResponse resp = s3Client.headObject(HeadObjectRequest.builder().bucket(properties.getBucket()).key(key).build());
return new ObjectMetadata(resp.contentLength(), resp.contentType(), resp.lastModified());
}
}

22
server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/StorageProperties.java Normal file
View file

@ -0,0 +1,22 @@
package com.iflytek.skillhub.storage;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
@Component
@ConfigurationProperties(prefix = "skillhub.storage")
public class StorageProperties {
private String provider = "local";
private LocalProperties local = new LocalProperties();
public String getProvider() { return provider; }
public void setProvider(String provider) { this.provider = provider; }
public LocalProperties getLocal() { return local; }
public void setLocal(LocalProperties local) { this.local = local; }
public static class LocalProperties {
private String basePath = "./data/storage";
public String getBasePath() { return basePath; }
public void setBasePath(String basePath) { this.basePath = basePath; }
}
}

64
server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/LocalFileStorageServiceTest.java Normal file
View file

@ -0,0 +1,64 @@
package com.iflytek.skillhub.storage;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.List;
import static org.junit.jupiter.api.Assertions.*;
class LocalFileStorageServiceTest {
@TempDir Path tempDir;
private LocalFileStorageService storageService;
@BeforeEach
void setUp() {
StorageProperties props = new StorageProperties();
props.getLocal().setBasePath(tempDir.toString());
storageService = new LocalFileStorageService(props);
}
@Test void shouldPutAndGetObject() throws Exception {
String key = "skills/1/1/SKILL.md";
byte[] content = "# Hello".getBytes(StandardCharsets.UTF_8);
storageService.putObject(key, new ByteArrayInputStream(content), content.length, "text/markdown");
try (InputStream result = storageService.getObject(key)) { assertArrayEquals(content, result.readAllBytes()); }
}
@Test void shouldCheckExistence() {
assertFalse(storageService.exists("test/exists.txt"));
byte[] content = "data".getBytes(StandardCharsets.UTF_8);
storageService.putObject("test/exists.txt", new ByteArrayInputStream(content), content.length, "text/plain");
assertTrue(storageService.exists("test/exists.txt"));
}
@Test void shouldDeleteObject() {
byte[] content = "data".getBytes(StandardCharsets.UTF_8);
storageService.putObject("test/delete.txt", new ByteArrayInputStream(content), content.length, "text/plain");
assertTrue(storageService.exists("test/delete.txt"));
storageService.deleteObject("test/delete.txt");
assertFalse(storageService.exists("test/delete.txt"));
}
@Test void shouldDeleteMultipleObjects() {
byte[] content = "data".getBytes(StandardCharsets.UTF_8);
storageService.putObject("a/1.txt", new ByteArrayInputStream(content), content.length, "text/plain");
storageService.putObject("a/2.txt", new ByteArrayInputStream(content), content.length, "text/plain");
storageService.deleteObjects(List.of("a/1.txt", "a/2.txt"));
assertFalse(storageService.exists("a/1.txt"));
assertFalse(storageService.exists("a/2.txt"));
}
@Test void shouldGetMetadata() {
byte[] content = "hello world".getBytes(StandardCharsets.UTF_8);
storageService.putObject("test/meta.txt", new ByteArrayInputStream(content), content.length, "text/plain");
ObjectMetadata metadata = storageService.getMetadata("test/meta.txt");
assertEquals(content.length, metadata.size());
assertNotNull(metadata.lastModified());
}
}