mirror of
https://github.com/iflytek/skillhub.git
synced 2026-10-07 02:57:51 +00:00
feat(storage): support IAM authentication for S3 storage
When access-key / secret-key are left blank, fall back to the AWS DefaultCredentialsProvider chain so that deployments on EC2, ECS, and EKS can authenticate via instance profile, task role, or IRSA without static credentials. - Extract buildCredentialsProvider() in S3StorageService - Add sts dependency for Web Identity Token (EKS) support - Add unit tests for credential provider selection - Update storage-spi docs (zh + en) and env example
This commit is contained in:
parent
f707cfee61
commit
003f811292
6 changed files with 123 additions and 7 deletions
|
|
@ -34,6 +34,8 @@ SKILLHUB_STORAGE_PROVIDER=local
|
|||
SKILLHUB_STORAGE_S3_ENDPOINT=https://oss-cn-example.aliyuncs.com
|
||||
SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT=
|
||||
SKILLHUB_STORAGE_S3_BUCKET=skillhub-prod
|
||||
# Static credentials for S3-compatible storage (MinIO, Alibaba OSS, etc.).
|
||||
# Leave both blank to use IAM authentication (EC2 instance profile, ECS task role, EKS IRSA).
|
||||
SKILLHUB_STORAGE_S3_ACCESS_KEY=replace-me
|
||||
SKILLHUB_STORAGE_S3_SECRET_KEY=replace-me
|
||||
SKILLHUB_STORAGE_S3_REGION=cn-shanghai
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ S3 协议兼容实现,支持:
|
|||
|
||||
## 配置
|
||||
|
||||
### 静态凭据(Access Key / Secret Key)
|
||||
|
||||
```bash
|
||||
# 选择存储提供方
|
||||
SKILLHUB_STORAGE_PROVIDER=s3
|
||||
|
|
@ -45,6 +47,27 @@ SKILLHUB_STORAGE_S3_ACCESS_KEY=xxx
|
|||
SKILLHUB_STORAGE_S3_SECRET_KEY=xxx
|
||||
```
|
||||
|
||||
### IAM 认证
|
||||
|
||||
部署在 AWS 上时,可以不配置 Access Key / Secret Key,让 SDK 自动使用 IAM 角色认证([Default Credentials Provider Chain](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials-chain.html)):
|
||||
|
||||
```bash
|
||||
SKILLHUB_STORAGE_PROVIDER=s3
|
||||
SKILLHUB_STORAGE_S3_BUCKET=skillhub
|
||||
SKILLHUB_STORAGE_S3_REGION=us-east-1
|
||||
# 留空或不设置 ACCESS_KEY / SECRET_KEY,SDK 自动使用 IAM 认证
|
||||
SKILLHUB_STORAGE_S3_ACCESS_KEY=
|
||||
SKILLHUB_STORAGE_S3_SECRET_KEY=
|
||||
```
|
||||
|
||||
支持的 IAM 认证方式(按 SDK 优先级):
|
||||
- 环境变量(`AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`)
|
||||
- Java 系统属性
|
||||
- Web Identity Token(EKS IRSA)
|
||||
- AWS 配置文件(`~/.aws/credentials`)
|
||||
- EC2 Instance Profile
|
||||
- ECS Task Role
|
||||
|
||||
## 自定义实现
|
||||
|
||||
实现 `ObjectStorageService` 接口,注册为 Spring Bean 即可。
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ S3 protocol compatible implementation, supports:
|
|||
|
||||
## Configuration
|
||||
|
||||
### Static Credentials (Access Key / Secret Key)
|
||||
|
||||
```bash
|
||||
# Select storage provider
|
||||
SKILLHUB_STORAGE_PROVIDER=s3
|
||||
|
|
@ -45,6 +47,27 @@ SKILLHUB_STORAGE_S3_ACCESS_KEY=xxx
|
|||
SKILLHUB_STORAGE_S3_SECRET_KEY=xxx
|
||||
```
|
||||
|
||||
### IAM Authentication
|
||||
|
||||
When deployed on AWS, you can omit the Access Key / Secret Key and let the SDK use IAM role authentication via the [Default Credentials Provider Chain](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials-chain.html):
|
||||
|
||||
```bash
|
||||
SKILLHUB_STORAGE_PROVIDER=s3
|
||||
SKILLHUB_STORAGE_S3_BUCKET=skillhub
|
||||
SKILLHUB_STORAGE_S3_REGION=us-east-1
|
||||
# Leave ACCESS_KEY / SECRET_KEY blank to use IAM authentication
|
||||
SKILLHUB_STORAGE_S3_ACCESS_KEY=
|
||||
SKILLHUB_STORAGE_S3_SECRET_KEY=
|
||||
```
|
||||
|
||||
Supported IAM authentication methods (in SDK priority order):
|
||||
- Environment variables (`AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`)
|
||||
- Java system properties
|
||||
- Web Identity Token (EKS IRSA)
|
||||
- AWS config file (`~/.aws/credentials`)
|
||||
- EC2 Instance Profile
|
||||
- ECS Task Role
|
||||
|
||||
## Custom Implementation
|
||||
|
||||
Implement `ObjectStorageService` interface and register as Spring Bean.
|
||||
|
|
|
|||
|
|
@ -30,6 +30,11 @@
|
|||
<artifactId>apache-client</artifactId>
|
||||
<version>2.20.26</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>software.amazon.awssdk</groupId>
|
||||
<artifactId>sts</artifactId>
|
||||
<version>2.20.26</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import org.slf4j.LoggerFactory;
|
|||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Service;
|
||||
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.http.apache.ApacheHttpClient;
|
||||
|
|
@ -50,19 +52,32 @@ public class S3StorageService implements ObjectStorageService {
|
|||
.connectionAcquisitionTimeout(properties.getConnectionAcquisitionTimeout());
|
||||
this.s3Client = buildS3Client(httpClientBuilder);
|
||||
this.s3Presigner = buildPresigner();
|
||||
String authMode = isStaticCredentials() ? "static credentials" : "IAM credentials (default provider chain)";
|
||||
if (properties.isAutoCreateBucket()) {
|
||||
log.info("Initialized S3 storage client for bucket '{}' (bucket auto-creation is deferred until first write)",
|
||||
properties.getBucket());
|
||||
log.info("Initialized S3 storage client for bucket '{}' using {} (bucket auto-creation is deferred until first write)",
|
||||
properties.getBucket(), authMode);
|
||||
} else {
|
||||
log.info("Initialized S3 storage client for bucket '{}'", properties.getBucket());
|
||||
log.info("Initialized S3 storage client for bucket '{}' using {}", properties.getBucket(), authMode);
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isStaticCredentials() {
|
||||
return properties.getAccessKey() != null && !properties.getAccessKey().isBlank()
|
||||
&& properties.getSecretKey() != null && !properties.getSecretKey().isBlank();
|
||||
}
|
||||
|
||||
AwsCredentialsProvider buildCredentialsProvider() {
|
||||
if (isStaticCredentials()) {
|
||||
return StaticCredentialsProvider.create(
|
||||
AwsBasicCredentials.create(properties.getAccessKey(), properties.getSecretKey()));
|
||||
}
|
||||
return DefaultCredentialsProvider.create();
|
||||
}
|
||||
|
||||
protected S3Client buildS3Client(ApacheHttpClient.Builder httpClientBuilder) {
|
||||
var builder = S3Client.builder()
|
||||
.region(Region.of(properties.getRegion()))
|
||||
.credentialsProvider(StaticCredentialsProvider.create(
|
||||
AwsBasicCredentials.create(properties.getAccessKey(), properties.getSecretKey())))
|
||||
.credentialsProvider(buildCredentialsProvider())
|
||||
.forcePathStyle(properties.isForcePathStyle())
|
||||
.httpClientBuilder(httpClientBuilder)
|
||||
.overrideConfiguration(config -> config
|
||||
|
|
@ -77,8 +92,7 @@ public class S3StorageService implements ObjectStorageService {
|
|||
S3Presigner buildPresigner() {
|
||||
var presignerBuilder = S3Presigner.builder()
|
||||
.region(Region.of(properties.getRegion()))
|
||||
.credentialsProvider(StaticCredentialsProvider.create(
|
||||
AwsBasicCredentials.create(properties.getAccessKey(), properties.getSecretKey())))
|
||||
.credentialsProvider(buildCredentialsProvider())
|
||||
.serviceConfiguration(S3Configuration.builder()
|
||||
.pathStyleAccessEnabled(properties.isForcePathStyle())
|
||||
.build());
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
package com.iflytek.skillhub.storage;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
|
||||
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
|
||||
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
|
||||
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
|
||||
import software.amazon.awssdk.core.sync.RequestBody;
|
||||
import software.amazon.awssdk.http.apache.ApacheHttpClient;
|
||||
|
|
@ -183,6 +186,52 @@ class S3StorageServiceTest {
|
|||
verify(client, times(2)).putObject(any(PutObjectRequest.class), any(RequestBody.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldUseStaticCredentialsWhenAccessKeyAndSecretKeyProvided() {
|
||||
S3StorageProperties props = createProperties(true);
|
||||
S3StorageService service = new S3StorageService(props);
|
||||
|
||||
AwsCredentialsProvider provider = service.buildCredentialsProvider();
|
||||
|
||||
assertThat(provider).isInstanceOf(StaticCredentialsProvider.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldUseDefaultCredentialsProviderWhenAccessKeyIsBlank() {
|
||||
S3StorageProperties props = createProperties(true);
|
||||
props.setAccessKey("");
|
||||
props.setSecretKey("");
|
||||
S3StorageService service = new S3StorageService(props);
|
||||
|
||||
AwsCredentialsProvider provider = service.buildCredentialsProvider();
|
||||
|
||||
assertThat(provider).isInstanceOf(DefaultCredentialsProvider.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldUseDefaultCredentialsProviderWhenAccessKeyIsNull() {
|
||||
S3StorageProperties props = createProperties(true);
|
||||
props.setAccessKey(null);
|
||||
props.setSecretKey(null);
|
||||
S3StorageService service = new S3StorageService(props);
|
||||
|
||||
AwsCredentialsProvider provider = service.buildCredentialsProvider();
|
||||
|
||||
assertThat(provider).isInstanceOf(DefaultCredentialsProvider.class);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldUseDefaultCredentialsProviderWhenOnlyAccessKeyIsSet() {
|
||||
S3StorageProperties props = createProperties(true);
|
||||
props.setAccessKey("some-key");
|
||||
props.setSecretKey(null);
|
||||
S3StorageService service = new S3StorageService(props);
|
||||
|
||||
AwsCredentialsProvider provider = service.buildCredentialsProvider();
|
||||
|
||||
assertThat(provider).isInstanceOf(DefaultCredentialsProvider.class);
|
||||
}
|
||||
|
||||
private S3StorageProperties properties(boolean autoCreateBucket) {
|
||||
S3StorageProperties properties = createProperties(true);
|
||||
properties.setBucket("skillhub");
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue