mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
f9ddb5d5c1
* removes unused filesize function * removes ioutil usage * updates ioutil.ReadAll to io.ReadAll * updates ioutil.TempFile to os.CreateTemp * updates ioutil.TempDir to os.MkdirTemp * updates ioutil.ReadAll to os.ReadAll * update ioutil.WriteFile to os.WriteFile * updates ioutil.Discard to io.Discard * updates ioutil.ReadDir to os.ReadDir where applicable * removes unused code in idk * creates type to use for context value keys * replaces assert.Nil with assert.NoError for error checks
99 lines
2.5 KiB
Go
99 lines
2.5 KiB
Go
package internal
|
|
|
|
import (
|
|
"bytes"
|
|
"net/url"
|
|
"os"
|
|
"strings"
|
|
|
|
"github.com/aws/aws-sdk-go/aws"
|
|
"github.com/aws/aws-sdk-go/aws/awserr"
|
|
"github.com/aws/aws-sdk-go/service/s3"
|
|
"github.com/aws/aws-sdk-go/service/s3/s3iface"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
var FileOrURLNotFound = errors.New("file or url does not exist")
|
|
|
|
// ReadFileOrURL reads a path from the filesystem or an s3 URL.
|
|
// The s3client parameter is required if reading an s3 URL.
|
|
// ReadFileOrURL returns FileOrURLNotFound when the local filesystem
|
|
// path or remote s3 location is not found.
|
|
func ReadFileOrURL(name string, s3client s3iface.S3API) ([]byte, error) {
|
|
var content []byte
|
|
var err error
|
|
if strings.HasPrefix(name, "s3://") {
|
|
if s3client == nil {
|
|
return nil, errors.New("missing s3 client")
|
|
}
|
|
u, err := url.Parse(name)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "parsing S3 URL %v", name)
|
|
}
|
|
bucket := u.Host
|
|
key := u.Path[1:] // strip leading slash
|
|
|
|
result, err := s3client.GetObject(&s3.GetObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(key),
|
|
})
|
|
if err != nil {
|
|
if aerr, ok := err.(awserr.Error); ok {
|
|
switch aerr.Code() {
|
|
case s3.ErrCodeNoSuchBucket:
|
|
return nil, FileOrURLNotFound
|
|
case s3.ErrCodeNoSuchKey:
|
|
return nil, FileOrURLNotFound
|
|
}
|
|
}
|
|
return nil, errors.Wrapf(err, "fetching S3 object %v", name)
|
|
}
|
|
|
|
buf := new(bytes.Buffer)
|
|
_, err = buf.ReadFrom(result.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
content = buf.Bytes()
|
|
} else {
|
|
content, err = os.ReadFile(name)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return nil, FileOrURLNotFound
|
|
}
|
|
return nil, errors.Wrapf(err, "reading file %v", name)
|
|
}
|
|
}
|
|
return content, nil
|
|
}
|
|
|
|
func WriteFileOrURL(name string, contents []byte, s3client s3iface.S3API) error {
|
|
var err error
|
|
if strings.HasPrefix(name, "s3://") {
|
|
if s3client == nil {
|
|
return errors.New("missing s3 client")
|
|
}
|
|
u, err := url.Parse(name)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "parsing S3 URL %v", name)
|
|
}
|
|
bucket := u.Host
|
|
key := u.Path[1:] // strip leading slash
|
|
|
|
_, err = s3client.PutObject(&s3.PutObjectInput{
|
|
Bucket: aws.String(bucket),
|
|
Key: aws.String(key),
|
|
Body: bytes.NewReader(contents),
|
|
ContentLength: aws.Int64(int64(len(contents))),
|
|
})
|
|
if err != nil {
|
|
return errors.Wrapf(err, "putting S3 object %v", name)
|
|
}
|
|
} else {
|
|
err = os.WriteFile(name, contents, 0644)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "reading file %v", name)
|
|
}
|
|
}
|
|
return nil
|
|
}
|