From c7bf12be0312c91467ced6a3562da9e7b2041093 Mon Sep 17 00:00:00 2001
From: Peter Sanchez <peter@netlandish.com>
Date: Sat, 1 Mar 2025 07:08:57 -0600
Subject: [PATCH] Adding `ReadSizer` interface for use in the storage module.

This will allow backends to stream data versus copying the entire blob
into memory just to get the size of the data.

Adjusted the s3 storage backend to check for ReadSizer and use it if
present.
---
 storage/s3.go      | 27 ++++++++++++++++++++-------
 storage/storage.go | 25 +++++++++++++++++++++++++
 2 files changed, 45 insertions(+), 7 deletions(-)

diff --git a/storage/s3.go b/storage/s3.go
index 7112588..5cf8e07 100644
--- a/storage/s3.go
+++ b/storage/s3.go
@@ -132,22 +132,35 @@ func (s *S3Service) GetObject(ctx context.Context, path string) (*Object, error)
 
 // PutObject uploads an object to Amazon S3 compat bucket, at prefix
 func (s *S3Service) PutObject(ctx context.Context, path string, content io.Reader) error {
-	var data bytes.Buffer
+	var (
+		data io.Reader
+		size int64
+	)
 	fullpath := pathutil.Join(s.Prefix, path)
 	meta := make(map[string]string)
 	if s.perm != "" {
 		meta["x-amz-acl"] = s.perm
 	}
-	_, err := io.Copy(&data, content)
-	if err != nil {
-		return err
+
+	if sizer, ok := content.(ReadSizer); ok {
+		size = sizer.Size()
+		data = content
+	} else {
+		var bufData bytes.Buffer
+		_, err := io.Copy(&bufData, content)
+		if err != nil {
+			return err
+		}
+		data = &bufData
+		size = int64(bufData.Len())
 	}
-	_, err = s.Client.PutObject(
+
+	_, err := s.Client.PutObject(
 		ctx,
 		s.Bucket,
 		fullpath,
-		&data,
-		int64(data.Len()),
+		data,
+		size,
 		minio.PutObjectOptions{UserMetadata: meta, ContentType: guessMimeType(path)},
 	)
 	return err
diff --git a/storage/storage.go b/storage/storage.go
index 9c3c266..ca38e74 100644
--- a/storage/storage.go
+++ b/storage/storage.go
@@ -21,6 +21,12 @@ type (
 		LastModified time.Time
 	}
 
+	// ObjectWrap is a simple type to implement the ReadSizer interface
+	ObjectWrap struct {
+		file io.Reader
+		size int64
+	}
+
 	// Service is a generic interface for storage backends
 	Service interface {
 		ListObjects(ctx context.Context, prefix string) ([]*Object, error)
@@ -28,6 +34,13 @@ type (
 		PutObject(ctx context.Context, path string, content io.Reader) error
 		DeleteObject(ctx context.Context, path string) error
 	}
+
+	// ReadSizer is just a helper interface to determine content length, mainly for
+	// PutObject method of the Service interface
+	ReadSizer interface {
+		Read(p []byte) (n int, err error)
+		Size() int64
+	}
 )
 
 // HasExtension determines whether or not an object contains a file extension
@@ -69,3 +82,15 @@ func guessMimeType(fname string) string {
 	}
 	return ctype
 }
+
+func (o *ObjectWrap) Read(b []byte) (n int, err error) {
+	return o.file.Read(b)
+}
+
+func (o *ObjectWrap) Size() int64 {
+	return o.size
+}
+
+func NewObjectWrap(content io.Reader, size int64) *ObjectWrap {
+	return &ObjectWrap{content, size}
+}
-- 
2.45.3