11package main
22
33import (
4+ "errors"
45 "fmt"
56 "io"
6- "log"
77 "net/http"
88 "os"
9+ "path/filepath"
910
1011 "github.com/aws/aws-sdk-go/aws"
1112 "github.com/aws/aws-sdk-go/aws/session"
1213 "github.com/aws/aws-sdk-go/service/s3"
14+ log "github.com/sirupsen/logrus"
1315)
1416
1517var (
1618 s3Service * s3.S3
1719 bucketName string
20+ cachePath string
1821)
1922
23+ // getFile checks if we have a local copy otherwise downloads from S3
24+ func getFile (key string ) (FileWrapper , error ) {
25+ if cachePath != "" {
26+ log .Debug ("Trying to get file from cache" )
27+ obj , err := getFileFromCache (key )
28+
29+ // Directly return file from Cache if we didn't got an error
30+ if err == nil {
31+ log .Info ("Returning cached file" )
32+ return obj , nil
33+ } else {
34+ log .Debug (err )
35+ }
36+ }
37+
38+ obj , err := getFileFromBucket (key )
39+ if err != nil {
40+ return FileWrapper {}, err
41+ }
42+
43+ log .Debug ("Returning file from Bucket" )
44+ return obj , nil
45+
46+ }
47+
48+ func getFileFromCache (key string ) (FileWrapper , error ) {
49+ filePath := filepath .Join (cachePath , key )
50+
51+ if fileStat , err := os .Stat (filePath ); err == nil {
52+ // file in cache. check expire
53+ headRequest , err := s3Service .HeadObject (& s3.HeadObjectInput {
54+ Bucket : aws .String (bucketName ),
55+ Key : aws .String (key ),
56+ })
57+
58+ if err != nil {
59+ // We have a local file, but HeadObject returned an error, so we can
60+ // assume that the file no longer exists in the bucket
61+ os .Remove (filePath )
62+ log .Debug ("Deleting local file" )
63+ return FileWrapper {}, err
64+ }
65+
66+ if fileStat .ModTime ().Before (* headRequest .LastModified ) {
67+ // Our file is older than the one in the bucket
68+ os .Remove (filePath )
69+ return FileWrapper {}, errors .New ("file not up to date" )
70+ }
71+
72+ fh , err := os .Open (filePath )
73+ if err != nil {
74+ // Couldn't open cached file
75+ return FileWrapper {}, err
76+ }
77+
78+ return FileWrapper {
79+ File : fh ,
80+ HeadOutput : headRequest ,
81+ GetOutput : nil ,
82+ }, nil
83+
84+ } else {
85+ // File not in cache or otherwise not accessible
86+ return FileWrapper {}, err
87+ }
88+ }
89+
90+ func getFileFromBucket (key string ) (FileWrapper , error ) {
91+ log .Info ("Getting file from Bucket" )
92+
93+ obj , err := s3Service .GetObject (& s3.GetObjectInput {
94+ Bucket : aws .String (bucketName ),
95+ Key : aws .String (key ),
96+ })
97+
98+ if err != nil {
99+ log .Errorf ("Error while getting %q from S3: %s\n " , key , err .Error ())
100+ return FileWrapper {}, err
101+ }
102+
103+ s3File := FileWrapper {
104+ File : nil ,
105+ HeadOutput : nil ,
106+ GetOutput : obj ,
107+ }
108+
109+ if cachePath != "" {
110+ path , err := saveFileToCache (key , obj )
111+ if err != nil {
112+ // We couldn't save the file to the cache but still return the Get response from S3
113+ log .Error (err )
114+ return s3File , nil
115+ }
116+
117+ fh , _ := os .Open (path )
118+ return FileWrapper {
119+ File : fh ,
120+ HeadOutput : nil ,
121+ GetOutput : obj ,
122+ }, nil
123+
124+ }
125+
126+ return s3File , nil
127+ }
128+
129+ // createWithFolders creates the full nested directory structure and then creates the requested file
130+ func createWithFolders (p string ) (* os.File , error ) {
131+ if err := os .MkdirAll (filepath .Dir (p ), 0770 ); err != nil {
132+ return nil , err
133+ }
134+ return os .Create (p )
135+ }
136+
137+ func saveFileToCache (key string , obj * s3.GetObjectOutput ) (string , error ) {
138+ log .Debug ("Saving file to cache" )
139+ filePath := filepath .Join (cachePath , key )
140+
141+ outFile , err := createWithFolders (filePath )
142+ if err != nil {
143+ log .Error ("Couldn't create cache dir" )
144+ return "" , err
145+ }
146+ defer outFile .Close ()
147+
148+ io .Copy (outFile , obj .Body )
149+
150+ return filePath , nil
151+
152+ }
153+
20154func handler (w http.ResponseWriter , r * http.Request ) {
21155 defer r .Body .Close ()
22156
@@ -27,24 +161,30 @@ func handler(w http.ResponseWriter, r *http.Request) {
27161 return
28162 }
29163
30- input := & s3. GetObjectInput {
31- Bucket : aws . String ( bucketName ) ,
32- Key : aws . String ( key ),
33- }
34- obj , err := s3Service . GetObject ( input )
164+ log . WithFields (log. Fields {
165+ "key" : key ,
166+ }). Info ( "Got a request" )
167+
168+ obj , err := getFile ( key )
35169 if err != nil {
36- log .Printf ("Error while getting %q: %s\n " , key , err .Error ())
37170 w .WriteHeader (http .StatusForbidden )
38171 w .Write ([]byte ("Forbidden" ))
39172 return
40173 }
41174
42- defer obj .Body .Close ()
175+ // Set correct ContentType
176+ w .Header ().Set ("Content-Type" , obj .GetContentType ())
43177
44- w .Header ().Set ("Content-Type" , * obj .ContentType )
178+ // Check for additional metadata
179+ metadata := obj .GetMetadata ()
180+ if len (metadata ) > 0 {
181+ for k , v := range metadata {
182+ w .Header ().Set (k , * v )
183+ }
184+ }
45185
46186 // Directly copy all bytes from the S3 object into the HTTP reponse
47- io .Copy (w , obj .Body )
187+ io .Copy (w , obj .GetContent () )
48188}
49189
50190func envOrDefault (name string , defaultValue string ) string {
@@ -59,18 +199,39 @@ func main() {
59199 region := envOrDefault ("S3PROXY_REGION" , "eu-central-1" )
60200 port := envOrDefault ("S3PROXY_PORT" , "3000" )
61201 bucketName = envOrDefault ("S3PROXY_BUCKET" , "" )
202+ cachePath = envOrDefault ("S3PROXY_CACHE" , "" )
203+ logLevel := envOrDefault ("S3PROXY_LOGGING" , "WARN" )
204+
205+ l , err := log .ParseLevel (logLevel )
206+ if err != nil {
207+ log .Error ("Unknown loglevel provided. Defaulting to WARN" )
208+ log .SetLevel (log .WarnLevel )
209+ } else {
210+ log .SetLevel (l )
211+ }
62212
63213 if bucketName == "" {
64214 log .Fatal ("You need to provide S3PROXY_BUCKET" )
65215 }
66216
217+ if cachePath != "" {
218+ // Check if we have write access to the cache directory
219+ testPath := filepath .Join (cachePath , ".testfile" )
220+ file , err := createWithFolders (testPath )
221+ if err != nil {
222+ log .Fatal ("No write access to the cache dir" )
223+ }
224+ defer file .Close ()
225+
226+ }
227+
67228 sess := session .Must (session .NewSession (& aws.Config {
68229 Region : aws .String (region ),
69230 }))
70231 s3Service = s3 .New (sess )
71232
72233 http .HandleFunc ("/" , handler )
73234
74- log .Printf ("Listening on :%s \n " , port )
235+ log .Info ("Listening on :%s \n " , port )
75236 log .Fatal (http .ListenAndServe (fmt .Sprintf (":%s" , port ), nil ))
76237}
0 commit comments