[history server] move storage interface

historyserver/pkg/collector/eventserver/eventserver.go

@@ -18,7 +18,7 @@ import (
 	"github.com/fsnotify/fsnotify"
 	"github.com/sirupsen/logrus"
 
-	"github.com/ray-project/kuberay/historyserver/pkg/collector/logcollector/storage"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage"
 )
 
 type Event struct {

historyserver/pkg/collector/logcollector/runtime/logcollector/collector.go

@@ -17,7 +17,7 @@ import (
 	"github.com/fsnotify/fsnotify"
 	"github.com/sirupsen/logrus"
 
-	"github.com/ray-project/kuberay/historyserver/pkg/collector/logcollector/storage"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage"
 	"github.com/ray-project/kuberay/historyserver/pkg/utils"
 )
 

historyserver/pkg/collector/logcollector/runtime/runtime.go

@@ -8,8 +8,8 @@ import (
 	"time"
 
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/logcollector/runtime/logcollector"
-	"github.com/ray-project/kuberay/historyserver/pkg/collector/logcollector/storage"
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/types"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage"
 	"github.com/ray-project/kuberay/historyserver/pkg/utils"
 )
 

historyserver/pkg/collector/registry.go

@@ -1,11 +1,11 @@
 package collector
 
 import (
-	"github.com/ray-project/kuberay/historyserver/pkg/collector/logcollector/storage"
-	"github.com/ray-project/kuberay/historyserver/pkg/collector/logcollector/storage/aliyunoss/ray"
-	"github.com/ray-project/kuberay/historyserver/pkg/collector/logcollector/storage/localtest"
-	"github.com/ray-project/kuberay/historyserver/pkg/collector/logcollector/storage/s3"
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/types"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage/aliyunoss/ray"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage/localtest"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage/s3"
 )
 
 type WriterRegistry map[string]func(globalData *types.RayCollectorConfig, data map[string]interface{}) (storage.StorageWriter, error)

historyserver/pkg/collector/logcollector/storage/aliyunoss/ray/ray.go → historyserver/pkg/storage/aliyunoss/ray/ray.go

@@ -30,9 +30,9 @@ import (
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 	"github.com/sirupsen/logrus"
 
-	"github.com/ray-project/kuberay/historyserver/pkg/collector/logcollector/storage"
-	"github.com/ray-project/kuberay/historyserver/pkg/collector/logcollector/storage/aliyunoss/rrsa"
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/types"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage/aliyunoss/rrsa"
 	"github.com/ray-project/kuberay/historyserver/pkg/utils"
 )
 

historyserver/pkg/collector/logcollector/storage/localtest/reader.go → historyserver/pkg/storage/localtest/reader.go

@@ -4,8 +4,8 @@ import (
 	"io"
 	"strings"
 
-	"github.com/ray-project/kuberay/historyserver/pkg/collector/logcollector/storage"
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/types"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage"
 	"github.com/ray-project/kuberay/historyserver/pkg/utils"
 )
 

historyserver/pkg/collector/logcollector/storage/localtest/writer.go → historyserver/pkg/storage/localtest/writer.go

@@ -3,8 +3,8 @@ package localtest
 import (
 	"io"
 
-	"github.com/ray-project/kuberay/historyserver/pkg/collector/logcollector/storage"
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/types"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage"
 )
 
 // MockWriter is a mock implementation of the StorageWriter interface

historyserver/pkg/collector/logcollector/storage/s3/s3.go → historyserver/pkg/storage/s3/s3.go

@@ -33,8 +33,8 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/sirupsen/logrus"
 
-	"github.com/ray-project/kuberay/historyserver/pkg/collector/logcollector/storage"
 	"github.com/ray-project/kuberay/historyserver/pkg/collector/types"
+	"github.com/ray-project/kuberay/historyserver/pkg/storage"
 	"github.com/ray-project/kuberay/historyserver/pkg/utils"
 )
 

historyserver/pkg/storage/storage.go

@@ -1 +1,26 @@
 package storage
+
+import (
+	"io"
+
+	"github.com/ray-project/kuberay/historyserver/pkg/utils"
+)
+
+// StorageWriter is the interface for storage writer.
+type StorageWriter interface {
+	CreateDirectory(path string) error
+	WriteFile(file string, reader io.ReadSeeker) error
+}
+
+// HistoryServer create readers for each storage runtime
+// Depends on the implementation of front end.
+type StorageReader interface {
+	// need to return all available files from backend
+	List() []utils.ClusterInfo
+	// for log, pass filename to this func
+	// for metadata, fileName is generated by historyserver, which will obey the same rule as collector to make sure the historyserver can ready the right file.
+	//
+	GetContent(clusterId string, fileName string) io.Reader
+
+	ListFiles(clusterId string, dir string) []string
+}