Stopped passing non-UTF8 attributes in the metadata map over gRPC, as protobuf does not support this

Author: jessegeens

changelog/unreleased/eos-nonutf8.md

@@ -0,0 +1,13 @@
+Bugfix: stop sending non-UTF8 strings over gRPC
+
+EOS supports having non-UTF8 attributes, which get returned in a Stat. This is problematic for us, as we pass these attributes in the ArbitraryMetadata, which gets sent over gRPC. However, the protobuf language specification states:
+
+>   A string must always contain UTF-8 encoded or 7-bit ASCII text, and cannot be longer than 2^32.
+
+An example of such an attribute is:
+
+user.$KERNEL.PURGE.SEC.FILEHASH="S��ϫ]���z��#1}��uU�v��8�L0R�9j�j��e?�2K�T<sJ�*�l���Dǭ��_[�>η�...��w�w[��Yg"
+
+We fix this by stripping non-UTF8 metadata entries before sending the ResourceInfo over gRPC
+
+https://github.com/cs3org/reva/pull/5119

internal/grpc/services/gateway/storageprovider.go

@@ -1218,7 +1218,7 @@ func (s *svc) stat(ctx context.Context, req *provider.StatRequest) (*provider.St
 		}
 		rsp, err := c.Stat(ctx, req)
 		if err != nil || rsp.Status.Code != rpc.Code_CODE_OK {
-			log.Error().Err(err).Any("Status", rsp.Status).Msg("Failed to stat " + resPath)
+			log.Error().Err(err).Msg("Failed to stat " + resPath)
 			return rsp, err
 		}
 		return rsp, nil

internal/grpc/services/storageprovider/storageprovider.go

@@ -29,6 +29,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"unicode/utf8"
 
 	rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
 	provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
@@ -800,6 +801,7 @@ func (s *service) Stat(ctx context.Context, req *provider.StatRequest) (*provide
 		}, nil
 	}
 	s.fixPermissions(md)
+	s.stripNonUtf8Metadata(ctx, md)
 	res := &provider.StatResponse{
 		Status: status.NewOK(ctx),
 		Info:   md,
@@ -826,6 +828,28 @@ func (s *service) fixPermissions(md *provider.ResourceInfo) {
 	}
 }
 
+// This method removes any entries in the ArbitraryMetadata map that
+// are not valid UTF-8
+// This is necessary because protobuf requires strings to only contain valid UTF-8
+func (s *service) stripNonUtf8Metadata(ctx context.Context, md *provider.ResourceInfo) {
+	log := appctx.GetLogger(ctx)
+	if md.ArbitraryMetadata == nil {
+		return
+	}
+
+	toDelete := []string{}
+	for k, v := range md.ArbitraryMetadata.Metadata {
+		if !utf8.ValidString(v) {
+			toDelete = append(toDelete, k)
+		}
+	}
+
+	for _, k := range toDelete {
+		log.Debug().Str("attribute", k).Msg("Dropping non-UTF8 metadata entry")
+		delete(md.ArbitraryMetadata.Metadata, k)
+	}
+}
+
 func (s *service) statVirtualView(ctx context.Context, ref *provider.Reference) (*provider.StatResponse, error) {
 	// The reference in the request encompasses this provider
 	// So we need to stat root, and update the required path
@@ -945,7 +969,7 @@ func (s *service) ListContainer(ctx context.Context, req *provider.ListContainer
 		case errtypes.PermissionDenied:
 			st = status.NewPermissionDenied(ctx, err, "permission denied")
 		default:
-			log.Error().Str("path", newRef.Path).Err(err).Msg("storageprovider: error listing container")
+			log.Error().Any("ref", newRef).Err(err).Msg("storageprovider: error listing container")
 			st = status.NewInternal(ctx, err, "error listing container: "+req.Ref.String())
 		}
 		return &provider.ListContainerResponse{
@@ -962,6 +986,7 @@ func (s *service) ListContainer(ctx context.Context, req *provider.ListContainer
 			}, nil
 		}
 		s.fixPermissions(md)
+		s.stripNonUtf8Metadata(ctx, md)
 		infos = append(infos, md)
 	}
 	res := &provider.ListContainerResponse{

pkg/storage/utils/eosfs/eosfs.go

@@ -1121,7 +1121,7 @@ func (fs *eosfs) GetMD(ctx context.Context, ref *provider.Reference, mdKeys []st
 	// and lightweight accounts don't have a uid
 	auth, err := fs.getDaemonAuth(ctx)
 	if err != nil {
-		return nil, fmt.Errorf("error getting daemon aut")
+		return nil, fmt.Errorf("error getting daemon auth")
 	}
 
 	if ref.ResourceId != nil {