Merge pull request #3130 from kmlebedev/fix_rm_parent_dir_via_nextcloud

2025-09-19 13:07:53 +08:00 · 2022-06-07 08:56:55 -07:00
parent 666e620b63 c07820178f
commit f43ec9f363
6 changed files with 71 additions and 34 deletions
--- a/weed/s3api/s3api_object_handlers.go
+++ b/weed/s3api/s3api_object_handlers.go
@@ -92,16 +92,20 @@ func (s3a *S3ApiServer) PutObjectHandler(w http.ResponseWriter, r *http.Request)
 	}
 	defer dataReader.Close()

+	objectContentType := r.Header.Get("Content-Type")
 	if strings.HasSuffix(object, "/") {
 		if err := s3a.mkdir(s3a.option.BucketsPath, bucket+strings.TrimSuffix(object, "/"), func(entry *filer_pb.Entry) {
-			entry.Attributes.Mime = r.Header.Get("Content-Type")
+			if objectContentType == "" {
+				objectContentType = "httpd/unix-directory"
+			}
+			entry.Attributes.Mime = objectContentType
 		}); err != nil {
 			s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
 			return
 		}
 	} else {
 		uploadUrl := s3a.toFilerUrl(bucket, object)
-		if r.Header.Get("Content-Type") == "" {
+		if objectContentType == "" {
 			dataReader = mimeDetect(r, dataReader)
 		}

--- a/weed/s3api/s3api_objects_list_handlers.go
+++ b/weed/s3api/s3api_objects_list_handlers.go
@@ -133,6 +133,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 		reqDir = reqDir[1:]
 	}
 	bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket)
+	bucketPrefixLen := len(bucketPrefix)
 	reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir)
 	if strings.HasSuffix(reqDir, "/") {
 		reqDir = strings.TrimSuffix(reqDir, "/")
@@ -147,21 +148,23 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 	// check filer
 	err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

-		_, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) {
+		_, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, false, false, bucketPrefixLen, func(dir string, entry *filer_pb.Entry) {
 			if entry.IsDirectory {
 				if delimiter == "/" {
 					commonPrefixes = append(commonPrefixes, PrefixEntry{
-						Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
+						Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[bucketPrefixLen:],
 					})
 				}
-				return
+				if !(entry.IsDirectoryKeyObject() && strings.HasSuffix(entry.Name, "/")) {
+					return
+				}
 			}
 			storageClass := "STANDARD"
 			if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
 				storageClass = string(v)
 			}
 			contents = append(contents, ListEntry{
-				Key:          fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
+				Key:          fmt.Sprintf("%s/%s", dir, entry.Name)[bucketPrefixLen:],
 				LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
 				ETag:         "\"" + filer.ETag(entry) + "\"",
 				Size:         int64(filer.FileSize(entry)),
@@ -172,6 +175,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 				StorageClass: StorageClass(storageClass),
 			})
 		})
+		glog.V(4).Infof("end doListFilerEntries isTruncated:%v nextMarker:%v reqDir: %v prefix: %v", isTruncated, nextMarker, reqDir, prefix)
 		if doErr != nil {
 			return doErr
 		}
@@ -180,21 +184,21 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 			nextMarker = ""
 		}

-		if len(contents) == 0 && maxKeys > 0 {
+		if len(contents) == 0 && len(commonPrefixes) == 0 && maxKeys > 0 {
 			if strings.HasSuffix(originalPrefix, "/") && prefix == "" {
 				reqDir, prefix = filepath.Split(strings.TrimSuffix(reqDir, "/"))
 				reqDir = strings.TrimSuffix(reqDir, "/")
 			}
-			_, _, _, doErr = s3a.doListFilerEntries(client, reqDir, prefix, 1, prefix, delimiter, true, func(dir string, entry *filer_pb.Entry) {
-				if entry.IsDirectory && entry.Attributes.Mime != "" && entry.Name == prefix {
+			_, _, _, doErr = s3a.doListFilerEntries(client, reqDir, prefix, 1, prefix, delimiter, true, false, bucketPrefixLen, func(dir string, entry *filer_pb.Entry) {
+				if entry.IsDirectoryKeyObject() && entry.Name == prefix {
 					storageClass := "STANDARD"
 					if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
 						storageClass = string(v)
 					}
 					contents = append(contents, ListEntry{
-						Key:          fmt.Sprintf("%s/%s", dir, entry.Name+"/")[len(bucketPrefix):],
+						Key:          fmt.Sprintf("%s/%s/", dir, entry.Name)[bucketPrefixLen:],
 						LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
-						ETag:         "\"" + filer.ETag(entry) + "\"",
+						ETag:         "\"" + fmt.Sprintf("%x", entry.Attributes.Md5) + "\"",
 						Size:         int64(filer.FileSize(entry)),
 						Owner: CanonicalUser{
 							ID:          fmt.Sprintf("%x", entry.Attributes.Uid),
@@ -204,6 +208,13 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 					})
 				}
 			})
+			if doErr != nil {
+				return doErr
+			}
+		}
+
+		if len(nextMarker) > 0 {
+			nextMarker = nextMarker[bucketPrefixLen:]
 		}

 		response = ListBucketResult{
@@ -224,7 +235,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
 	return
 }

-func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, inclusiveStartFrom bool, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) {
+func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, inclusiveStartFrom bool, subEntries bool, bucketPrefixLen int, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) {
 	// invariants
 	//   prefix and marker should be under dir, marker may contain "/"
 	//   maxKeys should be updated for each recursion
@@ -237,20 +248,27 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 	}

 	if strings.Contains(marker, "/") {
-		sepIndex := strings.Index(marker, "/")
-		subDir, subMarker := marker[0:sepIndex], marker[sepIndex+1:]
-		glog.V(4).Infoln("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys)
-		subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, false, eachEntryFn)
-		if subErr != nil {
-			err = subErr
-			return
+		if strings.HasSuffix(marker, "/") {
+			marker = strings.TrimSuffix(marker, "/")
+		}
+		sepIndex := strings.Index(marker, "/")
+		if sepIndex != -1 {
+			subPrefix, subMarker := marker[0:sepIndex], marker[sepIndex+1:]
+			subDir := fmt.Sprintf("%s/%s", dir[0:bucketPrefixLen-1], subPrefix)
+			if strings.HasPrefix(subDir, dir) {
+				subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, subDir, "", maxKeys, subMarker, delimiter, false, false, bucketPrefixLen, eachEntryFn)
+				if subErr != nil {
+					err = subErr
+					return
+				}
+				counter += subCounter
+				isTruncated = isTruncated || subIsTruncated
+				maxKeys -= subCounter
+				nextMarker = subNextMarker
+				// finished processing this sub directory
+				marker = subPrefix
+			}
 		}
-		counter += subCounter
-		isTruncated = isTruncated || subIsTruncated
-		maxKeys -= subCounter
-		nextMarker = subDir + "/" + subNextMarker
-		// finished processing this sub directory
-		marker = subDir
 	}
 	if maxKeys <= 0 {
 		return
@@ -288,40 +306,46 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
 			return
 		}
 		entry := resp.Entry
-		nextMarker = entry.Name
+		nextMarker = dir + "/" + entry.Name
 		if entry.IsDirectory {
 			// println("ListEntries", dir, "dir:", entry.Name)
 			if entry.Name == ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys
 				continue
 			}
-			if delimiter != "/" {
+			if delimiter == "" {
 				eachEntryFn(dir, entry)
 				// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter)
-				subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, false, eachEntryFn)
+				subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, false, true, bucketPrefixLen, eachEntryFn)
 				if subErr != nil {
 					err = fmt.Errorf("doListFilerEntries2: %v", subErr)
 					return
 				}
 				// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter, "subCounter", subCounter, "subNextMarker", subNextMarker, "subIsTruncated", subIsTruncated)
+				if subCounter == 0 && entry.IsDirectoryKeyObject() {
+					entry.Name += "/"
+					eachEntryFn(dir, entry)
+					counter++
+				}
 				counter += subCounter
-				nextMarker = entry.Name + "/" + subNextMarker
+				nextMarker = subNextMarker
 				if subIsTruncated {
 					isTruncated = true
 					return
 				}
-			} else {
+			} else if delimiter == "/" {
 				var isEmpty bool
-				if !s3a.option.AllowEmptyFolder {
+				if !s3a.option.AllowEmptyFolder && !entry.IsDirectoryKeyObject() {
 					if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil {
 						glog.Errorf("check empty folder %s: %v", dir, err)
 					}
 				}
 				if !isEmpty {
+					nextMarker += "/"
 					eachEntryFn(dir, entry)
 					counter++
 				}
 			}
-		} else {
+		} else if !(delimiter == "/" && subEntries) {
 			// println("ListEntries", dir, "file:", entry.Name)
 			eachEntryFn(dir, entry)
 			counter++