Merge branch 'dev' into add-content-api-v2

Author: gmelodie

api/v1/handlers.go

@@ -3399,7 +3399,7 @@ func (s *apiV1) handleCommitCollection(c echo.Context, u *util.User) error {
 // @Description  This endpoint is used to get contents in a collection. If no colpath query param is passed
 // @Tags         collections
 // @Produce      json
-// @Success      200  {object}  []collectionListResponse
+// @Success      200  {object}  []collections.CollectionListResponse
 // @Failure      400  {object}  util.HttpError
 // @Failure      500  {object}  util.HttpError
 // @Param        coluuid  path      string  true   "coluuid"
@@ -3442,97 +3442,12 @@ func (s *apiV1) handleGetCollectionContents(c echo.Context, u *util.User) error
 	// if queryDir is set, do the content listing
 	queryDir = filepath.Clean(queryDir)
 
-	dirs := make(map[string]bool)
-	var out []collectionListResponse
-	for _, r := range refs {
-		if r.Path == "" || r.Name == "" {
-			continue
-		}
-
-		relp, err := getRelativePath(r, queryDir)
-		if err != nil {
-			return c.JSON(http.StatusInternalServerError, fmt.Errorf("errored while calculating relative contentPath queryDir=%s, contentPath=%s", queryDir, r.Path))
-		}
-
-		// if the relative contentPath requires pathing up, its definitely not in this queryDir
-		if strings.HasPrefix(relp, "..") {
-			continue
-		}
-
-		if relp == "." { // Query directory is the complete path containing the content.
-			// trying to list a CID queryDir, not allowed
-			if r.Type == util.Directory {
-				return c.JSON(http.StatusBadRequest, fmt.Errorf("listing CID directories is not allowed"))
-			}
-
-			out = append(out, collectionListResponse{
-				Name:      r.Name,
-				Size:      r.Size,
-				ContID:    r.ID,
-				Cid:       &util.DbCID{CID: r.Cid.CID},
-				Dir:       queryDir,
-				ColUuid:   coluuid,
-				UpdatedAt: r.UpdatedAt,
-			})
-		} else { // Query directory has a subdirectory, which contains the actual content.
-
-			// if CID is a queryDir, set type as Dir and mark Dir as listed so we don't list it again
-			//if r.Type == util.Directory {
-			//	if !dirs[relp] {
-			//		dirs[relp] = true
-			//		out = append(out, collectionListResponse{
-			//			Name:    relp,
-			//			Type:    Dir,
-			//			Size:    r.Size,
-			//			ContID:  r.ID,
-			//			Cid:     &r.Cid,
-			//			Dir:     queryDir,
-			//			ColUuid: coluuid,
-			//		})
-			//	}
-			//	continue
-			//}
-
-			// if relative contentPath has a /, the file is in a subdirectory
-			// print the directory the file is in if we haven't already
-			if strings.Contains(relp, "/") {
-				parts := strings.Split(relp, "/")
-				subDir := parts[0]
-				if !dirs[subDir] {
-					dirs[subDir] = true
-					out = append(out, collectionListResponse{
-						Name:    subDir,
-						Type:    Dir,
-						Dir:     queryDir,
-						ColUuid: coluuid,
-					})
-					continue
-				}
-			}
-		}
-
-		//var contentType CidType
-		contentType := File
-		if r.Type == util.Directory {
-			contentType = Dir
-		}
-		out = append(out, collectionListResponse{
-			Name:    r.Name,
-			Type:    contentType,
-			Size:    r.Size,
-			ContID:  r.ID,
-			Cid:     &util.DbCID{CID: r.Cid.CID},
-			Dir:     queryDir,
-			ColUuid: coluuid,
-		})
+	out, err := collections.GetDirectoryContents(refs, queryDir, coluuid)
+	if err != nil {
+		return err
 	}
-	return c.JSON(http.StatusOK, out)
-}
 
-func getRelativePath(r util.ContentWithPath, queryDir string) (string, error) {
-	contentPath := r.Path
-	relp, err := filepath.Rel(queryDir, contentPath)
-	return relp, err
+	return c.JSON(http.StatusOK, out)
 }
 
 // handleDeleteCollection godoc
@@ -5116,9 +5031,7 @@ func (s *apiV1) handleShuttleRepinAll(c echo.Context) error {
 type CidType string
 
 const (
-	Dir    CidType = "directory"
-	File   CidType = "file"
-	ColDir string  = "dir"
+	ColDir string = "dir"
 )
 
 type collectionListResponse struct {

cmd/estuary-shuttle/main.go

@@ -1616,7 +1616,7 @@ func (d *Shuttle) doPinning(ctx context.Context, op *operation.PinningOperation,
 	ctx, span := d.Tracer.Start(ctx, "doPinning")
 	defer span.End()
 
-	prs, _ := operation.UnSerializePeers(op.Peers)
+	prs := operation.UnSerializePeers(op.Peers)
 	for _, pi := range prs {
 		if err := d.Node.Host.Connect(ctx, *pi); err != nil {
 			log.Warnf("failed to connect to origin node for pinning operation: %s", err)

collections/collections.go

@@ -3,6 +3,7 @@ package collections
 import (
 	"fmt"
 	"net/http"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -31,6 +32,24 @@ type CollectionRef struct {
 	Path       *string `gorm:"not null"`
 }
 
+type CidType string
+
+type CollectionListResponse struct {
+	Name      string      `json:"name"`
+	Type      CidType     `json:"type"`
+	Size      int64       `json:"size"`
+	ContID    uint        `json:"contId"`
+	Cid       *util.DbCID `json:"cid,omitempty"`
+	Dir       string      `json:"dir"`
+	ColUuid   string      `json:"coluuid"`
+	UpdatedAt time.Time   `json:"updatedAt"`
+}
+
+const (
+	CidTypeDir  CidType = "directory"
+	CidTypeFile CidType = "file"
+)
+
 func GetCollection(coluuid string, db *gorm.DB, u *util.User) (Collection, error) {
 	var col Collection
 	if err := db.First(&col, "uuid = ?", coluuid).Error; err != nil {
@@ -75,3 +94,86 @@ func GetContentsInPath(coluuid string, path string, db *gorm.DB, u *util.User) (
 
 	return selectedRefs, nil
 }
+
+func GetDirectoryContents(refs []util.ContentWithPath, queryDir, coluuid string) ([]*CollectionListResponse, error) {
+	dirs := make(map[string]bool)
+	var result []*CollectionListResponse
+	for _, r := range refs {
+		directoryContent, err := getDirectoryContent(r, queryDir, coluuid)
+
+		if err != nil {
+			return nil, err
+		}
+
+		if directoryContent != nil { // if there was content
+			if directoryContent.Type == CidTypeDir { // if the content was a directory
+				subDir := directoryContent.Dir
+				if dirs[subDir] { // if the directory had already been added to response, continue
+					continue
+				}
+				dirs[subDir] = true
+			}
+			result = append(result, directoryContent)
+		}
+	}
+	return result, nil
+}
+
+func getDirectoryContent(r util.ContentWithPath, queryDir, coluuid string) (*CollectionListResponse, error) {
+	if r.Path == "" || r.Name == "" {
+		return nil, nil
+	}
+
+	if !strings.HasPrefix(r.Path, queryDir) {
+		return nil, nil
+	}
+
+	relp, err := getRelativePath(r, queryDir)
+	if err != nil {
+		return nil, &util.HttpError{
+			Code:    http.StatusInternalServerError,
+			Reason:  util.ERR_INTERNAL_SERVER,
+			Details: fmt.Sprintf("errored while calculating relative contentPath queryDir=%s, contentPath=%s", queryDir, r.Path),
+		}
+	}
+
+	// Query directory has a subdirectory, which contains the actual content.
+	// if relative contentPath has a /, the file is in a subdirectory
+	// print the directory the file is in if we haven't already
+	if strings.Contains(relp, "/") {
+		parts := strings.Split(relp, "/")
+		subDir := parts[0]
+		return &CollectionListResponse{
+			Name:      subDir,
+			Type:      CidTypeDir,
+			Dir:       queryDir,
+			ColUuid:   coluuid,
+			UpdatedAt: r.UpdatedAt,
+		}, nil
+	}
+
+	// trying to list a CID queryDir, not allowed
+	if r.Type == util.Directory {
+		return nil, &util.HttpError{
+			Code:    http.StatusBadRequest,
+			Reason:  util.ERR_BAD_REQUEST,
+			Details: fmt.Sprintf("listing CID directories is not allowed"),
+		}
+	}
+	return &CollectionListResponse{
+		Name:      r.Name,
+		Type:      CidTypeFile,
+		Size:      r.Size,
+		ContID:    r.ID,
+		Cid:       &util.DbCID{CID: r.Cid.CID},
+		Dir:       queryDir,
+		ColUuid:   coluuid,
+		UpdatedAt: r.UpdatedAt,
+	}, nil
+}
+
+func getRelativePath(r util.ContentWithPath, queryDir string) (string, error) {
+	contentPath := r.Path
+	relp, err := filepath.Rel(queryDir, contentPath)
+	return relp, err
+}

collections/collections_test.go

@@ -0,0 +1,115 @@
+package collections
+
+import (
+	"github.com/application-research/estuary/util"
+	"testing"
+)
+
+const (
+	name        = "a"
+	contentPath = "/dir1/dir2/dir3/a"
+	contentId   = uint(1)
+	coluuid     = "collection uuid"
+)
+
+func setupRef() util.ContentWithPath {
+	return util.ContentWithPath{
+		Path: contentPath,
+		Content: util.Content{
+			ID:   contentId,
+			Name: name,
+		},
+	}
+}
+
+func TestSubDirectory(t *testing.T) {
+	ref := setupRef()
+	queryDir := "/dir1"
+	expectedResponse := CollectionListResponse{
+		Name:    "dir2",
+		Type:    CidTypeDir,
+		Size:    0,
+		ContID:  0,
+		Dir:     queryDir,
+		ColUuid: coluuid,
+	}
+	runTest(t, ref, queryDir, coluuid, &expectedResponse)
+}
+
+func TestSubDirectory2(t *testing.T) {
+	ref := setupRef()
+	queryDir := "/dir1/dir2"
+	expectedResponse := CollectionListResponse{
+		Name:    "dir3",
+		Type:    CidTypeDir,
+		Size:    0,
+		ContID:  0,
+		Dir:     queryDir,
+		ColUuid: coluuid,
+	}
+	runTest(t, ref, queryDir, coluuid, &expectedResponse)
+}
+
+func TestSubDirectoryWithFileInIt(t *testing.T) {
+	ref := setupRef()
+	queryDir := "/dir1/dir2/dir3"
+	expectedResponse := CollectionListResponse{
+		Name:    name,
+		Type:    CidTypeFile,
+		Size:    0,
+		ContID:  contentId,
+		Dir:     queryDir,
+		ColUuid: coluuid,
+	}
+	runTest(t, ref, queryDir, coluuid, &expectedResponse)
+}
+
+func TestFullPath(t *testing.T) {
+	ref := setupRef()
+	queryDir := "/dir1/dir2/dir3/a"
+	expectedResponse := CollectionListResponse{
+		Name:    name,
+		Type:    CidTypeFile,
+		Size:    0,
+		ContID:  contentId,
+		Dir:     queryDir,
+		ColUuid: coluuid,
+	}
+	runTest(t, ref, queryDir, coluuid, &expectedResponse)
+}
+
+func TestInvalidDirectory(t *testing.T) {
+	ref := setupRef()
+	queryDir := "/dir0"
+	response, _, err := getDirectoryContent(ref, queryDir, "collection uuid")
+
+	if err != nil {
+		t.Fatalf("Error: %v\n", err)
+	}
+
+	if response != nil {
+		t.Fatalf("Expected nil response but got %v\n", response)
+	}
+
+}
+
+func runTest(t *testing.T, ref util.ContentWithPath, queryDir, coluuid string, expectedResponse *CollectionListResponse) {
+	response, _, err := getDirectoryContent(ref, queryDir, coluuid)
+
+	if err != nil {
+		t.Fatalf("Error: %v\n", err)
+	}
+
+	parsedResponse := CollectionListResponse{
+		Name:    response.Name,
+		Type:    response.Type,
+		Size:    response.Size,
+		ContID:  response.ContID,
+		Dir:     response.Dir,
+		ColUuid: response.ColUuid,
+	}
+
+	if parsedResponse != *expectedResponse {
+		t.Fatalf("Expected %v, got %v\n", expectedResponse, response)
+	}
+}

contentmgr/pinning.go

@@ -70,7 +70,7 @@ func (cm *ContentManager) PinDelegatesForContent(cont util.Content) []string {
 		return out
 
 	} else {
-		ai, err := cm.addrInfoForShuttle(cont.Location)
+		ai, err := cm.addrInfoForContentLocation(cont.Location)
 		if err != nil {
 			cm.log.Errorf("failed to get address info for shuttle %q: %s", cont.Location, err)
 			return nil
@@ -176,7 +176,7 @@ func (cm *ContentManager) GetPinOperation(cont util.Content, peers []*peer.AddrI
 		Name:     cont.Name,
 		Peers:    operation.SerializePeers(peers),
 		Started:  cont.CreatedAt,
-		Status:   types.PinningStatusQueued,
+		Status:   types.GetContentPinningStatus(cont),
 		Replace:  replaceID,
 		Location: cont.Location,
 		MakeDeal: makeDeal,
@@ -428,7 +428,7 @@ func (cm *ContentManager) DoPinning(ctx context.Context, op *operation.PinningOp
 		}()
 	}
 
-	prs, _ := operation.UnSerializePeers(op.Peers)
+	prs := operation.UnSerializePeers(op.Peers)
 	for _, pi := range prs {
 		if err := cm.node.Host.Connect(ctx, *pi); err != nil {
 			cm.log.Warnf("failed to connect to origin node for pinning operation: %s", err)