Update postman metadata (#3852)

* Updated Postman metadata fields to contain location uniqueness and took out the unused fields of global_id, field_name, and variable_type.
* Disabled body scanning for now since the only body that is scanned is the currently selected radio button but secrets can still be saved in the other unselected radio button options.
* Updated link generation for more accuracy.
* Updated tests to not use global constant.

Author: casey-tran

pkg/output/plain.go

@@ -110,6 +110,16 @@ func structToMap(obj any) (m map[string]map[string]any, err error) {
 		return
 	}
 	err = json.Unmarshal(data, &m)
+	// Due to PostmanLocationType protobuf field being an enum, we want to be able to assign the string value of the enum to the field without needing to create another Protobuf field.
+	// To have the "UNKNOWN_POSTMAN = 0" value be assigned correctly to the field, we need to check if the Postman workspace ID is filled since every secret in the Postman source
+	// should have a valid workspace ID and the 0 value is considered nil for integers.
+	if m["Postman"]["workspace_uuid"] != nil {
+		if m["Postman"]["location_type"] == nil {
+			m["Postman"]["location_type"] = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN.String()
+		} else {
+			m["Postman"]["location_type"] = obj.(*source_metadatapb.MetaData_Postman).Postman.LocationType.String()
+		}
+	}
 	return
 }
 

pkg/pb/source_metadatapb/source_metadata.pb.go

@@ -27,7 +27,6 @@ import (
 const (
 	SourceType       = sourcespb.SourceType_SOURCE_TYPE_POSTMAN
 	LINK_BASE_URL    = "https://go.postman.co/"
-	GLOBAL_TYPE      = "globals"
 	ENVIRONMENT_TYPE = "environment"
 	AUTH_TYPE        = "authorization"
 	REQUEST_TYPE     = "request"
@@ -147,7 +146,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 		if err = json.Unmarshal(contents, &env); err != nil {
 			return err
 		}
-		s.scanVariableData(ctx, chunksChan, Metadata{EnvironmentName: env.ID, fromLocal: true, Link: envPath}, env)
+		s.scanVariableData(ctx, chunksChan, Metadata{EnvironmentID: env.ID, EnvironmentName: env.Name, fromLocal: true, Link: envPath, LocationType: source_metadatapb.PostmanLocationType_ENVIRONMENT_VARIABLE}, env)
 	}
 
 	// Scan local workspaces
@@ -230,7 +229,9 @@ func (s *Source) scanLocalWorkspace(ctx context.Context, chunksChan chan *source
 
 	for _, environment := range workspace.EnvironmentsRaw {
 		metadata.Link = strings.TrimSuffix(path.Base(filePath), path.Ext(filePath)) + "/environments/" + environment.ID + ".json"
+		metadata.LocationType = source_metadatapb.PostmanLocationType_ENVIRONMENT_VARIABLE
 		s.scanVariableData(ctx, chunksChan, metadata, environment)
+		metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 	for _, collection := range workspace.CollectionsRaw {
 		metadata.Link = strings.TrimSuffix(path.Base(filePath), path.Ext(filePath)) + "/collections/" + collection.Info.PostmanID + ".json"
@@ -265,13 +266,20 @@ func (s *Source) scanWorkspace(ctx context.Context, chunksChan chan *sources.Chu
 		metadata.Link = LINK_BASE_URL + "environments/" + envID.UUID
 		metadata.FullID = envVars.ID
 		metadata.EnvironmentID = envID.UUID
+		metadata.EnvironmentName = envVars.Name
 
 		ctx.Logger().V(2).Info("scanning environment vars", "environment_uuid", metadata.FullID)
 		for _, word := range strings.Split(envVars.Name, " ") {
 			s.attemptToAddKeyword(word)
 		}
-
+		metadata.LocationType = source_metadatapb.PostmanLocationType_ENVIRONMENT_VARIABLE
 		s.scanVariableData(ctx, chunksChan, metadata, envVars)
+		metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
+		metadata.Type = ""
+		metadata.Link = ""
+		metadata.FullID = ""
+		metadata.EnvironmentID = ""
+		metadata.EnvironmentName = ""
 		ctx.Logger().V(2).Info("finished scanning environment vars", "environment_uuid", metadata.FullID)
 	}
 	ctx.Logger().V(2).Info("finished scanning environments")
@@ -305,11 +313,13 @@ func (s *Source) scanCollection(ctx context.Context, chunksChan chan *sources.Ch
 		metadata.Link = LINK_BASE_URL + COLLECTION_TYPE + "/" + metadata.FullID
 	}
 
+	metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_VARIABLE
 	// variables must be scanned first before drilling down into the folders and events
 	// because we need to pick up the substitutions from the top level collection variables
 	s.scanVariableData(ctx, chunksChan, metadata, VariableData{
 		KeyValues: collection.Variables,
 	})
+	metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 
 	for _, event := range collection.Events {
 		s.scanEvent(ctx, chunksChan, metadata, event)
@@ -345,6 +355,7 @@ func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, c
 
 	// check if there are any requests in the folder
 	if item.Request.Method != "" {
+		metadata.FolderName = strings.Replace(metadata.FolderName, (" > " + item.Name), "", -1)
 		metadata.RequestID = item.ID
 		metadata.RequestName = item.Name
 		metadata.Type = REQUEST_TYPE
@@ -368,8 +379,16 @@ func (s *Source) scanItem(ctx context.Context, chunksChan chan *sources.Chunk, c
 		s.scanEvent(ctx, chunksChan, metadata, event)
 	}
 
+	if metadata.RequestID != "" {
+		metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_AUTHORIZATION
+	} else if metadata.FolderID != "" {
+		metadata.LocationType = source_metadatapb.PostmanLocationType_FOLDER_AUTHORIZATION
+	} else if metadata.CollectionInfo.UID != "" {
+		metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_AUTHORIZATION
+	}
 	// an auth all by its lonesome could be inherited to subfolders and requests
 	s.scanAuth(ctx, chunksChan, metadata, item.Auth, item.Request.URL)
+	metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 }
 
 func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, event Event) {
@@ -378,15 +397,24 @@ func (s *Source) scanEvent(ctx context.Context, chunksChan chan *sources.Chunk,
 
 	// Prep direct links. Ignore updating link if it's a local JSON file
 	if !metadata.fromLocal {
-		metadata.Link = LINK_BASE_URL + metadata.Type + "/" + metadata.FullID
+		metadata.Link = LINK_BASE_URL + (strings.Replace(metadata.Type, " > event", "", -1)) + "/" + metadata.FullID
 		if event.Listen == "prerequest" {
 			metadata.Link += "?tab=pre-request-scripts"
 		} else {
 			metadata.Link += "?tab=tests"
 		}
 	}
 
+	if strings.Contains(metadata.Type, REQUEST_TYPE) {
+		metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_SCRIPT
+	} else if strings.Contains(metadata.Type, FOLDER_TYPE) {
+		metadata.LocationType = source_metadatapb.PostmanLocationType_FOLDER_SCRIPT
+	} else if strings.Contains(metadata.Type, COLLECTION_TYPE) {
+		metadata.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_SCRIPT
+	}
+
 	s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(metadata, data)), metadata)
+	metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 }
 
 func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, auth Auth, u URL) {
@@ -471,7 +499,16 @@ func (s *Source) scanAuth(ctx context.Context, chunksChan chan *sources.Chunk, m
 	s.attemptToAddKeyword(authData)
 
 	m.FieldType = AUTH_TYPE
+
+	if strings.Contains(m.Type, REQUEST_TYPE) {
+		m.LocationType = source_metadatapb.PostmanLocationType_REQUEST_AUTHORIZATION
+	} else if strings.Contains(m.Type, FOLDER_TYPE) {
+		m.LocationType = source_metadatapb.PostmanLocationType_FOLDER_AUTHORIZATION
+	} else if strings.Contains(m.Type, COLLECTION_TYPE) {
+		m.LocationType = source_metadatapb.PostmanLocationType_COLLECTION_AUTHORIZATION
+	}
 	s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, authData)), m)
+	m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 }
 
 func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.Chunk, metadata Metadata, r Request) {
@@ -484,66 +521,38 @@ func (s *Source) scanHTTPRequest(ctx context.Context, chunksChan chan *sources.C
 			KeyValues: r.Header,
 		}
 		metadata.Type = originalType + " > header"
+		metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_HEADER
 		s.scanVariableData(ctx, chunksChan, metadata, vars)
+		metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 
 	if r.URL.Raw != "" {
 		metadata.Type = originalType + " > request URL (no query parameters)"
 		// Note: query parameters are handled separately
 		u := fmt.Sprintf("%s://%s/%s", r.URL.Protocol, strings.Join(r.URL.Host, "."), strings.Join(r.URL.Path, "/"))
+		metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_URL
 		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(metadata, u)), metadata)
+		metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 
 	if len(r.URL.Query) > 0 {
 		vars := VariableData{
 			KeyValues: r.URL.Query,
 		}
 		metadata.Type = originalType + " > GET parameters (query)"
+		metadata.LocationType = source_metadatapb.PostmanLocationType_REQUEST_QUERY_PARAMETER
 		s.scanVariableData(ctx, chunksChan, metadata, vars)
+		metadata.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 
 	if r.Auth.Type != "" {
 		metadata.Type = originalType + " > request auth"
 		s.scanAuth(ctx, chunksChan, metadata, r.Auth, r.URL)
 	}
 
-	if r.Body.Mode != "" {
-		metadata.Type = originalType + " > body"
-		s.scanBody(ctx, chunksChan, metadata, r.Body)
-	}
-}
-
-func (s *Source) scanBody(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, b Body) {
-	if !m.fromLocal {
-		m.Link = m.Link + "?tab=body"
-	}
-	originalType := m.Type
-	switch b.Mode {
-	case "formdata":
-		m.Type = originalType + " > form data"
-		vars := VariableData{
-			KeyValues: b.FormData,
-		}
-		s.scanVariableData(ctx, chunksChan, m, vars)
-	case "urlencoded":
-		m.Type = originalType + " > url encoded"
-		vars := VariableData{
-			KeyValues: b.URLEncoded,
-		}
-		s.scanVariableData(ctx, chunksChan, m, vars)
-	case "raw", "graphql":
-		data := b.Raw
-		if b.Mode == "graphql" {
-			m.Type = originalType + " > graphql"
-			data = b.GraphQL.Query + " " + b.GraphQL.Variables
-		}
-		if b.Mode == "raw" {
-			m.Type = originalType + " > raw"
-		}
-		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, data)), m)
-	default:
-		break
-	}
+	// We would scan the body, but currently the body has different radio buttons that can be scanned but only the selected one is scanned. The unselected radio button options can still
+	// have secrets in them but will not be scanned. The selction of the radio button will also change the secret metadata for that particular scanning pass and can create confusion for
+	// the user as to the status of a secret. We will reimplement at some point.
 }
 
 func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources.Chunk, m Metadata, response Response) {
@@ -558,13 +567,17 @@ func (s *Source) scanHTTPResponse(ctx context.Context, chunksChan chan *sources.
 			KeyValues: response.Header,
 		}
 		m.Type = originalType + " > response header"
+		m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_HEADER
 		s.scanVariableData(ctx, chunksChan, m, vars)
+		m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 
 	// Body in a response is just a string
 	if response.Body != "" {
 		m.Type = originalType + " > response body"
+		m.LocationType = source_metadatapb.PostmanLocationType_RESPONSE_BODY
 		s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(s.buildSubstitueSet(m, response.Body)), m)
+		m.LocationType = source_metadatapb.PostmanLocationType_UNKNOWN_POSTMAN
 	}
 
 	if response.OriginalRequest.Method != "" {
@@ -600,14 +613,22 @@ func (s *Source) scanVariableData(ctx context.Context, chunksChan chan *sources.
 	}
 
 	m.FieldType = m.Type + " variables"
+	switch m.FieldType {
+	case "request > GET parameters (query) variables":
+		m.Link = m.Link + "?tab=params"
+	case "request > header variables":
+		m.Link = m.Link + "?tab=headers"
+	}
 	s.scanData(ctx, chunksChan, s.formatAndInjectKeywords(values), m)
 }
 
 func (s *Source) scanData(ctx context.Context, chunksChan chan *sources.Chunk, data string, metadata Metadata) {
 	if data == "" {
 		return
 	}
-	metadata.FieldType = metadata.Type
+	if metadata.FieldType == "" {
+		metadata.FieldType = metadata.Type
+	}
 
 	chunksChan <- &sources.Chunk{
 		SourceType: s.Type(),
@@ -630,8 +651,7 @@ func (s *Source) scanData(ctx context.Context, chunksChan chan *sources.Chunk, d
 					FolderId:        metadata.FolderID,
 					FolderName:      metadata.FolderName,
 					FieldType:       metadata.FieldType,
-					FieldName:       metadata.FieldName,
-					VariableType:    metadata.VarType,
+					LocationType:    metadata.LocationType,
 				},
 			},
 		},

pkg/pb/source_metadatapb/source_metadata.pb.validate.go

@@ -6,6 +6,8 @@ import (
 	"io"
 	"net/http"
 	"time"
+
+	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 )
 
 const (
@@ -68,17 +70,16 @@ type Metadata struct {
 	EnvironmentID   string
 	CollectionInfo  Info
 	FolderID        string // UUID of the folder (but not full ID)
-	FolderName      string
+	FolderName      string // Folder path if the item is nested under one or more folders
 	RequestID       string // UUID of the request (but not full ID)
 	RequestName     string
 	FullID          string //full ID of the reference item (created_by + ID) OR just the UUID
 	Link            string //direct link to the folder (could be .json file path)
 	Type            string //folder, request, etc.
 	EnvironmentName string
-	VarType         string
-	FieldName       string
-	FieldType       string
+	FieldType       string // Path of the item type
 	fromLocal       bool
+	LocationType    source_metadatapb.PostmanLocationType // The distinct Postman location type that the item falls under
 }
 
 type Collection struct {

pkg/sources/postman/postman.go

@@ -66,14 +66,14 @@ func TestSource_BuildSubstituteSet(t *testing.T) {
 	s := &Source{
 		sub: NewSubstitution(),
 	}
-	s.sub.add(Metadata{Type: GLOBAL_TYPE}, "var1", "value1")
-	s.sub.add(Metadata{Type: GLOBAL_TYPE}, "var2", "value2")
-	s.sub.add(Metadata{Type: GLOBAL_TYPE}, "", "value2")
-	s.sub.add(Metadata{Type: GLOBAL_TYPE}, "continuation_token", "'{{continuation_token}}'")     // this caused an infinite loop in the original implementation
-	s.sub.add(Metadata{Type: GLOBAL_TYPE}, "continuation_token2", "'{{{continuation_token2}}}'") // this caused an infinite loop in the original implementation
+	s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "var1", "value1")
+	s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "var2", "value2")
+	s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "", "value2")
+	s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "continuation_token", "'{{continuation_token}}'")     // this caused an infinite loop in the original implementation
+	s.sub.add(Metadata{Type: ENVIRONMENT_TYPE}, "continuation_token2", "'{{{continuation_token2}}}'") // this caused an infinite loop in the original implementation
 
 	metadata := Metadata{
-		Type: GLOBAL_TYPE,
+		Type: ENVIRONMENT_TYPE,
 	}
 
 	testCases := []struct {

pkg/sources/postman/postman_client.go

@@ -296,10 +296,11 @@ message AzureRepos {
 }
 
 message Postman {
+  reserved 4, 14, 15;
+  reserved "globals_id", "field_name", "variable_type";
   string link = 1;
   string workspace_uuid = 2;
   string workspace_name = 3;
-  string globals_id = 4;
   string collection_id = 5;
   string collection_name = 6;
   string environment_id = 7;
@@ -308,9 +309,29 @@ message Postman {
   string request_name = 10;
   string folder_id = 11;
   string folder_name = 12;
-  string field_type = 13;
-  string field_name = 14;
-  string variable_type = 15;
+  string field_type = 13; // Do not use this field for transport. It is only used for output to STDOUT.
+  PostmanLocationType location_type = 16;
+}
+
+enum PostmanLocationType {
+  UNKNOWN_POSTMAN = 0;
+  REQUEST_QUERY_PARAMETER = 1;
+  REQUEST_AUTHORIZATION = 2;
+  REQUEST_HEADER = 3;
+  REQUEST_BODY_FORM_DATA = 4;
+  REQUEST_BODY_RAW = 5;
+  REQUEST_BODY_URL_ENCODED = 6;
+  REQUEST_BODY_GRAPHQL = 7;
+  REQUEST_SCRIPT = 8;
+  REQUEST_URL = 9;
+  ENVIRONMENT_VARIABLE = 10;
+  FOLDER_AUTHORIZATION = 11;
+  FOLDER_SCRIPT = 12;
+  COLLECTION_SCRIPT = 13;
+  COLLECTION_VARIABLE = 14;
+  COLLECTION_AUTHORIZATION = 15;
+  RESPONSE_BODY = 16;
+  RESPONSE_HEADER = 17;
 }
 
 message Vector {