Bugfixes, tests, and tools around checksums

classads/classads.go

@@ -54,53 +54,73 @@ func (c *ClassAd) Set(name string, value interface{}) {
 	c.attributes[name] = value
 }
 
+func writeValue(buffer *bytes.Buffer, value interface{}) {
+	switch v := value.(type) {
+	case string:
+		// Here, we want to ensure we remove any `\n` chars. We do this because if HTCondor ever uses
+		// the old classad mechanism (typically when the shadow talks to the schedd), \n's are special in
+		// the old syntax as they are the separator from name/value pairs.
+		v = strings.ReplaceAll(v, "\n", "\\n")
+		v = strings.ReplaceAll(v, "\r", "\\r")
+		buffer.WriteString(strconv.QuoteToASCII(v))
+	case map[string]string:
+		buffer.WriteString("[")
+		for key, value := range v {
+			buffer.WriteString(key)
+			buffer.WriteString(" = ")
+			value = strings.ReplaceAll(value, "\n", "\\n")
+			value = strings.ReplaceAll(value, "\r", "\\r")
+			fmt.Fprintf(buffer, "%s; ", strconv.QuoteToASCII(value))
+		}
+		buffer.WriteString("]")
+	case map[string]any:
+		buffer.WriteString("[")
+		for key, value := range v {
+			buffer.WriteString(key)
+			buffer.WriteString(" = ")
+			switch valueType := value.(type) {
+			case int, int64:
+				fmt.Fprintf(buffer, "%v; ", valueType)
+			case string:
+				// See above as to why we ensure the removal of `\n`
+				valueType = strings.ReplaceAll(valueType, "\n", "\\n")
+				valueType = strings.ReplaceAll(valueType, "\r", "\\r")
+				fmt.Fprintf(buffer, "%s; ", strconv.QuoteToASCII(valueType))
+			case bool:
+				fmt.Fprintf(buffer, "%t; ", valueType)
+			case float64:
+				fmt.Fprintf(buffer, "%.3f; ", valueType)
+			case time.Duration:
+				// convert to seconds rounded to nearest millisecond, get 3 significant figures
+				valueTypeFloat := float64(valueType.Round(time.Millisecond).Milliseconds()) / 1000.0
+				fmt.Fprintf(buffer, "%.3f; ", valueTypeFloat)
+			case map[string]string:
+				writeValue(buffer, value)
+				fmt.Fprintf(buffer, "; ")
+			case map[string]any:
+				writeValue(buffer, value)
+				fmt.Fprintf(buffer, "; ")
+			default:
+				if str, ok := valueType.(string); ok {
+					fmt.Fprintf(buffer, "%s; ", strconv.QuoteToASCII(str))
+				} else {
+					fmt.Fprintf(buffer, "%v; ", valueType)
+				}
+			}
+		}
+		buffer.WriteString("]")
+	default:
+		buffer.WriteString(fmt.Sprintf("%v", value))
+	}
+}
+
 func (c *ClassAd) String() string {
 	var buffer bytes.Buffer
 	buffer.WriteString("[")
 	for name, value := range c.attributes {
 		buffer.WriteString(name)
 		buffer.WriteString(" = ")
-		switch v := value.(type) {
-		case string:
-			// Here, we want to ensure we remove any `\n` chars. We do this because if HTCondor ever uses
-			// the old classad mechanism (typically when the shadow talks to the schedd), \n's are special in
-			// the old syntax as they are the separator from name/value pairs.
-			v = strings.ReplaceAll(v, "\n", "\\n")
-			v = strings.ReplaceAll(v, "\r", "\\r")
-			buffer.WriteString(strconv.QuoteToASCII(v))
-		case map[string]interface{}:
-			buffer.WriteString("[")
-			for key, value := range v {
-				buffer.WriteString(key)
-				buffer.WriteString(" = ")
-				switch valueType := value.(type) {
-				case int, int64:
-					fmt.Fprintf(&buffer, "%v; ", valueType)
-				case string:
-					// See above as to why we ensure the removal of `\n`
-					valueType = strings.ReplaceAll(valueType, "\n", "\\n")
-					valueType = strings.ReplaceAll(valueType, "\r", "\\r")
-					fmt.Fprintf(&buffer, "%s; ", strconv.QuoteToASCII(valueType))
-				case bool:
-					fmt.Fprintf(&buffer, "%t; ", valueType)
-				case float64:
-					fmt.Fprintf(&buffer, "%.3f; ", valueType)
-				case time.Duration:
-					// convert to seconds rounded to nearest millisecond, get 3 significant figures
-					valueTypeFloat := float64(valueType.Round(time.Millisecond).Milliseconds()) / 1000.0
-					fmt.Fprintf(&buffer, "%.3f; ", valueTypeFloat)
-				default:
-					if str, ok := valueType.(string); ok {
-						fmt.Fprintf(&buffer, "%s; ", strconv.QuoteToASCII(str))
-					} else {
-						fmt.Fprintf(&buffer, "%v; ", valueType)
-					}
-				}
-			}
-			buffer.WriteString("]")
-		default:
-			buffer.WriteString(fmt.Sprintf("%v", value))
-		}
+		writeValue(&buffer, value)
 		buffer.WriteString("; ")
 	}
 	buffer.WriteString("]")

classads/classads_test.go

@@ -74,6 +74,28 @@ func TestStringClassAd(t *testing.T) {
 	assert.Equal(t, localFileName1, localFileName2)
 }
 
+func TestSubClassAd(t *testing.T) {
+	ad := NewClassAd()
+	subMap := make(map[string]string)
+	subMap["LocalFileName"] = "/path/to/local/copy/of/foo"
+	subMap["Url"] = "url://server/some/directory//foo"
+	ad.Set("SubClassAd", subMap)
+	adStr := ad.String()
+	// We don't know the order of the attributes in the subclassad; test for both
+	option1 := "[SubClassAd = [LocalFileName = \"/path/to/local/copy/of/foo\"; Url = \"url://server/some/directory//foo\"; ]; ]"
+	option2 := "[SubClassAd = [Url = \"url://server/some/directory//foo\"; LocalFileName = \"/path/to/local/copy/of/foo\"; ]; ]"
+	assert.True(t, adStr == option1 || adStr == option2, "ClassAd.String() returned %s, expected %s or %s", adStr, option1, option2)
+
+	ad = NewClassAd()
+	subAd := make(map[string]any)
+	subAd2 := make(map[string]any)
+	subAd["foo"] = subAd2
+	subAd2["bar"] = "baz"
+	ad.Set("SubClassAd", subAd)
+	adStr = ad.String()
+	assert.Equal(t, "[SubClassAd = [foo = [bar = \"baz\"; ]; ]; ]", adStr)
+}
+
 func TestStringQuoteClassAd(t *testing.T) {
 	ad := NewClassAd()
 	ad.Set("StringValue", "Get quotes \"right\"")

client/fed_test.go

@@ -527,15 +527,33 @@ func TestObjectStat(t *testing.T) {
 			var got client.FileInfo
 			if export.Capabilities.PublicReads {
 				statInfo, err := client.DoStat(fed.Ctx, statUrl, client.WithTokenLocation(""))
-				got = *statInfo
 				require.NoError(t, err)
+				got = *statInfo
 			} else {
 				statInfo, err := client.DoStat(fed.Ctx, statUrl, client.WithTokenLocation(tempToken.Name()))
+				require.NoError(t, err)
+				got = *statInfo
+			}
+			assert.Equal(t, int64(13), got.Size)
+			assert.Equal(t, fmt.Sprintf("%s/hello_world.txt", export.FederationPrefix), got.Name)
+			assert.Nil(t, got.Checksums)
+
+			// Repeat the process with checksum requests
+			if export.Capabilities.PublicReads {
+				statInfo, err := client.DoStat(fed.Ctx, statUrl, client.WithTokenLocation(""), client.WithRequestChecksums([]client.ChecksumType{client.AlgCRC32C}))
+				require.NoError(t, err)
 				got = *statInfo
+			} else {
+				statInfo, err := client.DoStat(fed.Ctx, statUrl, client.WithTokenLocation(tempToken.Name()), client.WithRequestChecksums([]client.ChecksumType{client.AlgCRC32C}))
 				require.NoError(t, err)
+				got = *statInfo
 			}
 			assert.Equal(t, int64(13), got.Size)
 			assert.Equal(t, fmt.Sprintf("%s/hello_world.txt", export.FederationPrefix), got.Name)
+			assert.NotNil(t, got.Checksums)
+			val, ok := got.Checksums["crc32c"]
+			assert.True(t, ok)
+			assert.Equal(t, "4d551068", val)
 		}
 	})