Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Restore missing dryrun values that were removed in 10.27 #2923

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions cmd/copyEnumeratorInit.go
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,20 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
FromTo: cca.FromTo,
Source: src,
Destination: dst,
SourceSize: &transfer.SourceSize,
HttpHeaders: blob.HTTPHeaders{
BlobCacheControl: &transfer.CacheControl,
BlobContentDisposition: &transfer.ContentDisposition,
BlobContentEncoding: &transfer.ContentEncoding,
BlobContentLanguage: &transfer.ContentLanguage,
BlobContentMD5: transfer.ContentMD5,
BlobContentType: &transfer.ContentType,
},
Metadata: transfer.Metadata,
BlobTier: &transfer.BlobTier,
BlobVersion: &transfer.BlobVersionID,
BlobTags: transfer.BlobTags,
BlobSnapshot: &transfer.BlobSnapshotID,
}

buf, _ := json.Marshal(tx)
Expand Down
120 changes: 97 additions & 23 deletions cmd/zc_processor.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ package cmd
import (
"encoding/json"
"fmt"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
"net/url"
"strings"
Expand Down Expand Up @@ -64,21 +65,42 @@ func newCopyTransferProcessor(copyJobTemplate *common.CopyJobPartOrderRequest, n
}

type DryrunTransfer struct {
EntityType common.EntityType
BlobType common.BlobType
FromTo common.FromTo
Source string
Destination string
EntityType common.EntityType
BlobType common.BlobType
FromTo common.FromTo
Source string
Destination string
SourceSize *int64
HttpHeaders blob.HTTPHeaders
Metadata common.Metadata
BlobTier *blob.AccessTier
BlobVersion *string
BlobTags common.BlobTags
BlobSnapshot *string
}

type dryrunTransferSurrogate struct {
EntityType string
BlobType string
FromTo string
Source string
Destination string
SourceSize int64 `json:"SourceSize,omitempty"`
ContentType string `json:"ContentType,omitempty"`
ContentEncoding string `json:"ContentEncoding,omitempty"`
ContentDisposition string `json:"ContentDisposition,omitempty"`
ContentLanguage string `json:"ContentLanguage,omitempty"`
CacheControl string `json:"CacheControl,omitempty"`
ContentMD5 []byte `json:"ContentMD5,omitempty"`
BlobTags common.BlobTags `json:"BlobTags,omitempty"`
Metadata common.Metadata `json:"Metadata,omitempty"`
BlobTier blob.AccessTier `json:"BlobTier,omitempty"`
BlobVersion string `json:"BlobVersion,omitempty"`
BlobSnapshotID string `json:"BlobSnapshotID,omitempty"`
}

func (d *DryrunTransfer) UnmarshalJSON(bytes []byte) error {
var surrogate struct {
EntityType string
BlobType string
FromTo string
Source string
Destination string
}
var surrogate dryrunTransferSurrogate

err := json.Unmarshal(bytes, &surrogate)
if err != nil {
Expand All @@ -103,22 +125,59 @@ func (d *DryrunTransfer) UnmarshalJSON(bytes []byte) error {
d.Source = surrogate.Source
d.Destination = surrogate.Destination

d.SourceSize = &surrogate.SourceSize
d.HttpHeaders.BlobContentType = &surrogate.ContentType
d.HttpHeaders.BlobContentEncoding = &surrogate.ContentEncoding
d.HttpHeaders.BlobCacheControl = &surrogate.CacheControl
d.HttpHeaders.BlobContentDisposition = &surrogate.ContentDisposition
d.HttpHeaders.BlobContentLanguage = &surrogate.ContentLanguage
d.HttpHeaders.BlobContentMD5 = surrogate.ContentMD5
d.BlobTags = surrogate.BlobTags
d.Metadata = surrogate.Metadata
d.BlobTier = &surrogate.BlobTier
d.BlobVersion = &surrogate.BlobVersion
d.BlobSnapshot = &surrogate.BlobSnapshotID

return nil
}

func (d DryrunTransfer) MarshalJSON() ([]byte, error) {
surrogate := struct {
EntityType string
BlobType string
FromTo string
Source string
Destination string
}{
derefString := func(str *string) string {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we use common.IffNotNil here instead of this?

if str == nil {
return ""
}

return *str
}

var aTier blob.AccessTier
var srcSize int64
if d.SourceSize != nil {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same for this, I think we can just use common.IffNotNil here and for AccessTier

srcSize = *d.SourceSize
}

if d.BlobTier != nil {
aTier = *d.BlobTier
}

surrogate := dryrunTransferSurrogate{
d.EntityType.String(),
d.BlobType.String(),
d.FromTo.String(),
d.Source,
d.Destination,
srcSize,
derefString(d.HttpHeaders.BlobContentType),
derefString(d.HttpHeaders.BlobContentEncoding),
derefString(d.HttpHeaders.BlobContentDisposition),
derefString(d.HttpHeaders.BlobContentLanguage),
derefString(d.HttpHeaders.BlobCacheControl),
d.HttpHeaders.BlobContentMD5,
d.BlobTags,
d.Metadata,
aTier,
derefString(d.BlobVersion),
derefString(d.BlobSnapshot),
}

return json.Marshal(surrogate)
Expand Down Expand Up @@ -185,10 +244,25 @@ func (s *copyTransferProcessor) scheduleCopyTransfer(storedObject StoredObject)

if format == common.EOutputFormat.Json() {
tx := DryrunTransfer{
BlobType: common.FromBlobType(storedObject.blobType),
EntityType: storedObject.entityType,
FromTo: s.copyJobTemplate.FromTo,
Source: common.GenerateFullPath(s.copyJobTemplate.SourceRoot.Value, prettySrcRelativePath),
EntityType: storedObject.entityType,
BlobType: common.FromBlobType(storedObject.blobType),
FromTo: s.copyJobTemplate.FromTo,
Source: common.GenerateFullPath(s.copyJobTemplate.SourceRoot.Value, prettySrcRelativePath),
Destination: "",
SourceSize: &storedObject.size,
HttpHeaders: blob.HTTPHeaders{
BlobCacheControl: &storedObject.cacheControl,
BlobContentDisposition: &storedObject.contentDisposition,
BlobContentEncoding: &storedObject.contentEncoding,
BlobContentLanguage: &storedObject.contentLanguage,
BlobContentMD5: storedObject.md5,
BlobContentType: &storedObject.contentType,
},
Metadata: storedObject.Metadata,
BlobTier: &storedObject.blobAccessTier,
BlobVersion: &storedObject.blobVersionID,
BlobTags: storedObject.blobTags,
BlobSnapshot: &storedObject.blobSnapshotID,
}

if fromTo.To() != common.ELocation.None() && fromTo.To() != common.ELocation.Unknown() {
Expand Down
2 changes: 2 additions & 0 deletions e2etest/newe2e_runazcopy_stdout.go
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,8 @@ func (d *AzCopyParsedDryrunStdout) Write(p []byte) (n int, err error) {
if err != nil {
continue
}

d.Transfers = append(d.Transfers, tx)
}
}

Expand Down
71 changes: 71 additions & 0 deletions e2etest/zt_newe2e_dryrun_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package e2etest

import (
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-storage-azcopy/v10/common"
)

Expand Down Expand Up @@ -75,3 +76,73 @@ func (*DryrunSuite) Scenario_DownloadSync_Encoded(a *ScenarioVariationManager) {
"baz%bish": DryrunOpCopy,
})
}

func (*DryrunSuite) Scenario_ExtraProps(a *ScenarioVariationManager) {
meta := common.Metadata{
"foo": pointerTo("bar"),
}
tags := common.BlobTags{
"foo": "bar",
}

body := NewRandomObjectContentContainer(common.MegaByte * 10)
hash := body.MD5()
srcContainer := CreateResource[ContainerResourceManager](a, GetRootResource(a, common.ELocation.Blob()), ResourceDefinitionContainer{
Objects: ObjectResourceMappingFlat{
"foobar": ResourceDefinitionObject{
ObjectProperties: ObjectProperties{
HTTPHeaders: contentHeaders{
cacheControl: pointerTo("asdf"),
contentDisposition: pointerTo("asdf"),
contentEncoding: pointerTo("asdf"),
contentLanguage: pointerTo("asdf"),
contentType: pointerTo("asdf"),
contentMD5: hash[:],
},
Metadata: meta,
BlobProperties: BlobProperties{
Type: pointerTo(blob.BlobTypeBlockBlob),
Tags: tags,
BlockBlobAccessTier: pointerTo(blob.AccessTierCold),
},
},

Body: body,
},
},
})

dstContainer := CreateResource[ContainerResourceManager](a, GetRootResource(a, common.ELocation.Blob()), ResourceDefinitionContainer{})

stdout, _ := RunAzCopy(a, AzCopyCommand{
Verb: ResolveVariation(a, []AzCopyVerb{AzCopyVerbCopy, AzCopyVerbSync}),
Targets: []ResourceManager{
AzCopyTarget{ResourceManager: srcContainer, AuthType: EExplicitCredentialType.OAuth()},
AzCopyTarget{ResourceManager: dstContainer, AuthType: EExplicitCredentialType.OAuth()},
},
Flags: CopySyncCommonFlags{
Recursive: pointerTo(true),
DryRun: pointerTo(true),
S2SPreserveAccessTier: pointerTo(true),
S2SPreserveBlobTags: pointerTo(true),
},
})

if !a.Dryrun() {
dryrun, ok := stdout.(*AzCopyParsedDryrunStdout)
a.AssertNow("must be dryrun output", Equal{}, ok, true)
a.AssertNow("must contain one entry", Equal{}, len(dryrun.Transfers), 1)

tx := dryrun.Transfers[0]
a.Assert("cache control", Equal{}, DerefOrZero(tx.HttpHeaders.BlobCacheControl), "asdf")
a.Assert("content disposition", Equal{}, DerefOrZero(tx.HttpHeaders.BlobContentDisposition), "asdf")
a.Assert("content encoding", Equal{}, DerefOrZero(tx.HttpHeaders.BlobContentEncoding), "asdf")
a.Assert("content language", Equal{}, DerefOrZero(tx.HttpHeaders.BlobContentLanguage), "asdf")
a.Assert("content type", Equal{}, DerefOrZero(tx.HttpHeaders.BlobContentType), "asdf")
a.Assert("hash", Equal{Deep: true}, tx.HttpHeaders.BlobContentMD5, hash[:])
a.Assert("meta", Equal{Deep: true}, tx.Metadata, meta)
a.Assert("tags", Equal{Deep: true}, tx.BlobTags, tags)
a.Assert("blobType", Equal{}, tx.BlobType, common.EBlobType.BlockBlob())
a.Assert("access tier", Equal{}, DerefOrZero(tx.BlobTier), blob.AccessTierCold)
}
}
Loading