diff --git a/.clang-format b/.clang-format new file mode 100644 index 00000000..c8ec40c1 --- /dev/null +++ b/.clang-format @@ -0,0 +1,95 @@ +--- +Language: Cpp +BasedOnStyle: LLVM +AccessModifierOffset: -4 +AlignAfterOpenBracket: Align +AlignConsecutiveAssignments: false +AlignConsecutiveDeclarations: false +AlignEscapedNewlinesLeft: true +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortFunctionsOnASingleLine: Inline +AllowShortIfStatementsOnASingleLine: false +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: true +AlwaysBreakTemplateDeclarations: true +BinPackArguments: true +BinPackParameters: true +BraceWrapping: + AfterClass: false + AfterControlStatement: false + AfterEnum: false + AfterFunction: false + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: false + AfterUnion: false + BeforeCatch: false + BeforeElse: false + IndentBraces: false +BreakBeforeBinaryOperators: None +BreakBeforeBraces: Attach +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BreakAfterJavaFieldAnnotations: false +BreakStringLiterals: true +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 8 +ContinuationIndentWidth: 8 +Cpp11BracedListStyle: true +DerivePointerAlignment: false +DisableFormat: false +ExperimentalAutoDetectBinPacking: false +ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] +IncludeCategories: + - Regex: '^<.*\.h>' + Priority: 1 + - Regex: '^<.*' + Priority: 2 + - Regex: '.*' + Priority: 3 +IncludeIsMainRegex: '([-_](test|unittest))?$' +IndentCaseLabels: true +IndentWidth: 4 +IndentWrappedFunctionNames: false +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: false +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: false +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakString: 1000 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +ReflowComments: true +SortIncludes: true +SpaceAfterCStyleCast: false +SpaceBeforeAssignmentOperators: true +SpaceBeforeParens: ControlStatements +SpaceInEmptyParentheses: false +SpacesBeforeTrailingComments: 2 +SpacesInAngles: false +SpacesInContainerLiterals: true +SpacesInCStyleCastParentheses: false +SpacesInParentheses: false +SpacesInSquareBrackets: false +Standard: Auto +TabWidth: 8 +UseTab: Never +... diff --git a/.gitignore b/.gitignore index 2ccd54ac..2af83f46 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ goofys goofys.test xout s3proxy.jar +/s3proxy.sh +/mnt diff --git a/README.md b/README.md index 564dbb76..ba55748d 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,9 @@ # Overview +Goofys-blkread is a fork of Goofys with a read cache that aggressively +prefetches large data blocks. Enable this cache by `--block-read-cache`. + Goofys allows you to mount an S3 bucket as a filey system. It's a Filey System instead of a File System because goofys strives @@ -19,7 +22,7 @@ close-to-open. # Installation -* On Linux, install via [pre-built binaries](https://github.com/kahing/goofys/releases/latest/download/goofys). +* On Linux, install via [pre-built binaries](https://github.com/kahing/goofys/releases/latest/download/goofys). You may also need to install fuse too if you want to mount it on startup. * On macOS, install via [Homebrew](https://brew.sh/): diff --git a/api/api.go b/api/api.go index cf606aef..1ce4a0b0 100644 --- a/api/api.go +++ b/api/api.go @@ -40,6 +40,8 @@ func Mount( fuseLog.Level = logrus.DebugLevel log.Level = logrus.DebugLevel mountCfg.DebugLogger = GetStdLogger(fuseLog, logrus.DebugLevel) + } else { + GetLogger("fuse").Level = logrus.InfoLevel } if flags.Backend == nil { diff --git a/api/common/config.go b/api/common/config.go index e4f23ad5..d47386f1 100644 --- a/api/common/config.go +++ b/api/common/config.go @@ -45,11 +45,14 @@ type FlagStorage struct { Backend interface{} // Tuning - Cheap bool - ExplicitDir bool - StatCacheTTL time.Duration - TypeCacheTTL time.Duration - HTTPTimeout time.Duration + Cheap bool + ExplicitDir bool + BlockReadCache bool + BlockReadCacheSize uint64 + BlockReadCacheMemRatio float64 + StatCacheTTL time.Duration + TypeCacheTTL time.Duration + HTTPTimeout time.Duration // Debugging DebugFuse bool diff --git a/api/common/panic_logger.go b/api/common/panic_logger.go index 4f23603d..3c710dfa 100644 --- a/api/common/panic_logger.go +++ b/api/common/panic_logger.go @@ -66,10 +66,6 @@ func (fs FusePanicLogger) ForgetInode(ctx context.Context, op *fuseops.ForgetIno defer LogPanic(&err) return fs.Fs.ForgetInode(ctx, op) } -func (fs FusePanicLogger) BatchForget(ctx context.Context, op *fuseops.BatchForgetOp) (err error) { - defer LogPanic(&err) - return fs.Fs.BatchForget(ctx, op) -} func (fs FusePanicLogger) MkDir(ctx context.Context, op *fuseops.MkDirOp) (err error) { defer LogPanic(&err) return fs.Fs.MkDir(ctx, op) diff --git a/go.mod b/go.mod index 313fdd39..de508846 100644 --- a/go.mod +++ b/go.mod @@ -1,37 +1,86 @@ module github.com/kahing/goofys -go 1.14 +go 1.20 require ( - cloud.google.com/go/storage v1.14.0 + cloud.google.com/go/storage v1.30.1 github.com/Azure/azure-pipeline-go v0.2.3 - github.com/Azure/azure-sdk-for-go v61.4.0+incompatible - github.com/Azure/azure-storage-blob-go v0.14.0 - github.com/Azure/go-autorest/autorest v0.11.18 - github.com/Azure/go-autorest/autorest/adal v0.9.13 - github.com/Azure/go-autorest/autorest/azure/auth v0.5.7 - github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible + github.com/Azure/azure-storage-blob-go v0.15.0 + github.com/Azure/go-autorest/autorest v0.11.29 + github.com/Azure/go-autorest/autorest/adal v0.9.23 + github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 + github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 + github.com/aws/aws-sdk-go v1.44.278 + github.com/gofrs/uuid v4.4.0+incompatible + github.com/google/uuid v1.3.0 + github.com/hashicorp/golang-lru/v2 v2.0.3 + github.com/jacobsa/fuse v0.0.0-20230509090321-7263f3a2b474 + github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 + github.com/mitchellh/go-homedir v1.1.0 + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 + github.com/sevlyar/go-daemon v0.1.6 + github.com/shirou/gopsutil v3.21.11+incompatible + github.com/sirupsen/logrus v1.9.3 + github.com/urfave/cli v1.22.13 + golang.org/x/oauth2 v0.8.0 + golang.org/x/sync v0.2.0 + golang.org/x/sys v0.8.0 + google.golang.org/api v0.125.0 + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c + gopkg.in/ini.v1 v1.67.0 +) + +require ( + cloud.google.com/go v0.110.2 // indirect + cloud.google.com/go/compute v1.20.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v1.1.0 // indirect + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect - github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect - github.com/aws/aws-sdk-go v1.44.37 - github.com/go-ole/go-ole v1.2.5 // indirect - github.com/gofrs/uuid v4.2.0+incompatible - github.com/google/uuid v1.2.0 + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/StackExchange/wmi v1.2.1 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/dimchansky/utfbom v1.1.1 // indirect + github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/s2a-go v0.1.4 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect + github.com/googleapis/gax-go/v2 v2.10.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20210413103415-7d3cbed7d026 // indirect - github.com/jacobsa/fuse v0.0.0-20221016084658-a4cd154343d8 - github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 - github.com/mitchellh/go-homedir v1.1.0 - github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b - github.com/sevlyar/go-daemon v0.1.5 - github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984 - github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92 + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jstemmer/go-junit-report v1.0.0 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect + github.com/kr/pretty v0.2.1 // indirect + github.com/kr/text v0.1.0 // indirect + github.com/mattn/go-ieproxy v0.0.11 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect github.com/smartystreets/assertions v1.2.0 // indirect - github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a - golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.2.0 - google.golang.org/api v0.43.0 - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c - gopkg.in/ini.v1 v1.51.0 + github.com/smartystreets/goconvey v1.6.4 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/crypto v0.9.0 // indirect + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect + golang.org/x/mod v0.10.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/text v0.9.0 // indirect + golang.org/x/tools v0.9.3 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/grpc v1.55.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index ccb6a0a7..3df2a18c 100644 --- a/go.sum +++ b/go.sum @@ -19,15 +19,23 @@ cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPT cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0 h1:oqqswrt4x6b9OGBnNqdssxBl1xf0rSUNjU2BR4BZar0= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= +cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v1.20.0 h1:cUOcywWuowO9It2i1KX1lIb0HH7gLv6nENKuZGnlcSo= +cloud.google.com/go/compute v1.20.0/go.mod h1:kn5BhC++qUWR/AM3Dn21myV7QbgqejW04cAOrtppaQI= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/iam v1.1.0 h1:67gSqaPukx7O8WLLHMa0PNs3EBGd2eE4d+psbO/CO94= +cloud.google.com/go/iam v1.1.0/go.mod h1:nxdHjaKfCr7fNYx/HJMM8LgiMugmveWlkatear5gVyk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -39,37 +47,50 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0 h1:6RRlFMv1omScs6iq2hfE3IvgE+l6RfJPampq8UZc5TU= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= -github.com/Azure/azure-pipeline-go v0.2.2 h1:6oiIS9yaG6XCCzhgAgKFfIWyo4LLCiDhZot6ltoThhY= -github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go v32.1.0+incompatible h1:BPMo+kL2AuaRMHW4hA1glEvquAmBvH5nEuMmfe3+D38= -github.com/Azure/azure-sdk-for-go v32.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v61.4.0+incompatible h1:BF2Pm3aQWIa6q9KmxyF1JYKYXtVw67vtvu2Wd54NGuY= github.com/Azure/azure-sdk-for-go v61.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2 h1:2aS42GGoGBG81HdmxlNh8Sq29TlPNW1AkexyoRcIc18= -github.com/Azure/azure-storage-blob-go v0.7.1-0.20190724222048-33c102d4ffd2/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= +github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= +github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= +github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= +github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= +github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= +github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= github.com/Azure/go-autorest/autorest/azure/auth v0.5.7 h1:8DQB8yl7aLQuP+nuR5e2RO6454OvFlSTXXaNHshc16s= github.com/Azure/go-autorest/autorest/azure/auth v0.5.7/go.mod h1:AkzUsqkrdmNhfP2i54HqINVQopw0CLDnvHpJ88Zz1eI= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.12/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY= github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= @@ -80,24 +101,31 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 h1:5sXbqlSomvdjlRbWyNqkPsJ3Fg+tQZCbgeX1VGljbQY= github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aws/aws-sdk-go v1.44.37 h1:KvDxCX6dfJeEDC77U5GPGSP0ErecmNnhDHFxw+NIvlI= github.com/aws/aws-sdk-go v1.44.37/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/aws/aws-sdk-go v1.44.278 h1:jJFDO/unYFI48WQk7UGSyO3rBA/gnmRpNYNuAw/fPgE= +github.com/aws/aws-sdk-go v1.44.278/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -105,16 +133,21 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= @@ -125,10 +158,13 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8= +github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -141,17 +177,27 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= +github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -175,6 +221,10 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -189,11 +239,14 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -207,13 +260,20 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.2.4 h1:uGy6JWR/uMIILU8wbf+OkstIrNiMjGpEIyhx8f6W7s4= +github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.10.0 h1:ebSgKfMxynOdxw8QQuFOKMgomqeLGPqNLQox2bo42zg= +github.com/googleapis/gax-go/v2 v2.10.0/go.mod h1:4UOEnMCrxsSqQ940WnTiD6qJ63le2ev3xfyagutxiPw= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20210413103415-7d3cbed7d026 h1:VNVgNDgai+7LqdALaNTNKDD+EriKUN30tvQH6bgYjh8= github.com/gopherjs/gopherjs v0.0.0-20210413103415-7d3cbed7d026/go.mod h1:Opf9rtYVq0eTyX+aRVmRO9hE8ERAozcdrBxWG9Q6mkQ= @@ -221,6 +281,7 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -236,6 +297,10 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= +github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.3 h1:kmRrRLlInXvng0SmLxmQpQkpbYAvcXm7NPDrgxJa9mE= +github.com/hashicorp/golang-lru/v2 v2.0.3/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -244,16 +309,10 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jacobsa/fuse v0.0.0-20220303082815-1c9fe7bc84a5 h1:LkLYmg9wdUlhXv9H31druPMUFPSrpN9sWMxPCTvu5dI= -github.com/jacobsa/fuse v0.0.0-20220303082815-1c9fe7bc84a5/go.mod h1:xtZnnLxHY6QniCrfIpTwr5h8mH8zr+jsOFj0y9cfyp4= github.com/jacobsa/fuse v0.0.0-20221016084658-a4cd154343d8 h1:uv+7DeBJF6+a54ZUihuXR+uTzhv2JslllK5ByILxxbg= github.com/jacobsa/fuse v0.0.0-20221016084658-a4cd154343d8/go.mod h1:liOmRdJd8oTwHCQ5M9JemRE3CebdlYcZWLk+ZjQeuq0= -github.com/jacobsa/oglematchers v0.0.0-20150720000706-141901ea67cd/go.mod h1:TlmyIZDpGmwRoTWiakdr+HA1Tukze6C6XbRVidYq02M= -github.com/jacobsa/oglemock v0.0.0-20150831005832-e94d794d06ff/go.mod h1:gJWba/XXGl0UoOmBQKRWCJdHrr3nE0T65t6ioaj3mLI= -github.com/jacobsa/ogletest v0.0.0-20170503003838-80d50a735a11/go.mod h1:+DBdDyfoO2McrOyDemRBq0q9CMEByef7sYl7JH5Q3BI= -github.com/jacobsa/reqtrace v0.0.0-20150505043853-245c9e0234cb/go.mod h1:ivcmUvxXWjb27NsPEaiYK7AidlZXS7oQ5PowUS9z3I4= -github.com/jacobsa/syncutil v0.0.0-20180201203307-228ac8e5a6c3/go.mod h1:mPvulh9VKXvo+yOlrD4VYOOYuLdZJ36wa/5QIrtXvWs= -github.com/jacobsa/timeutil v0.0.0-20170205232429-577e5acbbcf6/go.mod h1:JEWKD6V8xETMW+DEv+IQVz++f8Cn8O/X0HPeDY3qNis= +github.com/jacobsa/fuse v0.0.0-20230509090321-7263f3a2b474 h1:6z3Yj4PZKk3n18T2s7RYCa4uBCOpeLoQfDH/mUZTrVo= +github.com/jacobsa/fuse v0.0.0-20230509090321-7263f3a2b474/go.mod h1:XUKuYy1M4vamyxQjW8/WZBTxyZ0NnUiq+kkA+WWOfeI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -263,6 +322,8 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jstemmer/go-junit-report v1.0.0 h1:8X1gzZpR+nVQLAht+L/foqOeX2l9DTZoaIPbEQHxsds= +github.com/jstemmer/go-junit-report v1.0.0/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= @@ -272,8 +333,9 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -281,14 +343,12 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw= -github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= +github.com/mattn/go-ieproxy v0.0.11 h1:MQ/5BuGSgDAHZOJe6YY80IF2UVCfGkwfo6AeD7HtHYo= +github.com/mattn/go-ieproxy v0.0.11/go.mod h1:/NsJd+kxZBmjMc5hrJCKMbP57B84rvq9BiDRbtO9AS0= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -308,6 +368,8 @@ github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJE github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -327,16 +389,21 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= -github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sevlyar/go-daemon v0.1.5 h1:Zy/6jLbM8CfqJ4x4RPr7MJlSKt90f00kNM1D401C+Qk= github.com/sevlyar/go-daemon v0.1.5/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW3MebLDRKE= +github.com/sevlyar/go-daemon v0.1.6 h1:EUh1MDjEM4BI109Jign0EaknA2izkOyi0LV3ro3QQGs= +github.com/sevlyar/go-daemon v0.1.6/go.mod h1:6dJpPatBT9eUwM5VCw9Bt6CdX9Tk6UWvhW3MebLDRKE= github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984 h1:wsZAb4P8F7uQSwsnxE1gk9AHCcc5U0wvyDzcLwFY0Eo= github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= @@ -346,6 +413,8 @@ github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJV github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92 h1:cCFqi2EDJ/eRGFkhvqpkh/6t8dubmVVejQatc7BxG2A= github.com/sirupsen/logrus v1.4.3-0.20190807103436-de736cf91b92/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= @@ -362,21 +431,39 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a h1:Q372CokLjPoo4bx4qmpjaxZ0yEJR05rZkeEyXcEvSTY= github.com/urfave/cli v1.21.1-0.20190807111034-521735b7608a/go.mod h1:pLnliFgEwGWwRKy8217bLxXBDmG6pBQeZcIzf+m7U+k= +github.com/urfave/cli v1.22.13 h1:wsLILXG8qCJNse/qAgLNf23737Cx05GflHg/PJGe1Ok= +github.com/urfave/cli v1.22.13/go.mod h1:VufqObjsMTF2BBwKawpx9R8eAneNEWhoO0yx8Vd+FkE= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -386,6 +473,9 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -397,11 +487,18 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -426,6 +523,8 @@ golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -437,6 +536,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1 h1:Kvvh58BN8Y9/lBi7hTekvtMpm07eUZ0ck5pRHpsMWrY= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -475,11 +577,16 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220526153639-5463443f8c37 h1:lUkvobShwKsOesNfWWlCS5q7fnbG1MEliIzwu886fn8= golang.org/x/net v0.0.0-20220526153639-5463443f8c37/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -493,6 +600,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558 h1:D7nTwh4J0i+5mW4Zjzn5omvlr6YBcWywE6KOcatyNxY= golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -505,6 +614,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -555,15 +667,21 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -574,6 +692,11 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -628,11 +751,16 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -655,6 +783,8 @@ google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjR google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0 h1:4sAyIHT6ZohtAQDoxws+ez7bROYmUlOVvsUscYCDTqA= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.125.0 h1:7xGvEY4fyWbhWMHf3R2/4w7L4fXyfpRGE9g6lp8+DCk= +google.golang.org/api v0.125.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -686,6 +816,7 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= @@ -704,6 +835,12 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6 h1:4Xw2NwItrJOFR5s6PnK98PI6Bgw1LhMP1j/rO5WP0S4= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -717,11 +854,15 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0 h1:o1bcQ6imQMIOpdrO3SWf2z5RV72WbDwdXuK0MDlc8As= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -733,25 +874,33 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/backend_azblob.go b/internal/backend_azblob.go index b83fedbb..0c174dde 100644 --- a/internal/backend_azblob.go +++ b/internal/backend_azblob.go @@ -805,7 +805,10 @@ func (b *AZBlob) PutBlob(param *PutBlobInput) (*PutBlobOutput, error) { azblob.BlobHTTPHeaders{ ContentType: NilStr(param.ContentType), }, - nilMetadata(param.Metadata), azblob.BlobAccessConditions{}, azblob.AccessTierNone, azblob.BlobTagsMap{}, azblob.ClientProvidedKeyOptions{}) + nilMetadata(param.Metadata), azblob.BlobAccessConditions{}, + azblob.AccessTierNone, azblob.BlobTagsMap{}, + azblob.ClientProvidedKeyOptions{}, + azblob.ImmutabilityPolicyOptions{}) if err != nil { return nil, mapAZBError(err) } @@ -872,7 +875,8 @@ func (b *AZBlob) MultipartBlobCommit(param *MultipartBlobCommitInput) (*Multipar resp, err := blob.CommitBlockList(context.TODO(), parts, azblob.BlobHTTPHeaders{}, nilMetadata(param.Metadata), - azblob.BlobAccessConditions{}, azblob.AccessTierNone, azblob.BlobTagsMap{}, azblob.ClientProvidedKeyOptions{}) + azblob.BlobAccessConditions{}, azblob.AccessTierNone, azblob.BlobTagsMap{}, azblob.ClientProvidedKeyOptions{}, + azblob.ImmutabilityPolicyOptions{}) if err != nil { return nil, mapAZBError(err) } diff --git a/internal/backend_s3.go b/internal/backend_s3.go index 5bb4c3ba..06fcf71d 100644 --- a/internal/backend_s3.go +++ b/internal/backend_s3.go @@ -22,6 +22,7 @@ import ( "net/url" "strconv" "strings" + "sync" "sync/atomic" "syscall" "time" @@ -49,18 +50,28 @@ type S3Backend struct { aws bool gcs bool v2Signer bool + + // get stats + getStatsLock *sync.Mutex + totalGetCnt uint64 + totalGetBytes uint64 + prevGetPrintCnt uint64 + prevGetPrintTime time.Time } +const S3_PRICE_PER_1K_GET = 0.0004 + func NewS3(bucket string, flags *FlagStorage, config *S3Config) (*S3Backend, error) { awsConfig, err := config.ToAwsConfig(flags) if err != nil { return nil, err } s := &S3Backend{ - bucket: bucket, - awsConfig: awsConfig, - flags: flags, - config: config, + bucket: bucket, + awsConfig: awsConfig, + flags: flags, + config: config, + getStatsLock: &sync.Mutex{}, cap: Capabilities{ Name: "s3", MaxMultipartSize: 5 * 1024 * 1024 * 1024, @@ -743,6 +754,7 @@ func (s *S3Backend) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) { if err != nil { return nil, mapAwsError(err) } + s.updateGetStats(uint64(*resp.ContentLength)) return &GetBlobOutput{ HeadBlobOutput: HeadBlobOutput{ @@ -761,6 +773,33 @@ func (s *S3Backend) GetBlob(param *GetBlobInput) (*GetBlobOutput, error) { }, nil } +func (s *S3Backend) updateGetStats(size uint64) { + getCnt := atomic.AddUint64(&s.totalGetCnt, 1) + getBytes := atomic.AddUint64(&s.totalGetBytes, size) + + if getCnt < atomic.LoadUint64(&s.prevGetPrintCnt)+5000 { + return + } + + curTime := time.Now() + s.getStatsLock.Lock() + if curTime.Sub(s.prevGetPrintTime).Seconds() >= 5 { + s.prevGetPrintTime = curTime + s.getStatsLock.Unlock() + atomic.StoreUint64(&s.prevGetPrintCnt, getCnt) + + cost := float64(getCnt) / 1000 * S3_PRICE_PER_1K_GET + log.Infof("S3: GET stats: req=%v avg_size=%.3fMiB"+ + " total_cost=$%.3f avg_cost=$%.3f/GiB", + getCnt, + float64(getBytes)/float64(getCnt*1024*1024), + cost, + cost/float64(getBytes)/1024/1024/1024) + } else { + s.getStatsLock.Unlock() + } +} + func getDate(resp *http.Response) *time.Time { date := resp.Header.Get("Date") if date != "" { diff --git a/internal/block_cache.go b/internal/block_cache.go new file mode 100644 index 00000000..ffada33f --- /dev/null +++ b/internal/block_cache.go @@ -0,0 +1,517 @@ +package internal + +import ( + . "github.com/kahing/goofys/api/common" + + "fmt" + "io" + "math" + "sync" + "sync/atomic" + "time" + + "github.com/hashicorp/golang-lru/v2" + "github.com/jacobsa/fuse/fuseops" + "github.com/pbnjay/memory" +) + +// file smaller than this will be read at once +const BLOCK_CACHE_READ_ALL_SIZE = 20 * 1024 * 1024 // 20MiB + +// maximum number of threads working on prefetch or readahead +// note: prefetch is to utilize latency and match throughput; readahead is to +// aggressively read future reads +const MAX_PREFETCH_THREADS = int32(16) + +type cacheKey struct { + inode fuseops.InodeID + blkId uint64 // ID of the block (offset div by BLOCK_SIZE) +} + +type CacheValue struct { + mu sync.RWMutex + Data []byte // nil if error is present + Err error // should be checked before trying to access the value +} + +// cache for read-only files based on large blocks +type BlockCache struct { + mu sync.RWMutex + lru *lru.Cache[cacheKey, *CacheValue] + blockSize uint64 + + // number of alive prefetch threads + numPrefetchThreads int32 + + nrLatencyBlock int // max blocks that can be read without more latency + nrThroughputBlock int // min blocks that can be read with max throughput + benchmarkDone sync.WaitGroup + benchmarkOnce sync.Once +} + +// result of reading the cache +type cacheReadResult struct { + blkId uint64 // block ID + block *CacheValue // the cache block; guaranteed to be non-nil + + // whether this block is newly created; if it is true, write lock of block + // is acquired + isNewAlloc bool +} + +func NewBlockCache(flags *FlagStorage) *BlockCache { + size := int(float64(memory.TotalMemory()/flags.BlockReadCacheSize) * + flags.BlockReadCacheMemRatio) + size = MaxInt(size, + (BLOCK_CACHE_READ_ALL_SIZE*2-1)/int(flags.BlockReadCacheSize)+1) + cache, err := lru.New[cacheKey, *CacheValue](size) + if err != nil { + log.Panic("failed to allocate LRU cache", err) + } + log.Infof("block cache: size=%d mem=%.2fGiB", size, + float64(size)*float64(flags.BlockReadCacheSize)/1024/1024/1024) + ret := &BlockCache{ + lru: cache, + blockSize: flags.BlockReadCacheSize, + } + ret.benchmarkDone.Add(1) + return ret +} + +// Remove cache entries for a specific inode +func (bc *BlockCache) RemoveCache(inode *Inode) { + var keys []cacheKey + var key cacheKey + bc.mu.Lock() + defer bc.mu.Unlock() + + for _, key = range bc.lru.Keys() { + if key.inode == inode.Id { + keys = append(keys, key) + } + } + + for _, key = range keys { + bc.lru.Remove(key) + } +} + +// Get a cache node containing given offset (which must be within the file +// size); this function requests the remote header if cache is unavailable yet. +// Remember to check the error marker before accessing the data. +// If aggressive_prefetch is true, will try to prefetch further blocks. +func (bc *BlockCache) Get(fh *FileHandle, offset uint64) ( + cacheBlock *CacheValue, cacheOffset uint64) { + + fileSize := fh.inode.Attributes.Size + assert(offset < fileSize, "invalid Get") + + blkId := offset / bc.blockSize + cacheOffset = blkId * bc.blockSize + + headRead := bc.getOrAllocate(fh.inode, blkId) + cacheBlock = headRead.block + + if headRead.isNewAlloc { + var blocks []cacheReadResult + + if fileSize <= BLOCK_CACHE_READ_ALL_SIZE { + numBlocks := int( + DivUpUint64(fileSize, bc.blockSize) - headRead.blkId) + blocks = make([]cacheReadResult, 0, numBlocks) + blocks = append(blocks, headRead) + blocks = append(blocks, + bc.getOrAllocateMany(fh.inode, blkId+1, numBlocks-1)...) + } else { + bc.ensureBenchmark(fh) + blocks = bc.getReadAheadBlocks(fh, headRead) + } + + assert(len(blocks) > 0, "invalid read ahead blocks") + if len(blocks) == 1 { + bc.readMultipleBlocks(fh, blocks, false) + } else { + go bc.readMultipleBlocks( + fh, blocks, + fileSize > BLOCK_CACHE_READ_ALL_SIZE && + len(blocks) > bc.nrLatencyBlock) + } + } + + cacheBlock.mu.RLock() // wait the writer to finish + cacheBlock.mu.RUnlock() + assert((cacheBlock.Err == nil) == (cacheBlock.Data != nil), + "block not read") + return +} + +// Start a thread to do read ahead at given location. Return false if prefetch +// threads are exhausted, and true otherwise. +func (bc *BlockCache) StartReadAhead( + fh *FileHandle, offset uint64, size uint64) bool { + + assert(size > 0 && offset < fh.inode.Attributes.Size, "invalid readahead") + + if !bc.acquirePrefetchThread() { + return false + } + + head := bc.getOrAllocate(fh.inode, offset/bc.blockSize) + if !head.isNewAlloc { + // already cached (probably being worked on by existing read ahead), do + // not try to read again + bc.releasePrefetchThread() + return true + } + + go func() { + nrBlock := int(DivUpUint64( + MinUInt64(fh.inode.Attributes.Size, offset+size), + bc.blockSize) - head.blkId) + assert(nrBlock > 0, "invalid nrBlock") + blocks := make([]cacheReadResult, 0, nrBlock) + blocks = append(blocks, head) + blocks = append(blocks, + bc.getOrAllocateMany(fh.inode, head.blkId+1, nrBlock-1)...) + bc.readMultipleBlocks(fh, blocks, true) + }() + return true +} + +// Read into consicutive blocks. +// Write locks will be released after the read is finished. +// If ownPrefetchThread is true, the prefetch thread will be released +func (bc *BlockCache) readMultipleBlocks( + fh *FileHandle, blocks []cacheReadResult, ownPrefetchThread bool) { + + assert(len(blocks) > 0 && blocks[0].isNewAlloc, + "empty blocks for readMultipleBlocks") + + for i := 0; i < len(blocks); i++ { + assert(blocks[i].blkId-blocks[0].blkId == uint64(i), + "block not consecutive") + } + + // remove trailing blocks that are already prepared + for !blocks[len(blocks)-1].isNewAlloc { + blocks = blocks[:len(blocks)-1] + } + + totSize := MinUInt64( + bc.blockSize*uint64(len(blocks)), + fh.inode.Attributes.Size-blocks[0].blkId*bc.blockSize) + resp, err := fh.cloud.GetBlob(&GetBlobInput{ + Key: fh.key, + Start: blocks[0].blkId * bc.blockSize, + Count: totSize, + }) + finished := 0 + + // release the thread, and set error markers + defer func() { + if ownPrefetchThread { + bc.releasePrefetchThread() + } + + for i := finished; i < len(blocks); i++ { + assert(err != nil, "unhandled block read") + if blocks[i].isNewAlloc { + c := blocks[i].block + c.Err = err + c.mu.Unlock() + } + } + }() + + if err != nil { + return + } + + reader := resp.Body + defer reader.Close() + + remainSize := totSize + + for ; finished < len(blocks); finished++ { + blkDone := 0 + curRead := 0 + assert(remainSize > 0, "remain_size=0: blocks exceed file size") + bufSize := int(MinUInt64(remainSize, bc.blockSize)) + remainSize -= uint64(bufSize) + buf := make([]byte, bufSize) + for blkDone < bufSize { + curRead, err = reader.Read(buf[blkDone:]) + blkDone += curRead + if err != nil && (err != io.EOF || blkDone != bufSize) { + // the deferred function will set error markers + return + } + } + if blocks[finished].isNewAlloc { + c := blocks[finished].block + c.Data = buf + c.mu.Unlock() + } + } +} + +// Get blocks for read ahead; blk_id is the first block requested by client +// This function also acquires a prefetch thread if return length is more than +// latency_blocks +func (bc *BlockCache) getReadAheadBlocks( + fh *FileHandle, head cacheReadResult) (ret []cacheReadResult) { + + ret = make([]cacheReadResult, 0, bc.nrThroughputBlock) + ret = append(ret, head) + + // max number of blocks to read + maxBlocks := int(MinUInt64( + DivUpUint64(fh.inode.Attributes.Size, bc.blockSize)-head.blkId, + uint64(bc.nrThroughputBlock))) + + ret = append(ret, bc.getOrAllocateMany(fh.inode, head.blkId+1, + MinInt(maxBlocks, bc.nrLatencyBlock)-1)...) + + if bc.nrLatencyBlock >= maxBlocks { + return + } + + assert(len(ret) == bc.nrLatencyBlock, "ret size does not match") + + if !bc.acquirePrefetchThread() { + return + } + + // read ahead to get max throughput, but stop at the first existing block + ret = append(ret, + bc.getOrAllocateMany2( + fh.inode, + head.blkId+uint64(bc.nrLatencyBlock), + maxBlocks-bc.nrLatencyBlock, + true)...) + + if len(ret) == bc.nrLatencyBlock { + // no new blocks are added + bc.releasePrefetchThread() + } + + return +} + +// try to acquire a prefetch thread; return false if the limit is reached +func (bc *BlockCache) acquirePrefetchThread() bool { + num := atomic.AddInt32(&bc.numPrefetchThreads, 1) + if num > MAX_PREFETCH_THREADS { + bc.releasePrefetchThread() + assert(num <= MAX_PREFETCH_THREADS+10, + "num_prefetch_threads too high") + return false + } + return true +} + +func (bc *BlockCache) releasePrefetchThread() { + num := atomic.AddInt32(&bc.numPrefetchThreads, -1) + assert(num >= 0, "num_prefetch_threads < 0") +} + +// Get a cache block. new_alloc indicates if this block is newly allocated. If +// it is true, the write lock is acquired, and the caller should fill in the +// data. +func (bc *BlockCache) getOrAllocate( + inode *Inode, blkId uint64) (ret cacheReadResult) { + return bc.getOrAllocateMany(inode, blkId, 1)[0] +} + +// Get consecutive cache blocks starting at blk_id +func (bc *BlockCache) getOrAllocateMany( + inode *Inode, blkId uint64, num int) (ret []cacheReadResult) { + return bc.getOrAllocateMany2(inode, blkId, num, false) +} + +// return immediately if requireNewAlloc is true but the block already exists +func (bc *BlockCache) getOrAllocateMany2( + inode *Inode, blkId uint64, num int, requireNewAlloc bool) ( + ret []cacheReadResult) { + + assert(num >= 0, "invalid num") + ret = make([]cacheReadResult, 0, num) + + if num == 0 { + return nil + } + + var ok bool + var block *CacheValue + + hasBcWriteLock := false + bc.mu.RLock() + + for i := 0; i < num; i++ { + key := cacheKey{inode.Id, blkId + uint64(i)} + assert(key.blkId*bc.blockSize < inode.Attributes.Size, + "block exceeds file size") + + block, ok = bc.lru.Get(key) + if ok { + if requireNewAlloc { + break + } + ret = append(ret, cacheReadResult{ + blkId: key.blkId, + block: block, + isNewAlloc: false, + }) + continue + } + + if !hasBcWriteLock { + bc.mu.RUnlock() + bc.mu.Lock() + i-- // try getting again with write lock + hasBcWriteLock = true + continue + } + + // create a new block now + block = &CacheValue{} + block.mu.Lock() // acquire cache wlock before returning + ret = append(ret, cacheReadResult{ + blkId: key.blkId, + block: block, + isNewAlloc: true, + }) + bc.lru.Add(key, block) + } + + if hasBcWriteLock { + bc.mu.Unlock() + } else { + bc.mu.RUnlock() + } + return +} + +// remove a cache entry, usually used due to read error +func (bc *BlockCache) Remove(inode *Inode, offset uint64) { + key := cacheKey{inode.Id, offset / bc.blockSize} + bc.mu.Lock() + bc.lru.Remove(key) + bc.mu.Unlock() +} + +func assert(cond bool, msg string) { + if !cond { + panic(msg) + } +} + +// Run the benchmark if it is not done; file size must be at least +// BLOCK_CACHE_FILE_SIZE_MIN +func (bc *BlockCache) ensureBenchmark(fh *FileHandle) { + + // run a single benchmark and return its execution time in seconds + benchmark_one := func(size int) (secs float64) { + buf := make([]byte, size) + + do := func() float64 { + begin := time.Now() + resp, err := fh.cloud.GetBlob(&GetBlobInput{ + Key: fh.key, + Start: 0, + Count: uint64(size), + }) + if err != nil { + panic(fmt.Sprintf("failed to benchmark: %v", err)) + } + reader := resp.Body + defer reader.Close() + done := 0 + var n int + for done < size { + n, err = reader.Read(buf[done:]) + done += n + if err != nil && (err != io.EOF || done != size) { + panic(fmt.Sprintf("failed to benchmark: %v", err)) + } + } + return time.Since(begin).Seconds() + } + + do() // warm up + + nrun := 0 + totTime := float64(0) + totTime2 := float64(0) + var std float64 + for { + nrun++ + thisTime := do() + totTime += thisTime + totTime2 += thisTime * thisTime + if nrun >= 2 && totTime >= 0.1 { + n := float64(nrun) + mean := totTime / n + std = math.Sqrt(math.Max( + (totTime2/n-mean*mean)*(1+1/(n-1)), 0)) + if std/mean < 0.1 || totTime > 2 { + secs = mean + break + } + } + } + fuseLog.Infof("benchmark: size=%.2fMiB time=%.2fms"+ + "(+-%.2f) thrpt=%.2fMiB/s (%d runs, tot %.2fs)", + float64(size)/1024/1024, secs*1000, std*1000, + float64(size)/1024/1024/secs, nrun, totTime) + return secs + } + + benchmark := func() { + blockSize := int(bc.blockSize) + latency := benchmark_one(blockSize) + throughput := BLOCK_CACHE_READ_ALL_SIZE / + benchmark_one(BLOCK_CACHE_READ_ALL_SIZE) + + maxNrBlock := BLOCK_CACHE_READ_ALL_SIZE / blockSize + + nrBlock := 2 + bc.nrLatencyBlock = maxNrBlock + for nrBlock < maxNrBlock { + curLat := benchmark_one(blockSize * nrBlock) + if curLat > latency*1.2 { + bc.nrLatencyBlock = nrBlock / 2 + break + } + nrBlock *= 2 + } + + nrBlock = maxNrBlock / 2 + bc.nrThroughputBlock = bc.nrLatencyBlock + for nrBlock > bc.nrLatencyBlock { + size := blockSize * nrBlock + curThrpt := float64(size) / benchmark_one(size) + if curThrpt < throughput*0.8 { + bc.nrThroughputBlock = nrBlock * 2 + break + } + nrBlock /= 2 + } + bc.benchmarkDone.Done() + + blockSizeMb := float64(blockSize) / 1024 / 1024 + fuseLog.Infof( + "block cache benchmark: latency=%.2fms throughput=%.2fMiB/s"+ + " latency_blocks=%d(%.2fMiB) throughput_blocks=%d(%.2fMiB)", + latency*1000, throughput/1024/1024, + bc.nrLatencyBlock, + float64(bc.nrLatencyBlock)*blockSizeMb, + bc.nrThroughputBlock, + float64(bc.nrThroughputBlock)*blockSizeMb) + } + + bc.benchmarkOnce.Do(benchmark) + bc.benchmarkDone.Wait() + assert(0 < bc.nrLatencyBlock && bc.nrLatencyBlock <= bc.nrThroughputBlock, + "invalid benchmark result") +} diff --git a/internal/file.go b/internal/file.go index 69cf5b6d..0066f4f0 100644 --- a/internal/file.go +++ b/internal/file.go @@ -34,9 +34,11 @@ type FileHandle struct { mpuName *string dirty bool - writeInit sync.Once + writeInit sync.Once // used for MPU init mpuWG sync.WaitGroup + writeCachePurgeInit sync.Once // purge the read cache of this file + mu sync.Mutex mpuId *MultipartBlobCommitInput nextWriteOffset int64 @@ -51,11 +53,15 @@ type FileHandle struct { reader io.ReadCloser readBufOffset int64 + blockCacheLastOffset int64 + blockCacheReadAheadEnabled bool + // parallel read - buffers []*S3ReadBuffer + buffers []*S3ReadBuffer // prefetch buffers existingReadahead int seqReadAmount uint64 - numOOORead uint64 // number of out of order read + numOOOPrefetch uint64 // number of out of order read with prefetch + numOOOReadReq uint64 // number of out of order read requests // User space PID. All threads created by a process will have the same TGID, // but different PIDs[1]. // This value can be nil if we fail to get TGID from PID[2]. @@ -74,7 +80,7 @@ const READAHEAD_CHUNK = uint32(20 * 1024 * 1024) func NewFileHandle(inode *Inode, opMetadata fuseops.OpContext) *FileHandle { tgid, err := GetTgid(opMetadata.Pid) if err != nil { - log.Debugf( + fuseLog.Debugf( "Failed to retrieve tgid for the given pid. pid: %v err: %v inode id: %v err: %v", opMetadata.Pid, err, inode.Id, err) } @@ -230,6 +236,13 @@ func (fh *FileHandle) uploadCurrentBuf(parallel bool) (err error) { func (fh *FileHandle) WriteFile(offset int64, data []byte) (err error) { fh.inode.logFuse("WriteFile", offset, len(data)) + fh.writeCachePurgeInit.Do(func() { + bc := fh.inode.fs.blockCache + if bc != nil { + go bc.RemoveCache(fh.inode) + } + }) + fh.mu.Lock() defer fh.mu.Unlock() @@ -409,6 +422,7 @@ func (fh *FileHandle) readFromReadAhead(offset uint64, buf []byte) (bytesRead in // we've exhausted the first buffer readAheadBuf.buf.Close() fh.buffers = fh.buffers[1:] + fh.numOOOPrefetch = 0 // clear the counter after reading one buffer } buf = buf[nread:] @@ -423,8 +437,16 @@ func (fh *FileHandle) readFromReadAhead(offset uint64, buf []byte) (bytesRead in } func (fh *FileHandle) readAhead(offset uint64, needAtLeast int) (err error) { + expectOffset := offset existingReadahead := uint32(0) - for _, b := range fh.buffers { + for bidx, b := range fh.buffers { + if b.offset != expectOffset { + err = errors.New(fmt.Sprintf( + "read ahead buffer offset mismatch: file=%s buf=%v expect=%v got=%v", + *fh.inode.FullName(), bidx, expectOffset, b.offset)) + return + } + expectOffset += uint64(b.size) existingReadahead += b.size } @@ -495,17 +517,13 @@ func (fh *FileHandle) ReadFile(offset int64, buf []byte) (bytesRead int, err err return } -func (fh *FileHandle) readFile(offset int64, buf []byte) (bytesRead int, err error) { - defer func() { - if bytesRead > 0 { - fh.readBufOffset += int64(bytesRead) - fh.seqReadAmount += uint64(bytesRead) - } - - fh.inode.logFuse("< readFile", bytesRead, err) - }() +// read file using block cache +func (fh *FileHandle) readFileBlockCache( + offset int64, buf []byte) (bytesRead int, err error) { + bytesRead = 0 + err = nil - if uint64(offset) >= fh.inode.Attributes.Size { + if offset < 0 || uint64(offset) >= fh.inode.Attributes.Size { // nothing to read if fh.inode.Invalid { err = fuse.ENOENT @@ -517,35 +535,102 @@ func (fh *FileHandle) readFile(offset int64, buf []byte) (bytesRead int, err err return } - fs := fh.inode.fs - - if fh.poolHandle == nil { - fh.poolHandle = fs.bufferPool + bc := fh.inode.fs.blockCache + cache, cacheOff := bc.Get(fh, uint64(offset)) + if cache.Err != nil { + err = cache.Err + fuseLog.Errorf("readFileBlockCache (%s: %v(+%v)/%v): %v", + *fh.inode.FullName(), offset, len(buf), fh.inode.Attributes.Size, + err) + return } - if fh.readBufOffset != offset { - // XXX out of order read, maybe disable prefetching - fh.inode.logFuse("out of order read", offset, fh.readBufOffset) + cachedData := cache.Data[offset-int64(cacheOff):] + bytesRead = MinInt(len(cachedData), len(buf)) + assert(bytesRead > 0, "zero bytes read") + copy(buf[:bytesRead], cachedData[:bytesRead]) - fh.readBufOffset = offset + if AbsInt64(offset-fh.blockCacheLastOffset) < int64(bc.blockSize)*2 { + fh.seqReadAmount += uint64(bytesRead) + if fh.seqReadAmount >= uint64(READAHEAD_CHUNK) { + if !fh.blockCacheReadAheadEnabled { + fh.blockCacheReadAheadEnabled = true + fuseLog.Infof("enable block cache read ahead for %s", + *fh.inode.FullName()) + } + } + } else { fh.seqReadAmount = 0 - if fh.reader != nil { - fh.reader.Close() - fh.reader = nil + if fh.blockCacheReadAheadEnabled { + fh.blockCacheReadAheadEnabled = false + fuseLog.Infof("disable block cache read ahead for %s", + *fh.inode.FullName()) } + } + fh.blockCacheLastOffset = offset + int64(bytesRead) - if fh.buffers != nil { - // we misdetected - fh.numOOORead++ + if fh.blockCacheReadAheadEnabled { + fh.startBlockCacheReadAhead(uint64(offset)) + } + + return +} + +func (fh *FileHandle) startBlockCacheReadAhead(offset uint64) { + CHUNK := uint64(READAHEAD_CHUNK) + offset = (offset + CHUNK - 1) / CHUNK * CHUNK + maxOffset := MinUInt64( + fh.inode.Attributes.Size, offset+uint64(MAX_READAHEAD)) + bc := fh.inode.fs.blockCache + for offset+CHUNK <= maxOffset { + if !bc.StartReadAhead(fh, offset, CHUNK) { + return } + offset += CHUNK + } +} +// close all readahead buffers and clear the buffer varaible +func (fh *FileHandle) closeAllBuffers() { + if fh.buffers != nil { for _, b := range fh.buffers { b.buf.Close() } fh.buffers = nil } +} + +func (fh *FileHandle) readFile(offset int64, buf []byte) (bytesRead int, err error) { + fs := fh.inode.fs + + if fs.blockCache != nil { + return fh.readFileBlockCache(offset, buf) + } + + if fs.flags.DebugFuse { + defer func() { + fh.inode.logFuse("< readFile", bytesRead, err) + }() + } + + if uint64(offset) >= fh.inode.Attributes.Size { + // nothing to read + if fh.inode.Invalid { + err = fuse.ENOENT + } else if fh.inode.KnownSize == nil { + err = io.EOF + } else { + err = io.EOF + } + return + } - if !fs.flags.Cheap && fh.seqReadAmount >= uint64(READAHEAD_CHUNK) && fh.numOOORead < 3 { + if fh.poolHandle == nil { + fh.poolHandle = fs.bufferPool + } + + if !fs.flags.Cheap && fh.seqReadAmount >= uint64(READAHEAD_CHUNK) && + fh.numOOOPrefetch < 3 { if fh.reader != nil { fh.inode.logFuse("cutover to the parallel algorithm") fh.reader.Close() @@ -558,26 +643,19 @@ func (fh *FileHandle) readFile(offset int64, buf []byte) (bytesRead int, err err return } else { // fall back to read serially - fh.inode.logFuse("not enough memory, fallback to serial read") + fuseLog.Errorf("read ahead failed: %v; fallback to serial read", err) fh.seqReadAmount = 0 - for _, b := range fh.buffers { - b.buf.Close() - } - fh.buffers = nil + fh.closeAllBuffers() } } - bytesRead, err = fh.readFromStream(offset, buf) - + bytesRead, err = fh.readFromUpstream(offset, buf) return } func (fh *FileHandle) Release() { // read buffers - for _, b := range fh.buffers { - b.buf.Close() - } - fh.buffers = nil + fh.closeAllBuffers() if fh.reader != nil { fh.reader.Close() @@ -604,10 +682,18 @@ func (fh *FileHandle) Release() { } } -func (fh *FileHandle) readFromStream(offset int64, buf []byte) (bytesRead int, err error) { +// read from upstream, maintaining fh.reader related states +func (fh *FileHandle) readFromUpstream( + offset int64, buf []byte) (bytesRead int, err error) { + defer func() { if fh.inode.fs.flags.DebugFuse { - fh.inode.logFuse("< readFromStream", bytesRead) + fh.inode.logFuse("< readFromUpstream", bytesRead) + } + + if bytesRead > 0 { + fh.readBufOffset += int64(bytesRead) + fh.seqReadAmount += uint64(bytesRead) } }() @@ -616,6 +702,22 @@ func (fh *FileHandle) readFromStream(offset int64, buf []byte) (bytesRead int, e return } + if fh.readBufOffset != offset && fh.reader != nil { + // XXX out of order read, maybe disable prefetching + fh.inode.logFuse("out of order read", offset, fh.readBufOffset) + + fh.reader.Close() + fh.reader = nil + + fh.numOOOReadReq++ + + if fh.buffers != nil { + // we misdetected + fh.numOOOPrefetch++ + fh.closeAllBuffers() + } + } + if fh.reader == nil { resp, err := fh.cloud.GetBlob(&GetBlobInput{ Key: fh.key, @@ -626,6 +728,8 @@ func (fh *FileHandle) readFromStream(offset int64, buf []byte) (bytesRead int, e } fh.reader = resp.Body + fh.readBufOffset = offset + fh.seqReadAmount = 0 } bytesRead, err = fh.reader.Read(buf) diff --git a/internal/flags.go b/internal/flags.go index 0cea21f4..4ffb25f8 100644 --- a/internal/flags.go +++ b/internal/flags.go @@ -221,6 +221,21 @@ func NewApp() (app *cli.App) { Usage: "Reduce S3 operation costs at the expense of some performance (default: off)", }, + cli.BoolFlag{ + Name: "block-read-cache", + Usage: "Enable block read cache, which aggressively reads ahead based on blocks", + }, + cli.Uint64Flag{ + Name: "block-read-cache-size", + Usage: "Size of each block of block read cache, in bytes", + Value: 1024 * 256, + }, + cli.Float64Flag{ + Name: "block-read-cache-mem-ratio", + Usage: "Ratio of total RAM to be reserved for block read cache", + Value: 0.1, + }, + cli.BoolFlag{ Name: "no-implicit-dir", Usage: "Assume all directory objects (\"dir/\") exist (default: off)", @@ -276,7 +291,9 @@ func NewApp() (app *cli.App) { flagCategories[f] = "aws" } - for _, f := range []string{"cheap", "no-implicit-dir", "stat-cache-ttl", "type-cache-ttl", "http-timeout"} { + for _, f := range []string{"cheap", "block-read-cache", + "block-read-cache-size", "block-read-cache-mem-ratio", "no-implicit-dir", + "stat-cache-ttl", "type-cache-ttl", "http-timeout"} { flagCategories[f] = "tuning" } @@ -328,11 +345,14 @@ func PopulateFlags(c *cli.Context) (ret *FlagStorage) { Gid: uint32(c.Int("gid")), // Tuning, - Cheap: c.Bool("cheap"), - ExplicitDir: c.Bool("no-implicit-dir"), - StatCacheTTL: c.Duration("stat-cache-ttl"), - TypeCacheTTL: c.Duration("type-cache-ttl"), - HTTPTimeout: c.Duration("http-timeout"), + Cheap: c.Bool("cheap"), + BlockReadCache: c.Bool("block-read-cache"), + BlockReadCacheSize: c.Uint64("block-read-cache-size"), + BlockReadCacheMemRatio: c.Float64("block-read-cache-mem-ratio"), + ExplicitDir: c.Bool("no-implicit-dir"), + StatCacheTTL: c.Duration("stat-cache-ttl"), + TypeCacheTTL: c.Duration("type-cache-ttl"), + HTTPTimeout: c.Duration("http-timeout"), // Common Backend Config Endpoint: c.String("endpoint"), diff --git a/internal/goofys.go b/internal/goofys.go index 4f4bd76f..e35e68d1 100644 --- a/internal/goofys.go +++ b/internal/goofys.go @@ -92,6 +92,8 @@ type Goofys struct { restorers *Ticket forgotCnt uint32 + + blockCache *BlockCache } var s3Log = GetLogger("s3") @@ -178,6 +180,10 @@ func newGoofys(ctx context.Context, bucket string, flags *FlagStorage, umask: 0122, } + if flags.BlockReadCache { + fs.blockCache = NewBlockCache(flags) + } + var prefix string colon := strings.Index(bucket, ":") if colon != -1 { diff --git a/internal/utils.go b/internal/utils.go index 51e07800..3c492bd6 100644 --- a/internal/utils.go +++ b/internal/utils.go @@ -26,6 +26,17 @@ import ( var TIME_MAX = time.Unix(1<<63-62135596801, 999999999) +func DivUpUint64(a, b uint64) uint64 { + return (a + b - 1) / b +} + +func AbsInt64(x int64) int64 { + if x < 0 { + return -x + } + return x +} + func MaxInt(a, b int) int { if a > b { return a @@ -245,4 +256,4 @@ func ConvertBytesToIEC(size int64) string { exp += 1 } return fmt.Sprintf("%.1f %ciB", curSize, "KMGTPEZY"[exp-1]) -} \ No newline at end of file +} diff --git a/main.go b/main.go index 112b8595..6e9a3f66 100644 --- a/main.go +++ b/main.go @@ -134,7 +134,7 @@ func massageArg0() { var Version = "use `make build' to fill version hash correctly" func main() { - VersionNumber = "0.24.0" + VersionNumber = "0.24.0-blkread.2" VersionHash = Version massagePath() diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 00000000..55d544b1 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1,2 @@ +/test_parallel_read +/output diff --git a/test/mount_parallel_read.sh b/test/mount_parallel_read.sh new file mode 100755 index 00000000..ccba2081 --- /dev/null +++ b/test/mount_parallel_read.sh @@ -0,0 +1,7 @@ +#!/bin/bash -e + +echo 'building ...' +go build + +echo 'Starting the server ...' +exec ./goofys --endpoint=http://127.0.0.1:9090 -f "$@" test mnt diff --git a/test/prepare_parallel_read.sh b/test/prepare_parallel_read.sh new file mode 100755 index 00000000..fb9e2ba0 --- /dev/null +++ b/test/prepare_parallel_read.sh @@ -0,0 +1,10 @@ +#!/bin/bash -e + +mkdir -p /tmp/s3proxy/test +for i in f1 f2 f3 f4; do + of=/tmp/s3proxy/test/$i + [ -f $of ] || dd if=/dev/urandom of=$of bs=1M count=1024 +done +of=/tmp/s3proxy/test/small +[ -f $of ] || dd if=/dev/urandom of=$of bs=1 count=1234513 +./s3proxy.sh --properties $(dirname $0)/test_parallel_read.properties diff --git a/test/run_parallel_read.sh b/test/run_parallel_read.sh new file mode 100755 index 00000000..79fbbd50 --- /dev/null +++ b/test/run_parallel_read.sh @@ -0,0 +1,25 @@ +#!/bin/bash -e + +set -u -e + +cd "$(dirname "$0")"/.. +rm -rf test/output +mkdir test/output +args='' + +files="f1 f2 f3 f4" + +for i in $files; do + args="$args mnt/$i test/output/$i" +done + +./test/test_parallel_read $args + +files="$files small" + +cp mnt/small test/output/small + +for i in $files; do + echo "checking $i" + cmp /tmp/s3proxy/test/$i test/output/$i +done diff --git a/test/test_parallel_read.cpp b/test/test_parallel_read.cpp new file mode 100644 index 00000000..f597df48 --- /dev/null +++ b/test/test_parallel_read.cpp @@ -0,0 +1,148 @@ +// test parallel read from multiple threads on the same file; mainly used for +// testing and benchmarking block read cache + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include + +constexpr int NR_WORKERS = 4, NR_PARTS = 1024 * 1024 * 2; + +class ParallelReader { + std::string m_path; + std::unique_ptr m_buf; + std::vector> m_parts; + std::atomic_uint32_t m_next_part; + size_t m_file_size; + + static size_t get_file_size(const char* fpath) { + struct stat stat_buf; + int rc = stat(fpath, &stat_buf); + assert(rc == 0); + return stat_buf.st_size; + } + + // generate random m_parts + void gen_parts(std::default_random_engine& rng, size_t tot_size) { + // O(n) algorithm to permute objects and splits to generate a random + // partition + + std::uniform_real_distribution uniform{0.0, 1.0}; + size_t this_part = 1, + nr_parts = std::min(NR_PARTS, tot_size / 20), + nr_split = nr_parts - 1, nr_obj = tot_size - nr_split - 1; + + while (nr_split) { + if (uniform(rng) <= + static_cast(nr_obj) / + static_cast(nr_obj + nr_split)) { + ++this_part; + --nr_obj; + } else { + m_parts.emplace_back(0, this_part); + this_part = 1; + --nr_split; + } + } + m_parts.emplace_back(0, this_part + nr_obj); + assert(m_parts.size() == nr_parts); + + size_t off = 0; + for (size_t i = 0; i < nr_parts; ++i) { + m_parts[i].first = off; + assert(m_parts[i].second); + off += m_parts[i].second; + } + assert(off == tot_size); + + std::shuffle(m_parts.begin(), m_parts.end(), rng); + } + + void worker() { + int fd = open(m_path.c_str(), O_RDONLY); + assert(fd > 0); + for (;;) { + size_t job = m_next_part.fetch_add(1, std::memory_order_relaxed); + if (job >= m_parts.size()) { + break; + } + size_t off, size; + std::tie(off, size) = m_parts[job]; + while (size > 0) { + ssize_t r = pread(fd, m_buf.get() + off, size, off); + assert(r > 0); + size -= r; + off += r; + } + } + close(fd); + } + +public: + ParallelReader(std::default_random_engine& rng, std::string path) + : m_path{std::move(path)} { + size_t fsize = get_file_size(m_path.c_str()); + m_file_size = fsize; + gen_parts(rng, fsize); + m_buf.reset(new uint8_t[fsize]); + + std::vector workers; + for (int i = 0; i < NR_WORKERS; ++i) { + workers.emplace_back([this]() { worker(); }); + } + for (size_t done; (done = m_next_part.load(std::memory_order_relaxed)) < + m_parts.size();) { + printf("\rreading: %zu/%zu ", done, m_parts.size()); + fflush(stdout); + usleep(500'000); + } + printf("\n"); + for (auto& i : workers) { + i.join(); + } + } + + void write(const char* path) { + FILE* fout = fopen(path, "wb"); + assert(fout); + size_t w = fwrite(m_buf.get(), 1, m_file_size, fout); + assert(w == m_file_size); + fclose(fout); + } +}; + +int main(int argc, const char* const argv[]) { + if (argc == 1 || argc % 2 != 1) { + fprintf(stderr, "usage: %s ...\n", argv[0]); + return 1; + } + + std::vector workers; + std::random_device rd; + for (int i = 1; i < argc; i += 2) { + auto fn = [inp = argv[i], out = argv[i + 1], seed = rd()]() { + std::default_random_engine rng{seed}; + ParallelReader r{rng, inp}; + r.write(out); + }; + workers.emplace_back(fn); + } + + for (auto& i : workers) { + i.join(); + } +} + +// vim: syntax=cpp.doxygen foldmethod=marker foldmarker=f{{{,f}}} diff --git a/test/test_parallel_read.properties b/test/test_parallel_read.properties new file mode 100644 index 00000000..ac622f27 --- /dev/null +++ b/test/test_parallel_read.properties @@ -0,0 +1,4 @@ +s3proxy.authorization=none +s3proxy.endpoint=http://127.0.0.1:9090 +jclouds.provider=filesystem +jclouds.filesystem.basedir=/tmp/s3proxy