-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathCopy-Directory.ps1
More file actions
1047 lines (898 loc) · 45.4 KB
/
Copy pathCopy-Directory.ps1
File metadata and controls
1047 lines (898 loc) · 45.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
function Copy-Directory
{
<#
.SYNOPSIS
Copies a directory with optional recursion, directory and file exclusions, and parallel processing.
.DESCRIPTION
Copies files from a source path to a destination path, optionally recursing into
subdirectories. Provides the ability to exclude specific directories (e.g., .git,
node_modules, bin, obj) and files (e.g., *.log, .DS_Store) from the copy operation.
Supports multi-threaded copying for improved performance with large directory trees.
Includes safety checks to prevent recursive self-copy scenarios (for example, copying
a directory into itself or into one of its own subdirectories).
For very large trees, you can opt in to OS-native copy tools (robocopy on Windows,
rsync on macOS/Linux) using -UseNativeTools. This is cross-platform compatible
with PowerShell 5.1+ and PowerShell Core 6.2+.
.PARAMETER Source
The source directory path to copy from. Supports relative paths and tilde (~) expansion.
.PARAMETER Destination
The destination directory path to copy to. Will be created if it doesn't exist.
Supports relative paths and tilde (~) expansion.
Cannot be the same as Source. When -Recurse is used, Destination also cannot be
located inside Source.
.PARAMETER ExcludeDirectories
An array of directory names or wildcard patterns to exclude from the copy operation.
Exact names are matched case-insensitively (for example: .git, node_modules, bin, obj).
Wildcards are also supported (for example: cache-*, temp?, build[0-9]).
.PARAMETER ExcludeFiles
An array of file names or wildcard patterns to exclude from the copy operation.
Exact names are matched case-insensitively (for example: .DS_Store, Thumbs.db, appsettings.local.json).
Wildcards are also supported (for example: *.log, *.tmp, temp-*.json).
.PARAMETER UpdateMode
Specifies how to handle existing files at the destination.
Valid values are:
- Skip: Do not copy files if they already exist at the destination (default)
- Overwrite: Always overwrite existing files without prompting
- IfNewer: Only overwrite if the source file is newer than the destination file
- Prompt: Ask for confirmation for each existing file (not compatible with parallel mode)
.PARAMETER Recurse
When specified, copies subdirectories recursively. Without this switch, only files in the
root of the source directory are copied.
.PARAMETER ThrottleLimit
Specifies the maximum number of concurrent copy operations when using parallel processing.
Default is based on logical CPU count (min 2, max 32). Set to 1 to disable
parallel processing. Valid range is 1-32.
Note: Parallel processing is automatically disabled when UpdateMode is 'Prompt' since
user prompts cannot be handled across multiple threads.
For PowerShell 7+, uses ForEach-Object -Parallel for optimal performance.
For PowerShell 5.1/6.x, uses runspace pools for parallel execution.
.PARAMETER UseNativeTools
When specified, uses OS-native copy tools for large directory trees (robocopy on Windows,
rsync on macOS/Linux). Requires -Recurse and does not support UpdateMode 'Prompt'.
Output properties are limited to those supported by the native tool. Native-tool summaries
are best-effort for some counters.
For best performance, and large-scale copies, this option is HIGHLY recommended.
.PARAMETER WhatIf
Shows what would happen if the cmdlet runs without actually performing the copy operation.
.PARAMETER Confirm
Prompts for confirmation before copying files.
.EXAMPLE
PS > Copy-Directory -Source '.\MyProject' -Destination 'C:\Backup\MyProject' -ExcludeDirectories '.git', 'node_modules' -ExcludeFiles '*.log', '.DS_Store' -Recurse
Copies the MyProject directory to C:\Backup\MyProject, excluding '.git' and 'node_modules' directories plus log files and macOS metadata files.
.EXAMPLE
PS > Copy-Directory -Source 'C:\Dev\Project' -Destination 'D:\Archive\Project' -ExcludeDirectories 'bin', 'obj', '.vs' -UpdateMode Overwrite -Recurse
Copies the project directory excluding build artifacts, overwriting existing files without prompting.
.EXAMPLE
PS > Copy-Directory -Source '~/Documents/Code' -Destination '~/Backup/Code' -ExcludeDirectories '.git', 'dist', 'build' -Recurse
Copies the Code directory from Documents to Backup, excluding version control and build directories.
Uses tilde expansion which works cross-platform.
.EXAMPLE
PS > Copy-Directory -Source './app' -Destination './staging/app' -ExcludeDirectories '.git', '.github', 'node_modules', 'tests' -Recurse
PS > Compress-Archive -Path './staging/app/*' -DestinationPath './artifacts/app.zip' -Force
Prepares a clean deployable archive by copying only runtime assets before zipping for release.
.EXAMPLE
PS > Copy-Directory -Source 'C:\LargeProject' -Destination 'D:\Backup' -ThrottleLimit 8 -Recurse
Copies a large project using 8 parallel threads for faster copying.
.EXAMPLE
PS > Copy-Directory -Source '.\Project' -Destination '.\Backup' -ThrottleLimit 1 -Recurse
Copies the project using single-threaded mode (parallel processing disabled).
.EXAMPLE
PS > Copy-Directory -Source 'C:\LargeProject' -Destination 'D:\Backup' -Recurse -UseNativeTools -ThrottleLimit 16
Copies a large directory tree using OS-native tools (robocopy on Windows, rsync on macOS/Linux).
.OUTPUTS
System.Management.Automation.PSCustomObject
Returns an object with:
- TotalFiles
- TotalDirectories
- ExcludedDirectories
- FilesSkipped
- FilesOverwritten
- Duration
When -UseNativeTools is specified, the output only includes properties supported by the native tool...
robocopy:
- TotalFiles
- TotalDirectories
- FilesSkipped
- Duration
rsync:
- TotalFiles
- Duration
.NOTES
Parallel Processing:
- PowerShell 7+: Uses ForEach-Object -Parallel for native parallel processing
- PowerShell 5.1/6.x: Uses runspace pools for parallel execution
- Thread-safe counters using synchronized hashtables with Monitor locks
- Directory structure is created sequentially to ensure proper ordering
- Only file copy operations are parallelized
Native Tools (opt-in):
- Windows: robocopy
- macOS/Linux: rsync
- Requires -Recurse and does not support UpdateMode 'Prompt'
- UpdateMode mappings are best-effort and may not be exact
- ThrottleLimit maps to robocopy /MT on Windows and is ignored by rsync
- ExcludeDirectories and ExcludeFiles matching follows native tool behavior (case sensitivity may differ)
- FilesOverwritten counts are not available from native tools
Safety:
- Source and Destination cannot be the same directory
- When -Recurse is specified, Destination cannot be within Source
Author: Jon LaBelle
License: MIT
Source: https://github.com/jonlabelle/pwsh-profile/blob/main/Functions/Utilities/Copy-Directory.ps1
.LINK
https://github.com/jonlabelle/pwsh-profile/blob/main/Functions/Utilities/Copy-Directory.ps1
#>
[Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseSingularNouns', '')]
[CmdletBinding(SupportsShouldProcess)]
[OutputType([PSCustomObject])]
param(
[Parameter(Mandatory, Position = 0)]
[ValidateNotNullOrEmpty()]
[String]$Source,
[Parameter(Mandatory, Position = 1)]
[ValidateNotNullOrEmpty()]
[String]$Destination,
[Parameter(Position = 2)]
[ValidateNotNullOrEmpty()]
[String[]]$ExcludeDirectories = @(),
[Parameter()]
[ValidateNotNullOrEmpty()]
[String[]]$ExcludeFiles = @(),
[Parameter()]
[ValidateSet('Skip', 'Overwrite', 'IfNewer', 'Prompt')]
[String]$UpdateMode = 'Skip',
[Parameter()]
[Switch]$Recurse,
[Parameter()]
[ValidateRange(1, 32)]
[Int32]$ThrottleLimit = ([Math]::Min(32, [Math]::Max(2, [Environment]::ProcessorCount))),
[Parameter()]
[Switch]$UseNativeTools
)
begin
{
Write-Verbose 'Starting Copy-Directory'
# Detect platform once for path comparison and native tool selection.
$IsWindowsPlatform = $IsWindows -or $env:OS -eq 'Windows_NT'
$pathComparison = if ($IsWindowsPlatform) { [System.StringComparison]::OrdinalIgnoreCase } else { [System.StringComparison]::Ordinal }
$separatorChars = @([System.IO.Path]::DirectorySeparatorChar, [System.IO.Path]::AltDirectorySeparatorChar)
# Resolve paths to absolute paths (cross-platform compatible)
$Source = $PSCmdlet.SessionState.Path.GetUnresolvedProviderPathFromPSPath($Source)
$Destination = $PSCmdlet.SessionState.Path.GetUnresolvedProviderPathFromPSPath($Destination)
$Source = [System.IO.Path]::GetFullPath($Source)
$Destination = [System.IO.Path]::GetFullPath($Destination)
$sourceComparable = $Source.TrimEnd($separatorChars)
if ([String]::IsNullOrEmpty($sourceComparable))
{
$sourceComparable = [System.IO.Path]::DirectorySeparatorChar.ToString()
}
$destinationComparable = $Destination.TrimEnd($separatorChars)
if ([String]::IsNullOrEmpty($destinationComparable))
{
$destinationComparable = [System.IO.Path]::DirectorySeparatorChar.ToString()
}
Write-Verbose "Resolved source path: $Source"
Write-Verbose "Resolved destination path: $Destination"
# Validate parameter combinations
if ($UseNativeTools -and -not $Recurse)
{
throw 'The -UseNativeTools parameter requires -Recurse to be specified. Native copy tools (robocopy/rsync) only support recursive directory operations.'
}
if ($UseNativeTools -and $UpdateMode -eq 'Prompt')
{
throw "The -UseNativeTools parameter cannot be used with -UpdateMode 'Prompt'. Native copy tools do not support interactive prompts for file overwrites."
}
# Validate source exists
if (-not (Test-Path -Path $Source -PathType Container))
{
throw "Source directory does not exist: $Source"
}
if (Test-Path -Path $Destination -PathType Leaf)
{
throw "Destination path exists as a file. Specify a directory path instead: $Destination"
}
# Prevent recursive self-copy and source==destination scenarios.
if ([String]::Equals($sourceComparable, $destinationComparable, $pathComparison))
{
throw "Source and destination cannot be the same directory: $Source"
}
if ($Recurse.IsPresent)
{
$sourcePrefix = if ($sourceComparable.EndsWith([System.IO.Path]::DirectorySeparatorChar.ToString()))
{
$sourceComparable
}
else
{
$sourceComparable + [System.IO.Path]::DirectorySeparatorChar
}
if ($destinationComparable.StartsWith($sourcePrefix, $pathComparison))
{
throw "Destination cannot be inside the source directory when -Recurse is used. Source: $Source Destination: $Destination"
}
}
# Create destination if it doesn't exist
if (-not [System.IO.Directory]::Exists($Destination))
{
if ($PSCmdlet.ShouldProcess($Destination, 'Create destination directory'))
{
Write-Verbose "Creating destination directory: $Destination"
[System.IO.Directory]::CreateDirectory($Destination) | Out-Null
}
}
# Initialize thread-safe counters
# For PS7+ parallel mode, we use a synchronized hashtable that works across runspaces
# For PS5.1/6.x runspace pools, we use [ref] types with Interlocked operations
$script:Counters = [hashtable]::Synchronized(@{
FilesCopied = 0
DirectoriesCreated = 0
DirectoriesExcluded = 0
FilesSkipped = 0
FilesOverwritten = 0
})
$script:UsedNativeTools = $false
$script:NativeToolName = $null
# Use HashSet/List structures for fast, case-insensitive exclusion checks.
$SanitizedExcludeDirectories = [System.Collections.Generic.List[string]]::new()
$ExcludeSet = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
$ExcludeWildcardPatterns = [System.Collections.Generic.List[string]]::new()
foreach ($excludeDir in $ExcludeDirectories)
{
if ([String]::IsNullOrWhiteSpace($excludeDir))
{
continue
}
$SanitizedExcludeDirectories.Add($excludeDir)
if ($excludeDir.IndexOfAny([char[]]@('*', '?', '[')) -ge 0)
{
$ExcludeWildcardPatterns.Add($excludeDir)
continue
}
$null = $ExcludeSet.Add($excludeDir)
}
$SanitizedExcludeFiles = [System.Collections.Generic.List[string]]::new()
$ExcludeFileSet = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase)
$ExcludeFileWildcardPatterns = [System.Collections.Generic.List[string]]::new()
foreach ($excludeFile in $ExcludeFiles)
{
if ([String]::IsNullOrWhiteSpace($excludeFile))
{
continue
}
$SanitizedExcludeFiles.Add($excludeFile)
if ($excludeFile.IndexOfAny([char[]]@('*', '?', '[')) -ge 0)
{
$ExcludeFileWildcardPatterns.Add($excludeFile)
continue
}
$null = $ExcludeFileSet.Add($excludeFile)
}
Write-Verbose "Excluding directories: $($SanitizedExcludeDirectories -join ', ')"
Write-Verbose "Excluding files: $($SanitizedExcludeFiles -join ', ')"
Write-Verbose "UpdateMode: $UpdateMode"
Write-Verbose "ThrottleLimit: $ThrottleLimit"
Write-Verbose "UseNativeTools: $UseNativeTools"
# Check if parallel processing should be used
$UseParallel = $ThrottleLimit -gt 1 -and $UpdateMode -ne 'Prompt'
if ($UpdateMode -eq 'Prompt' -and $ThrottleLimit -gt 1)
{
Write-Warning "Parallel processing disabled: UpdateMode 'Prompt' requires sequential processing for user interaction."
$UseParallel = $false
}
$IncludeLastWriteTime = $UpdateMode -eq 'IfNewer'
# Detect PowerShell version for parallel implementation choice
$IsPowerShell7OrLater = $PSVersionTable.PSVersion.Major -ge 7
Write-Verbose "UseParallel: $UseParallel"
Write-Verbose "PowerShell 7+: $IsPowerShell7OrLater"
$StartTime = Get-Date
}
process
{
function Invoke-NativeDirectoryCopy
{
param(
[String]$SourcePath,
[String]$DestPath,
[String[]]$ExcludeDirs,
[String[]]$ExcludeFilePatterns,
[String]$Mode,
[Int32]$Throttle,
[Bool]$EnableRecurse,
[Bool]$IsWindowsPlatform,
[hashtable]$CountersRef
)
$nativeTool = $null
$nativeToolName = $null
$nativeArgs = @()
if ($IsWindowsPlatform)
{
$command = Get-Command -Name 'robocopy' -ErrorAction SilentlyContinue
if ($command)
{
$nativeTool = if ($command.Source) { $command.Source } else { $command.Path }
}
if (-not $nativeTool)
{
Write-Warning 'Native tool copy requested, but robocopy was not found. Falling back to PowerShell copy.'
return $false
}
$nativeToolName = 'robocopy'
$nativeArgs += $SourcePath
$nativeArgs += $DestPath
$nativeArgs += '/E' # Copy subdirectories, including empty ones
$nativeArgs += '/NJH' # No job header
$nativeArgs += '/NP' # No progress - don't display percentage copied
$nativeArgs += '/NDL' # No directory list - don't log directory names
$nativeArgs += '/NFL' # No file list - don't log file names
$nativeArgs += '/R:1' # Retry once on failed copies
$nativeArgs += '/W:1' # Wait 1 second between retries
$nativeArgs += '/FFT' # Use FAT file times (2-second granularity) when comparing timestamps
if ($Throttle -gt 1)
{
$nativeArgs += "/MT:$Throttle" # Multi-threaded copying with specified number of threads
}
switch ($Mode)
{
'Skip'
{
$nativeArgs += '/XC' # Exclude changed files
$nativeArgs += '/XN' # Exclude newer files
$nativeArgs += '/XO' # Exclude older files (effectively skip existing files)
}
'Overwrite'
{
$nativeArgs += '/IS' # Include same files (overwrite even if identical)
$nativeArgs += '/IT' # Include tweaked files (overwrite files with different attributes)
}
'IfNewer'
{
$nativeArgs += '/XO' # Exclude older files (only copy if source is newer)
}
}
if ($ExcludeDirs -and $ExcludeDirs.Count -gt 0)
{
$nativeArgs += '/XD' # Exclude directories
$nativeArgs += $ExcludeDirs
}
if ($ExcludeFilePatterns -and $ExcludeFilePatterns.Count -gt 0)
{
$nativeArgs += '/XF' # Exclude files
$nativeArgs += $ExcludeFilePatterns
}
}
else
{
$command = Get-Command -Name 'rsync' -ErrorAction SilentlyContinue
if ($command)
{
$nativeTool = if ($command.Source) { $command.Source } else { $command.Path }
}
if (-not $nativeTool)
{
Write-Warning 'Native tool copy requested, but rsync was not found. Falling back to PowerShell copy.'
return $false
}
$nativeToolName = 'rsync'
$nativeArgs += '-a' # Archive mode: preserve permissions, timestamps, symbolic links, etc.
$nativeArgs += '--stats' # Display file transfer statistics
switch ($Mode)
{
'Skip'
{
$nativeArgs += '--ignore-existing' # Skip files that already exist at destination
}
'Overwrite'
{
$nativeArgs += '--ignore-times' # Don't skip files that match in size and modification time
}
'IfNewer'
{
$nativeArgs += '-u' # Update: skip files that are newer on the receiver
}
}
if ($ExcludeDirs -and $ExcludeDirs.Count -gt 0)
{
foreach ($excludeDir in $ExcludeDirs)
{
if (-not [String]::IsNullOrWhiteSpace($excludeDir))
{
$nativeArgs += "--exclude=$excludeDir/" # Exclude directory pattern from transfer
}
}
}
if ($ExcludeFilePatterns -and $ExcludeFilePatterns.Count -gt 0)
{
foreach ($excludeFile in $ExcludeFilePatterns)
{
if (-not [String]::IsNullOrWhiteSpace($excludeFile))
{
$nativeArgs += "--exclude=$excludeFile"
}
}
}
# Ensure paths end with trailing slash for rsync directory sync behavior
$separatorChars = @([System.IO.Path]::DirectorySeparatorChar, [System.IO.Path]::AltDirectorySeparatorChar)
$sourceNormalized = $SourcePath.TrimEnd($separatorChars) + [System.IO.Path]::DirectorySeparatorChar
$destNormalized = $DestPath.TrimEnd($separatorChars) + [System.IO.Path]::DirectorySeparatorChar
$nativeArgs += $sourceNormalized
$nativeArgs += $destNormalized
}
Write-Verbose "Using native tool: $nativeToolName"
$script:UsedNativeTools = $true
$script:NativeToolName = $nativeToolName
if (-not $PSCmdlet.ShouldProcess($DestPath, "Copy directory from $SourcePath using $nativeToolName"))
{
return $true
}
try
{
$nativeOutput = & $nativeTool @nativeArgs 2>&1
}
catch
{
Write-Warning "Native tool copy failed to start: $($_.Exception.Message)"
$script:UsedNativeTools = $false
$script:NativeToolName = $null
return $false
}
$exitCode = $LASTEXITCODE
if ($IsWindowsPlatform)
{
if ($exitCode -ge 8)
{
Write-Warning "robocopy returned exit code $exitCode. See robocopy documentation for details."
}
foreach ($line in $nativeOutput)
{
if ($line -match '^\s*Dirs\s*:\s*(\d+)\s+(\d+)\s+(\d+)')
{
$CountersRef.DirectoriesCreated = [Int32]$matches[2]
}
elseif ($line -match '^\s*Files\s*:\s*(\d+)\s+(\d+)\s+(\d+)')
{
$CountersRef.FilesCopied = [Int32]$matches[2]
$CountersRef.FilesSkipped = [Int32]$matches[3]
}
}
}
else
{
if ($exitCode -ne 0)
{
Write-Warning "rsync returned exit code $exitCode. See rsync documentation for details."
}
foreach ($line in $nativeOutput)
{
if ($line -match '^Number of regular files transferred:\s*(\d+)')
{
$CountersRef.FilesCopied = [Int32]$matches[1]
}
elseif ($line -match '^Number of files transferred:\s*(\d+)')
{
$CountersRef.FilesCopied = [Int32]$matches[1]
}
}
}
return $true
}
function Test-IsExcludedEntryName
{
param(
[String]$EntryName,
[System.Collections.Generic.HashSet[string]]$ExactMatches,
[System.Collections.Generic.List[string]]$WildcardMatches
)
if ($ExactMatches.Contains($EntryName))
{
return $true
}
foreach ($wildcardPattern in $WildcardMatches)
{
if ($EntryName -like $wildcardPattern)
{
return $true
}
}
return $false
}
# Function to stream file operations while creating directories on the fly
function Get-CopyFileOperations
{
param(
[String]$SourcePath,
[String]$DestPath,
[System.Collections.Generic.HashSet[string]]$ExcludeSet,
[System.Collections.Generic.List[string]]$ExcludeWildcardPatterns,
[System.Collections.Generic.HashSet[string]]$ExcludeFileSet,
[System.Collections.Generic.List[string]]$ExcludeFileWildcardPatterns,
[Bool]$EnableRecurse,
[Bool]$IncludeLastWriteTime,
[hashtable]$CountersRef,
[ref]$FoundFiles
)
$directoriesToProcess = [System.Collections.Queue]::new()
$directoriesToProcess.Enqueue(@{ Source = $SourcePath; Dest = $DestPath })
while ($directoriesToProcess.Count -gt 0)
{
$current = $directoriesToProcess.Dequeue()
$currentSource = $current.Source
$currentDest = $current.Dest
try
{
$directoryInfo = [System.IO.DirectoryInfo]::new($currentSource)
foreach ($entry in $directoryInfo.EnumerateFileSystemInfos())
{
if ($entry -is [System.IO.DirectoryInfo])
{
$isExcludedDirectory = Test-IsExcludedEntryName -EntryName $entry.Name -ExactMatches $ExcludeSet -WildcardMatches $ExcludeWildcardPatterns
if ($isExcludedDirectory)
{
$CountersRef.DirectoriesExcluded++
continue
}
if (-not $EnableRecurse)
{
$CountersRef.DirectoriesExcluded++
continue
}
$destDirPath = [System.IO.Path]::Combine($currentDest, $entry.Name)
if (-not [System.IO.Directory]::Exists($destDirPath))
{
if ($PSCmdlet.ShouldProcess($destDirPath, 'Create directory'))
{
Write-Verbose "Creating directory: $destDirPath"
[System.IO.Directory]::CreateDirectory($destDirPath) | Out-Null
$CountersRef.DirectoriesCreated++
}
}
$directoriesToProcess.Enqueue(@{ Source = $entry.FullName; Dest = $destDirPath })
}
else
{
if (Test-IsExcludedEntryName -EntryName $entry.Name -ExactMatches $ExcludeFileSet -WildcardMatches $ExcludeFileWildcardPatterns)
{
continue
}
$FoundFiles.Value = $true
$destFilePath = [System.IO.Path]::Combine($currentDest, $entry.Name)
@{
SourcePath = $entry.FullName
DestPath = $destFilePath
LastWriteTime = if ($IncludeLastWriteTime) { $entry.LastWriteTime } else { $null }
}
}
}
}
catch
{
Write-Warning "Failed to access directory: $currentSource - $($_.Exception.Message)"
}
}
}
$usedNativeTools = $false
if ($UseNativeTools)
{
$usedNativeTools = Invoke-NativeDirectoryCopy -SourcePath $Source -DestPath $Destination -ExcludeDirs $SanitizedExcludeDirectories.ToArray() -ExcludeFilePatterns $SanitizedExcludeFiles.ToArray() -Mode $UpdateMode -Throttle $ThrottleLimit -EnableRecurse $Recurse.IsPresent -IsWindowsPlatform $IsWindowsPlatform -CountersRef $script:Counters
}
$hasFiles = $false
if (-not $usedNativeTools)
{
if (-not $UseParallel)
{
$copyHeaderWritten = $false
foreach ($fileOp in Get-CopyFileOperations -SourcePath $Source -DestPath $Destination -ExcludeSet $ExcludeSet -ExcludeWildcardPatterns $ExcludeWildcardPatterns -ExcludeFileSet $ExcludeFileSet -ExcludeFileWildcardPatterns $ExcludeFileWildcardPatterns -EnableRecurse $Recurse.IsPresent -IncludeLastWriteTime $IncludeLastWriteTime -CountersRef $script:Counters -FoundFiles ([ref]$hasFiles))
{
if (-not $copyHeaderWritten)
{
Write-Verbose 'Copying files sequentially...'
$copyHeaderWritten = $true
}
$shouldCopyFile = $true
if ([System.IO.File]::Exists($fileOp.DestPath))
{
switch ($UpdateMode)
{
'Skip'
{
Write-Verbose "Skipping existing file: $($fileOp.DestPath)"
$script:Counters.FilesSkipped++
$shouldCopyFile = $false
}
'Overwrite'
{
Write-Verbose "Overwriting existing file: $($fileOp.DestPath)"
$script:Counters.FilesOverwritten++
}
'IfNewer'
{
if ($fileOp.LastWriteTime -gt [System.IO.File]::GetLastWriteTime($fileOp.DestPath))
{
Write-Verbose "Overwriting with newer file: $($fileOp.DestPath)"
$script:Counters.FilesOverwritten++
}
else
{
Write-Verbose "Destination file is up-to-date, skipping: $($fileOp.DestPath)"
$script:Counters.FilesSkipped++
$shouldCopyFile = $false
}
}
'Prompt'
{
if (-not $PSCmdlet.ShouldProcess($fileOp.DestPath, "Overwrite existing file from $($fileOp.SourcePath)"))
{
Write-Verbose "User declined to overwrite: $($fileOp.DestPath)"
$script:Counters.FilesSkipped++
$shouldCopyFile = $false
}
else
{
Write-Verbose "User confirmed overwriting: $($fileOp.DestPath)"
$script:Counters.FilesOverwritten++
}
}
}
}
if ($shouldCopyFile -and $PSCmdlet.ShouldProcess($fileOp.DestPath, "Copy file from $($fileOp.SourcePath)"))
{
try
{
Write-Verbose "Copying file: $($fileOp.SourcePath) -> $($fileOp.DestPath)"
Copy-Item -Path $fileOp.SourcePath -Destination $fileOp.DestPath -Force -ErrorAction Stop
$script:Counters.FilesCopied++
}
catch
{
Write-Warning "Failed to copy file: $($fileOp.SourcePath) - $($_.Exception.Message)"
}
}
}
}
elseif ($IsPowerShell7OrLater)
{
# PowerShell 7+ parallel mode using ForEach-Object -Parallel
$copyHeaderWritten = $false
# Use synchronized hashtable for thread-safe counter access across parallel runspaces
$countersRef = $script:Counters
$updateModeValue = $UpdateMode
$whatIfEnabled = $WhatIfPreference
Get-CopyFileOperations -SourcePath $Source -DestPath $Destination -ExcludeSet $ExcludeSet -ExcludeWildcardPatterns $ExcludeWildcardPatterns -ExcludeFileSet $ExcludeFileSet -ExcludeFileWildcardPatterns $ExcludeFileWildcardPatterns -EnableRecurse $Recurse.IsPresent -IncludeLastWriteTime $IncludeLastWriteTime -CountersRef $script:Counters -FoundFiles ([ref]$hasFiles) | ForEach-Object {
if (-not $copyHeaderWritten)
{
Write-Verbose "Copying files in parallel (ThrottleLimit: $ThrottleLimit, PowerShell 7+ mode)..."
$copyHeaderWritten = $true
}
$_
} | ForEach-Object -ThrottleLimit $ThrottleLimit -Parallel {
$fileOp = $_
$localMode = $using:updateModeValue
$localCounters = $using:countersRef
$whatIfEnabled = $using:whatIfEnabled
$shouldCopyFile = $true
$destExists = [System.IO.File]::Exists($fileOp.DestPath)
if ($destExists)
{
switch ($localMode)
{
'Skip'
{
# Thread-safe increment on synchronized hashtable
[System.Threading.Monitor]::Enter($localCounters.SyncRoot)
try { $localCounters.FilesSkipped++ }
finally { [System.Threading.Monitor]::Exit($localCounters.SyncRoot) }
$shouldCopyFile = $false
}
'Overwrite'
{
[System.Threading.Monitor]::Enter($localCounters.SyncRoot)
try { $localCounters.FilesOverwritten++ }
finally { [System.Threading.Monitor]::Exit($localCounters.SyncRoot) }
}
'IfNewer'
{
if ($fileOp.LastWriteTime -gt [System.IO.File]::GetLastWriteTime($fileOp.DestPath))
{
[System.Threading.Monitor]::Enter($localCounters.SyncRoot)
try { $localCounters.FilesOverwritten++ }
finally { [System.Threading.Monitor]::Exit($localCounters.SyncRoot) }
}
else
{
[System.Threading.Monitor]::Enter($localCounters.SyncRoot)
try { $localCounters.FilesSkipped++ }
finally { [System.Threading.Monitor]::Exit($localCounters.SyncRoot) }
$shouldCopyFile = $false
}
}
}
}
if ($shouldCopyFile -and -not $whatIfEnabled)
{
try
{
Copy-Item -Path $fileOp.SourcePath -Destination $fileOp.DestPath -Force -ErrorAction Stop
[System.Threading.Monitor]::Enter($localCounters.SyncRoot)
try { $localCounters.FilesCopied++ }
finally { [System.Threading.Monitor]::Exit($localCounters.SyncRoot) }
}
catch
{
Write-Warning "Failed to copy file: $($fileOp.SourcePath) - $($_.Exception.Message)"
}
}
}
}
else
{
# PowerShell 5.1/6.x parallel mode using runspace pools
$copyHeaderWritten = $false
# Create runspace pool
$runspacePool = [System.Management.Automation.Runspaces.RunspaceFactory]::CreateRunspacePool(1, $ThrottleLimit)
$runspacePool.Open()
$runspaces = [System.Collections.ArrayList]::new()
# Use synchronized hashtable for thread-safe counter access
$countersRef = $script:Counters
$updateModeValue = $UpdateMode
$whatIfEnabled = $WhatIfPreference
$queueCapacity = [Math]::Max(32, $ThrottleLimit * 8)
$workQueue = [System.Collections.Concurrent.BlockingCollection[hashtable]]::new($queueCapacity)
for ($workerIndex = 0; $workerIndex -lt $ThrottleLimit; $workerIndex++)
{
$powershell = [System.Management.Automation.PowerShell]::Create()
$powershell.RunspacePool = $runspacePool
$null = $powershell.AddScript({
param(
[System.Collections.Concurrent.BlockingCollection[hashtable]]$WorkQueue,
[String]$Mode,
[hashtable]$Counters,
[Bool]$WhatIfEnabled
)
foreach ($fileOp in $WorkQueue.GetConsumingEnumerable())
{
$shouldCopyFile = $true
$destExists = [System.IO.File]::Exists($fileOp.DestPath)
if ($destExists)
{
switch ($Mode)
{
'Skip'
{
[System.Threading.Monitor]::Enter($Counters.SyncRoot)
try { $Counters.FilesSkipped++ }
finally { [System.Threading.Monitor]::Exit($Counters.SyncRoot) }
$shouldCopyFile = $false
}
'Overwrite'
{
[System.Threading.Monitor]::Enter($Counters.SyncRoot)
try { $Counters.FilesOverwritten++ }
finally { [System.Threading.Monitor]::Exit($Counters.SyncRoot) }
}
'IfNewer'
{
if ($fileOp.LastWriteTime -gt [System.IO.File]::GetLastWriteTime($fileOp.DestPath))
{
[System.Threading.Monitor]::Enter($Counters.SyncRoot)
try { $Counters.FilesOverwritten++ }
finally { [System.Threading.Monitor]::Exit($Counters.SyncRoot) }
}
else
{
[System.Threading.Monitor]::Enter($Counters.SyncRoot)
try { $Counters.FilesSkipped++ }
finally { [System.Threading.Monitor]::Exit($Counters.SyncRoot) }
$shouldCopyFile = $false
}
}
}
}
if ($shouldCopyFile -and -not $WhatIfEnabled)
{
try
{
Copy-Item -Path $fileOp.SourcePath -Destination $fileOp.DestPath -Force -ErrorAction Stop
[System.Threading.Monitor]::Enter($Counters.SyncRoot)
try { $Counters.FilesCopied++ }
finally { [System.Threading.Monitor]::Exit($Counters.SyncRoot) }
}
catch
{
Write-Warning "Failed to copy file: $($fileOp.SourcePath) - $($_.Exception.Message)"
}
}
}
})
$null = $powershell.AddParameter('WorkQueue', $workQueue)
$null = $powershell.AddParameter('Mode', $updateModeValue)
$null = $powershell.AddParameter('Counters', $countersRef)
$null = $powershell.AddParameter('WhatIfEnabled', $whatIfEnabled)
$handle = $powershell.BeginInvoke()
$null = $runspaces.Add(@{
PowerShell = $powershell
Handle = $handle
})
}
try
{
foreach ($fileOp in Get-CopyFileOperations -SourcePath $Source -DestPath $Destination -ExcludeSet $ExcludeSet -ExcludeWildcardPatterns $ExcludeWildcardPatterns -ExcludeFileSet $ExcludeFileSet -ExcludeFileWildcardPatterns $ExcludeFileWildcardPatterns -EnableRecurse $Recurse.IsPresent -IncludeLastWriteTime $IncludeLastWriteTime -CountersRef $script:Counters -FoundFiles ([ref]$hasFiles))
{
if (-not $copyHeaderWritten)
{
Write-Verbose "Copying files in parallel (ThrottleLimit: $ThrottleLimit, Runspace pool mode)..."
$copyHeaderWritten = $true
}
$workQueue.Add($fileOp)
}
}
finally
{
$workQueue.CompleteAdding()
}
# Wait for all runspaces to complete
foreach ($runspace in $runspaces)
{
try
{
$runspace.PowerShell.EndInvoke($runspace.Handle)
}
catch
{
Write-Warning "Runspace error: $($_.Exception.Message)"
}
finally
{
$runspace.PowerShell.Dispose()
}
}
# Clean up runspace pool
$runspacePool.Close()
$runspacePool.Dispose()
}
if (-not $hasFiles)
{
Write-Verbose 'No files to copy'
}
}
}
end
{