|
1 | | -# Azpype 🚀 |
| 1 | +# Azpype 🚀 [beta] |
2 | 2 |
|
3 | 3 | A Python wrapper for AzCopy that feels native and gets out of your way. |
4 | 4 |
|
@@ -52,41 +52,83 @@ Copy( |
52 | 52 |
|
53 | 53 | ### Working with Return Values |
54 | 54 |
|
55 | | -The `execute()` method returns a tuple of (exit_code, output): |
| 55 | +The `execute()` method returns an `AzCopyStdoutParser` object with parsed attributes - no manual string parsing needed! |
56 | 56 |
|
57 | 57 | ```python |
58 | | -# Capture and inspect the results |
59 | | -exit_code, output = Copy( |
| 58 | +# Execute returns a parsed object with useful attributes |
| 59 | +result = Copy( |
60 | 60 | source="./data", |
61 | 61 | destination="https://myaccount.blob.core.windows.net/mycontainer/" |
62 | 62 | ).execute() |
63 | 63 |
|
64 | | -if exit_code == 0: |
| 64 | +# Access structured data directly |
| 65 | +print(f"Job ID: {result.job_id}") |
| 66 | +print(f"Files transferred: {result.number_of_file_transfers_completed}") |
| 67 | +print(f"Files skipped: {result.number_of_file_transfers_skipped}") |
| 68 | +print(f"Bytes transferred: {result.total_bytes_transferred}") |
| 69 | +print(f"Elapsed time: {result.elapsed_time} minutes") |
| 70 | +print(f"Final status: {result.final_job_status}") |
| 71 | + |
| 72 | +# Use exit code for flow control |
| 73 | +if result.exit_code == 0: |
65 | 74 | print("Transfer successful!") |
66 | | - # Parse the output for job details |
67 | | - if "Job" in output: |
68 | | - job_id = output.split("Job ")[1].split(" ")[0] |
69 | | - print(f"Job ID: {job_id}") |
70 | 75 | else: |
71 | | - print(f"Transfer failed with code: {exit_code}") |
| 76 | + print(f"Transfer failed: {result.stdout}") |
| 77 | +``` |
| 78 | + |
| 79 | +#### Available Attributes |
| 80 | + |
| 81 | +The parser automatically extracts these attributes from AzCopy output: |
| 82 | + |
| 83 | +| Attribute | Type | Description | |
| 84 | +|-----------|------|-------------| |
| 85 | +| `exit_code` | int | Command exit code (0 = success) | |
| 86 | +| `job_id` | str | Unique job identifier for resuming | |
| 87 | +| `elapsed_time` | float | Transfer duration in minutes | |
| 88 | +| `final_job_status` | str | Status like "Completed", "CompletedWithSkipped", "Failed" | |
| 89 | +| `number_of_file_transfers` | int | Total files attempted | |
| 90 | +| `number_of_file_transfers_completed` | int | Successfully transferred files | |
| 91 | +| `number_of_file_transfers_skipped` | int | Files skipped (already exist, etc.) | |
| 92 | +| `number_of_file_transfers_failed` | int | Failed file transfers | |
| 93 | +| `total_bytes_transferred` | int | Total data transferred in bytes | |
| 94 | +| `total_number_of_transfers` | int | Total transfer operations | |
| 95 | +| `stdout` | str | Raw command output if needed | |
| 96 | +| `raw_stdout` | str | Unprocessed output with ANSI codes | |
| 97 | + |
| 98 | +#### Real-World Example: Pipeline Integration |
| 99 | + |
| 100 | +```python |
| 101 | +def smart_sync_with_monitoring(local_path, remote_path): |
| 102 | + """ |
| 103 | + Sync data and monitor transfer metrics |
| 104 | + """ |
| 105 | + result = Copy( |
| 106 | + source=local_path, |
| 107 | + destination=remote_path, |
| 108 | + overwrite="ifSourceNewer", |
| 109 | + recursive=True |
| 110 | + ).execute() |
72 | 111 |
|
73 | | -# Use in your data pipeline |
74 | | -def process_and_upload(data_path): |
75 | | - # Pre-process data |
76 | | - prepare_data(data_path) |
| 112 | + # Make decisions based on parsed results |
| 113 | + if result.exit_code != 0: |
| 114 | + raise Exception(f"Transfer failed: {result.final_job_status}") |
77 | 115 |
|
78 | | - # Upload with error handling |
79 | | - exit_code, output = Copy( |
80 | | - source=data_path, |
81 | | - destination="https://myaccount.blob.core.windows.net/processed/" |
82 | | - ).execute() |
| 116 | + if result.number_of_file_transfers_failed > 0: |
| 117 | + print(f"Warning: {result.number_of_file_transfers_failed} files failed") |
| 118 | + # Could trigger retry logic here |
| 119 | + |
| 120 | + if result.number_of_file_transfers_skipped == result.number_of_file_transfers: |
| 121 | + print("All files already up-to-date") |
| 122 | + return "NO_CHANGES" |
| 123 | + |
| 124 | + # Report transfer metrics |
| 125 | + gb_transferred = result.total_bytes_transferred / (1024**3) |
| 126 | + transfer_rate = gb_transferred / (result.elapsed_time / 60) # GB/hour |
83 | 127 |
|
84 | | - if exit_code != 0: |
85 | | - raise Exception(f"Upload failed: {output}") |
| 128 | + print(f"Transferred {gb_transferred:.2f} GB at {transfer_rate:.2f} GB/hour") |
| 129 | + print(f"Completed: {result.number_of_file_transfers_completed} files") |
86 | 130 |
|
87 | | - # Continue with post-processing |
88 | | - cleanup_local_data(data_path) |
89 | | - return output |
| 131 | + return result.job_id # Return for potential resume operations |
90 | 132 | ``` |
91 | 133 |
|
92 | 134 | ## Authentication |
|
0 commit comments