Skip to content

Commit

Permalink
[card] current.card.copy
Browse files Browse the repository at this point in the history
- cli command to copy cards from other tasks
- Exposing the interface in `current.card` so that cards can be copied over during runtime. This also allows users the flexibility to select which cards to copy during runtime.
- useful for the cases when users need to showcase the model card in downstream steps without having to re-render the card everytime.

- Example flow:
```python
from metaflow import FlowSpec, step, card, current

class CardCopyTestFlow(FlowSpec):
    """
    A flow that demonstrates the card copy functionality.
    """

    @card(type="blank")
    @step
    def start(self):
        """
        Create a card in the start step.
        """
        from metaflow.cards import Markdown
        current.card.append(Markdown("# Original Card"))
        current.card.append(Markdown("This card was created in the start step."))
        self.pathspec = current.pathspec
        self.next(self.copy_card)

    @card(type="blank")
    @step
    def copy_card(self):
        """
        Copy the card from the start step.
        """
        from metaflow.cards import Markdown

        # First, create a new card for this step
        current.card.append(Markdown("# Destination Card"))
        current.card.append(Markdown("This card was created in the copy_card step."))

        # Now, copy the card from the start step
        # Get the pathspec for the start step

        # Copy the card from the start step
        success = current.card.copy(self.pathspec)

        if success:
            current.card.append(Markdown("## Card Copy Success"))
            current.card.append(Markdown(f"Successfully copied card from {self.pathspec}"))
        else:
            current.card.append(Markdown("## Card Copy Failed"))
            current.card.append(Markdown(f"Failed to copy card from {self.pathspec}"))

        self.next(self.end)

    @step
    def end(self):
        """
        End the flow.
        """
        print("Flow completed successfully!")

if __name__ == "__main__":
    CardCopyTestFlow()
```
  • Loading branch information
valayDave committed Mar 5, 2025
1 parent 40ca1d9 commit d19cec5
Show file tree
Hide file tree
Showing 2 changed files with 196 additions and 5 deletions.
127 changes: 126 additions & 1 deletion metaflow/plugins/cards/card_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,7 @@ def card_read_options_and_arguments(func):
default=None,
show_default=True,
type=str,
help="Hash of the stored HTML",
help="UUID of the card",
)
@click.option(
"--type",
Expand Down Expand Up @@ -1094,3 +1094,128 @@ def _get_run_object(obj, run_id, user_namespace):

obj.echo("Using run-id %s" % run.pathspec, fg="blue", bold=False)
return run, follow_new_runs, None


@card.command(help="Copy cards from one pathspec to another")
@click.argument("origin_pathspec", type=str)
@click.argument("destination_pathspec", type=str)
@card_read_options_and_arguments
@click.pass_context
def copy(
ctx,
origin_pathspec,
destination_pathspec,
hash=None,
type=None,
id=None,
follow_resumed=True,
):
"""
Copy cards from one pathspec to another.
Args:
origin_pathspec: Pathspec of the card to copy from.
destination_pathspec: Pathspec of the card to copy to.
hash: Hash of the card to copy.
type: Type of the card to copy.
id: ID of the card to copy.
follow_resumed: Whether to follow the origin-task-id of resumed tasks to seek cards stored for resumed tasks.
"""
from metaflow import Task
from metaflow.exception import MetaflowNotFound

# Resolve the origin task
try:
origin_task = Task(origin_pathspec, _namespace_check=False)
if follow_resumed:
origin_taskpathspec = resumed_info(origin_task)
if origin_taskpathspec:
origin_task = Task(origin_taskpathspec, _namespace_check=False)

except MetaflowNotFound as ex:
click.echo(f"Error resolving origin pathspec: {str(ex)}", err=True)
return exit(1)

# Resolve the destination task
try:
dest_task = Task(destination_pathspec, _namespace_check=False)
except MetaflowNotFound as ex:
click.echo(f"Error resolving destination pathspec: {str(ex)}", err=True)
return exit(1)

flow_datastore = ctx.obj.flow_datastore

# Get the card paths from origin
card_paths, card_datastore = resolve_paths_from_task(
flow_datastore,
pathspec=origin_task.pathspec,
type=type,
hash=hash,
card_id=id,
)

if not card_paths:
click.echo(f"No cards found for pathspec: {origin_pathspec}", err=True)
return exit(1)

# Create destination card datastore
# Get the card paths from origin
dest_card_datastore = CardDatastore(
flow_datastore,
pathspec=dest_task.pathspec,
)

# Copy each card
copied_count = 0
for card_path in card_paths:
try:
# Get card info
card_info = card_datastore.info_from_path(card_path)

# Get card HTML content
card_html = card_datastore.get_card_html(card_path)

# Save to destination
dest_card_datastore.save_card(
uuid=card_info.hash,
card_type=card_info.type,
card_html=card_html,
card_id=card_info.id,
)

# Try to copy the data file if it exists
try:
data_paths = card_datastore.extract_data_paths(
card_type=card_info.type,
card_hash=card_info.hash,
card_id=card_info.id,
)

if data_paths:
data_path = data_paths[0]
card_data = card_datastore.get_card_data(data_path)
if card_data is not None:
dest_card_datastore.save_data(
uuid=card_info.hash,
card_type=card_info.type,
json_data=card_data,
card_id=card_info.id,
)
except Exception as e:
# If data copy fails, we still continue with the HTML copy
click.echo(
f"Warning: Could not copy data for card {card_info.type}/{card_info.id}: {str(e)}",
err=True,
)
else:
copied_count += 1
click.echo(
f"Copied card {card_info.type}/{card_info.id or ''} from {origin_pathspec} to {destination_pathspec}",
err=True,
)

except Exception as e:
click.echo(f"Error copying card {card_path}: {str(e)}", err=True)

click.echo(f"Successfully copied {copied_count} card(s)")
return exit(0)
74 changes: 70 additions & 4 deletions metaflow/plugins/cards/component_serializer.py
Original file line number Diff line number Diff line change
Expand Up @@ -803,10 +803,76 @@ def clear(self):

def refresh(self, *args, **kwargs):
# FIXME: document
if self._default_editable_card is not None:
self._card_component_store[self._default_editable_card].refresh(
*args, **kwargs
)
if self._default_editable_card is None:
return
self._card_component_store[self._default_editable_card].refresh(*args, **kwargs)

def copy(self, source_pathspec, id=None, type=None, follow_resumed=True):
"""
Copy a card from another pathspec to the current task. Like all other card related operations,
this will be best effort. If the card is not found or cannot be copied, this function will return
False but won't crash the metaflow task runtime.
Parameters
----------
source_pathspec : str
Pathspec of the card to copy from.
id : str, optional
ID of the card to copy.
type : str, optional
Type of the card to copy.
follow_resumed : bool, default: True
Whether to follow the origin-task-id of resumed tasks to seek cards stored for resumed tasks.
Returns
-------
bool
True if the card was copied successfully, False otherwise.
"""
from metaflow.metaflow_current import current
import subprocess
import sys
import os

# Get the current pathspec as the destination
if not current.pathspec:
return False

destination_pathspec = current.pathspec

# Build the command
executable = sys.executable
cmd = [
executable,
sys.argv[0],
]

cmd += self._card_creator._top_level_options + [
"card",
"copy",
source_pathspec,
destination_pathspec,
]

# Add optional arguments
if id:
cmd.extend(["--id", id])
if type:
cmd.extend(["--type", type])
if not follow_resumed:
cmd.append("--no-follow-resumed")

# Execute the command following the pattern in card_creator.py
env = os.environ.copy()
try:
subprocess.run(cmd, env=env, stderr=subprocess.PIPE, check=True)
return True
except subprocess.CalledProcessError as e:
self._warning("Error copying card: %s" % e.stderr.decode("utf-8"))
return False
except Exception as e:
self._warning("Error copying card: %s" % e)
return False

def _get_latest_data(self, card_uuid, final=False, mode=None):
"""
Expand Down

0 comments on commit d19cec5

Please sign in to comment.