Skip to content

[card] current.card.copy [exploratory] #2337

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 126 additions & 1 deletion metaflow/plugins/cards/card_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,7 @@ def card_read_options_and_arguments(func):
default=None,
show_default=True,
type=str,
help="Hash of the stored HTML",
help="UUID of the card",
)
@click.option(
"--type",
Expand Down Expand Up @@ -1094,3 +1094,128 @@ def _get_run_object(obj, run_id, user_namespace):

obj.echo("Using run-id %s" % run.pathspec, fg="blue", bold=False)
return run, follow_new_runs, None


@card.command(help="Copy cards from one pathspec to another")
@click.argument("origin_pathspec", type=str)
@click.argument("destination_pathspec", type=str)
@card_read_options_and_arguments
@click.pass_context
def copy(
ctx,
origin_pathspec,
destination_pathspec,
hash=None,
type=None,
id=None,
follow_resumed=True,
):
"""
Copy cards from one pathspec to another.

Args:
origin_pathspec: Pathspec of the card to copy from.
destination_pathspec: Pathspec of the card to copy to.
hash: Hash of the card to copy.
type: Type of the card to copy.
id: ID of the card to copy.
follow_resumed: Whether to follow the origin-task-id of resumed tasks to seek cards stored for resumed tasks.
"""
from metaflow import Task
from metaflow.exception import MetaflowNotFound

# Resolve the origin task
try:
origin_task = Task(origin_pathspec, _namespace_check=False)
if follow_resumed:
origin_taskpathspec = resumed_info(origin_task)
if origin_taskpathspec:
origin_task = Task(origin_taskpathspec, _namespace_check=False)

except MetaflowNotFound as ex:
click.echo(f"Error resolving origin pathspec: {str(ex)}", err=True)
return exit(1)

# Resolve the destination task
try:
dest_task = Task(destination_pathspec, _namespace_check=False)
except MetaflowNotFound as ex:
click.echo(f"Error resolving destination pathspec: {str(ex)}", err=True)
return exit(1)

flow_datastore = ctx.obj.flow_datastore

# Get the card paths from origin
card_paths, card_datastore = resolve_paths_from_task(
flow_datastore,
pathspec=origin_task.pathspec,
type=type,
hash=hash,
card_id=id,
)

if not card_paths:
click.echo(f"No cards found for pathspec: {origin_pathspec}", err=True)
return exit(1)

# Create destination card datastore
# Get the card paths from origin
dest_card_datastore = CardDatastore(
flow_datastore,
pathspec=dest_task.pathspec,
)

# Copy each card
copied_count = 0
for card_path in card_paths:
try:
# Get card info
card_info = card_datastore.info_from_path(card_path)

# Get card HTML content
card_html = card_datastore.get_card_html(card_path)

# Save to destination
dest_card_datastore.save_card(
uuid=card_info.hash,
card_type=card_info.type,
card_html=card_html,
card_id=card_info.id,
)

# Try to copy the data file if it exists
try:
data_paths = card_datastore.extract_data_paths(
card_type=card_info.type,
card_hash=card_info.hash,
card_id=card_info.id,
)

if data_paths:
data_path = data_paths[0]
card_data = card_datastore.get_card_data(data_path)
if card_data is not None:
dest_card_datastore.save_data(
uuid=card_info.hash,
card_type=card_info.type,
json_data=card_data,
card_id=card_info.id,
)
except Exception as e:
# If data copy fails, we still continue with the HTML copy
click.echo(
f"Warning: Could not copy data for card {card_info.type}/{card_info.id}: {str(e)}",
err=True,
)
else:
copied_count += 1
click.echo(
f"Copied card {card_info.type}/{card_info.id or ''} from {origin_pathspec} to {destination_pathspec}",
err=True,
)

except Exception as e:
click.echo(f"Error copying card {card_path}: {str(e)}", err=True)

click.echo(f"Successfully copied {copied_count} card(s)")
return exit(0)
74 changes: 70 additions & 4 deletions metaflow/plugins/cards/component_serializer.py
Original file line number Diff line number Diff line change
Expand Up @@ -803,10 +803,76 @@ def clear(self):

def refresh(self, *args, **kwargs):
# FIXME: document
if self._default_editable_card is not None:
self._card_component_store[self._default_editable_card].refresh(
*args, **kwargs
)
if self._default_editable_card is None:
return
self._card_component_store[self._default_editable_card].refresh(*args, **kwargs)

def copy(self, source_pathspec, id=None, type=None, follow_resumed=True):
"""
Copy a card from another pathspec to the current task. Like all other card related operations,
this will be best effort. If the card is not found or cannot be copied, this function will return
False but won't crash the metaflow task runtime.

Parameters
----------
source_pathspec : str
Pathspec of the card to copy from.
id : str, optional
ID of the card to copy.
type : str, optional
Type of the card to copy.
follow_resumed : bool, default: True
Whether to follow the origin-task-id of resumed tasks to seek cards stored for resumed tasks.

Returns
-------
bool
True if the card was copied successfully, False otherwise.
"""
from metaflow.metaflow_current import current
import subprocess
import sys
import os

# Get the current pathspec as the destination
if not current.pathspec:
return False

destination_pathspec = current.pathspec

# Build the command
executable = sys.executable
cmd = [
executable,
sys.argv[0],
]

cmd += self._card_creator._top_level_options + [
"card",
"copy",
source_pathspec,
destination_pathspec,
]

# Add optional arguments
if id:
cmd.extend(["--id", id])
if type:
cmd.extend(["--type", type])
if not follow_resumed:
cmd.append("--no-follow-resumed")

# Execute the command following the pattern in card_creator.py
env = os.environ.copy()
try:
subprocess.run(cmd, env=env, stderr=subprocess.PIPE, check=True)
return True
except subprocess.CalledProcessError as e:
self._warning("Error copying card: %s" % e.stderr.decode("utf-8"))
return False
except Exception as e:
self._warning("Error copying card: %s" % e)
return False

def _get_latest_data(self, card_uuid, final=False, mode=None):
"""
Expand Down
Loading