Unable to roll back broken controller node

[rpawlik]

I moved my controller node into Infrastructure but it did not actually assign the proper role in Chef:

root@opencenter-server:~# opencentercli task logs 24
=== Logs for task 24: needsrekick.openstack.com > run_chef ===
Running chef
[2013-04-03T22:08:22+00:00] INFO: *** Chef 11.2.0 ***
[2013-04-03T22:08:22+00:00] INFO: [inet6] no default interface, picking the first ipaddress
[2013-04-03T22:08:22+00:00] INFO: Run List is []
[2013-04-03T22:08:22+00:00] INFO: Run List expands to []
[2013-04-03T22:08:22+00:00] INFO: Starting Chef Run for needsrekick
[2013-04-03T22:08:22+00:00] INFO: Running start handlers
[2013-04-03T22:08:22+00:00] INFO: Start handlers complete.
[2013-04-03T22:08:23+00:00] INFO: Loading cookbooks []
[2013-04-03T22:08:23+00:00] WARN: Node needsrekick has an empty run list.
[2013-04-03T22:08:23+00:00] INFO: Chef Run complete in 0.131015889 seconds
[2013-04-03T22:08:23+00:00] INFO: Running report handlers
[2013-04-03T22:08:23+00:00] INFO: Report handlers complete

When I try to back it out and move it back to "unprovisioned", I get a 403:

root@opencenter-server:~# opencentercli --debug node move needsrekick.openstack.com unprovisioned
2013-04-03 22:27:19,631 - opencenter - DEBUG - CLI arguments: Namespace(cli_action='move', cli_noun='node', debug=True, endpoint='https://admin:password@localhost:8443', new_parent_id_or_name='unprovisioned', node_id_or_name='needsrekick.openstack.com')
2013-04-03 22:27:19,646 - opencenter.endpoint - DEBUG - Refreshing nodes for list
2013-04-03 22:27:19,657 - opencenter.endpoint - DEBUG - Request Made:
REQ: curl -i https://localhost:8443/nodes/filter -X POST -H "content-type: application/json" -d '{"filter": "name='needsrekick.openstack.com'"}'

2013-04-03 22:27:19,850 - opencenter.endpoint - DEBUG - Response Received:
RESP: [200] {'date': 'Wed, 03 Apr 2013 22:27:19 GMT', 'content-length': '9094', 'content-type': 'application/json'}
RESP BODY: {
"status": 200,
"message": "success",
"nodes": [
{
"name": "needsrekick.openstack.com",
"task_id": null,
"adventure_id": null,
"attrs": {
"last_checkin": 1365028037,
"locked": true,
"opencenter_agent_actions": {
"run_chef": {
"plugin": "chef",
"args": {},
"consequences": [],
"timeout": 600,
"constraints": []
},
"update_cookbooks": {
"plugin": "chef",
"args": {},
"consequences": [
"facts.chef_server_ready := true"
],
"timeout": 30,
"constraints": []
},
"service_restart": {
"plugin": "service",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"agent_restart": {
"plugin": "agent_restart",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"modules.reload": {
"plugin": "modules",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"rollback_install_chef_server": {
"plugin": "chef",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"service_stop": {
"plugin": "service",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"get_updates": {
"plugin": "packages",
"args": {},
"consequences": [],
"timeout": 300,
"constraints": []
},
"modules.load": {
"plugin": "modules",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"openstack_upload_images": {
"plugin": "openstack",
"args": {},
"consequences": [],
"timeout": 300,
"constraints": []
},
"download_cookbooks": {
"plugin": "chef",
"args": {
"CHEF_SERVER_COOKBOOK_CHANNELS": {
"type": "evaluated",
"expression": "self.facts.chef_server_cookbook_channels"
}
},
"consequences": [],
"timeout": 120,
"constraints": []
},
"get_latest_channel_version": {
"plugin": "chef",
"args": {
"channel_name": {
"required": true,
"type": "string"
}
},
"consequences": [],
"timeout": 30,
"constraints": []
},
"openstack_enable_host": {
"plugin": "openstack",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"get_chef_info": {
"plugin": "chef",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"logfile.tail": {
"plugin": "modules",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"rollback_install_chef": {
"plugin": "chef",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"logfile.watch": {
"plugin": "modules",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"openstack_evacuate_host": {
"plugin": "openstack",
"args": {},
"consequences": [],
"timeout": 1200,
"constraints": []
},
"subscribe_cookbook_channel": {
"plugin": "chef",
"args": {
"channel_name": {
"description": "The name of the cookbook channel to subscribe to.",
"default": "current",
"type": "string",
"required": true,
"placeholder": "current",
"friendly": "Channel Name"
}
},
"consequences": [
"facts.chef_server_cookbook_channels := "{channel_name}""
],
"timeout": 30,
"constraints": []
},
"upgrade_agent": {
"plugin": "packages",
"args": {},
"consequences": [],
"timeout": 300,
"constraints": []
},
"install_chef": {
"plugin": "chef",
"args": {
"chef_server": {
"required": true,
"type": "interface",
"name": "chef-server"
},
"CHEF_SERVER_URL": {
"type": "evaluated",
"expression": "nodes.{chef_server}.facts.chef_server_uri"
},
"CHEF_SERVER_PEM": {
"type": "evaluated",
"expression": "nodes.{chef_server}.facts.chef_server_pem"
},
"CHEF_SERVER_HOSTNAME": {
"type": "evaluated",
"expression": "nodes.{chef_server}.name"
}
},
"consequences": [
"facts.backends := union(facts.backends, "chef-client")",
"facts.chef_server_consumed := {chef_server}"
],
"timeout": 300,
"constraints": []
},
"install_chef_server": {
"plugin": "chef",
"args": {},
"consequences": [],
"timeout": 600,
"constraints": []
},
"do_updates": {
"plugin": "packages",
"args": {},
"consequences": [],
"timeout": 600,
"constraints": []
},
"uninstall_chef": {
"plugin": "chef",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"service_start": {
"plugin": "service",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"get_cookbook_channels": {
"plugin": "chef",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"modules.actions": {
"plugin": "modules",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"uninstall_chef_server": {
"plugin": "chef",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"modules.list": {
"plugin": "modules",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
},
"openstack_disable_host": {
"plugin": "openstack",
"args": {},
"consequences": [],
"timeout": 30,
"constraints": []
}
},
"registered": true,
"chef_client_version": "11.2.0",
"last_task": "ok",
"converged": true,
"opencenter_agent_output_modules": [
"modules",
"openstack",
"packages",
"agent_restart",
"chef",
"service"
]
},
"facts": {
"nova_use_single_default_gateway": false,
"osops_public": "10.240.0.0/24",
"chef_server_consumed": 5,
"nova_role": "nova-controller-master",
"nova_public_if": "10.240.0.0/24",
"keystone_admin_pw": "test1234",
"nova_vm_bridge": "br100",
"chef_environment": "NovaCluster",
"cpu_allocation_ratio": 16,
"ha_infra": false,
"osops_nova": "10.240.0.0/24",
"parent_id": 8,
"backends": [
"node",
"agent",
"nova-controller",
"nova",
"chef-environment",
"chef-client"
],
"osops_mgmt": "10.240.0.0/24",
"nova_vm_fixed_if": "eth0",
"ram_allocation_ratio": 1,
"libvirt_type": "kvm",
"nova_network_dhcp_name": "novalocal",
"nova_vm_fixed_range": "10.240.0.0/24",
"nova_dmz_cidr": "127.0.0.0/24"
},
"id": 6
}
]
}

2013-04-03 22:27:19,865 - opencenter.endpoint - DEBUG - payload: {'filter': "name='needsrekick.openstack.com'"}
2013-04-03 22:27:19,865 - opencenter.endpoint - DEBUG - Refreshing nodes for list
2013-04-03 22:27:19,865 - opencenter.endpoint - DEBUG - Request Made:
REQ: curl -i https://localhost:8443/nodes/filter -X POST -H "content-type: application/json" -d '{"filter": "name='unprovisioned'"}'

2013-04-03 22:27:19,882 - opencenter.endpoint - DEBUG - Response Received:
RESP: [200] {'date': 'Wed, 03 Apr 2013 22:27:19 GMT', 'content-length': '393', 'content-type': 'application/json'}
RESP BODY: {
"status": 200,
"message": "success",
"nodes": [
{
"name": "unprovisioned",
"task_id": null,
"adventure_id": null,
"attrs": {
"locked": true,
"display_name": "Available Nodes"
},
"facts": {
"parent_id": 1,
"backends": [
"container",
"node"
]
},
"id": 2
}
]
}

2013-04-03 22:27:19,886 - opencenter.endpoint - DEBUG - payload: {'filter': "name='unprovisioned'"}
2013-04-03 22:27:19,887 - opencenter.endpoint - DEBUG - Refreshing facts for list
2013-04-03 22:27:19,901 - opencenter.endpoint - DEBUG - Refreshing facts for list
2013-04-03 22:27:19,902 - opencenter.fact - DEBUG - Payload: {"node_id": 6, "key": "parent_id", "value": 2}
2013-04-03 22:27:19,902 - opencenter.endpoint - DEBUG - Request Made:
REQ: curl -i https://localhost:8443/facts/ -X POST -H "content-type: application/json" -d '{"node_id": 6, "key": "parent_id", "value": 2}'

2013-04-03 22:27:20,847 - opencenter.endpoint - DEBUG - Response Received:
RESP: [403] {'date': 'Wed, 03 Apr 2013 22:27:20 GMT', 'content-length': '91', 'content-type': 'application/json'}
RESP BODY: {
"status": 403,
"message": "cannot be solved",
"friendly": "Cannot solve action"
}

2013-04-03 22:27:20,849 - opencenter.fact - WARNING - status code 403 on post
2013-04-03 22:27:20,849 - opencenter.endpoint - DEBUG - Refreshing facts for post

Is there anything I can do to roll it back and try to give it the role again?

[jianghaitao]

I have the same exact issue. My controller node was somehow got registered with short hostname instead of FQDN, thing gets messy. So, I want to re-do / roll back the adventure, but have not find a way to do that. I delete the node and re-add it, the thing is that opencentercli recognize it but not the GUI.