diff --git a/bb/tests/keywords_pytest.robot b/bb/tests/keywords_pytest.robot index b080241b4..3326a0b5a 100644 --- a/bb/tests/keywords_pytest.robot +++ b/bb/tests/keywords_pytest.robot @@ -2,10 +2,11 @@ RunPyTest [Arguments] ${testcase} ${args} set test variable ${oldcmd} ${CMDLAUNCHER} - ${pysize}= Get File Size /usr/bin/python2 + ${pysize}= Get File Size /usr/bin/python3 - Run keyword if '${pysize} > 0' set test variable ${CMDLAUNCHER} python2 + Run keyword if '${pysize} > 0' set test variable ${CMDLAUNCHER} python3 Run keyword unless '${pysize} > 0' set test variable ${CMDLAUNCHER} python + Run keyword unless '${pysize} > 0' set test variable ${CMDLAUNCHER} python2 mpirun ${WORKDIR}/bb/wrappers/bbapi_main.py --libpath ${WORKDIR}/bb/lib --testpath ${WORKDIR}/bb/wrappers --testcase ${testcase} ${args} set test variable ${CMDLAUNCHER} ${oldcmd} diff --git a/bb/wrappers/Procedures.py b/bb/wrappers/Procedures.py index 69c4085fe..12b395838 100644 --- a/bb/wrappers/Procedures.py +++ b/bb/wrappers/Procedures.py @@ -1,574 +1,574 @@ -#!/usr/bin/python -#################################################### -# Procedures.py -# -# Copyright IBM Corporation 2017,2017. All Rights Reserved -# -# This program is licensed under the terms of the Eclipse Public License -# v1.0 as published by the Eclipse Foundation and available at -# http://www.eclipse.org/legal/epl-v10.html -# -# U.S. Government Users Restricted Rights: Use, duplication or disclosure -# restricted by GSA ADP Schedule Contract with IBM Corp. -################################################### - -""" - Common Burst Buffer Procedures -""" - -import time - -import bb -from bberror import * -from bbapi import * -from bbapiAdmin import * -from bbapiTest import * - -# -# Interfaces directly invoked from the command line -# These can be run under the user's profile -# - -def CancelTransfers(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - l_TargetHandle = pEnv["handle"] - l_CancelScope = pEnv["cancelscope"] - l_Handles = bb.getHandles() - for l_Handle in l_Handles: - try: - if (l_TargetHandle == bb.NO_HANDLE or l_TargetHandle == l_Handle): - BB_CancelTransfer(l_Handle, l_CancelScope) - except BBError as error: - error.handleError() - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def GetHandle(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - l_Contrib = pEnv["contrib"] - l_Handle = BB_GetTransferHandle(pEnv["tag"], len(l_Contrib), l_Contrib) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def GetTransferInfo(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - l_Handle = pEnv["handle"] - l_Info = BB_GetTransferInfo(l_Handle) - bb.printStruct(l_Info) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def GetUsage(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - l_Mount = pEnv["MOUNT"] - BB_GetUsage(l_Mount) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def GetWaitForReplyCount(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - l_Attempts = 0 - l_Continue = 1 - l_PrevServer = None - l_PrevValue = None - l_PrintOption = False; - while (l_Continue > 0): - l_ActiveServer = BB_GetServer("active", False) - if (l_ActiveServer != ""): - if (l_Attempts % 120 == 0): - l_PrintOption = True - l_Value = int(BB_GetServerByName(l_ActiveServer, "waitforreplycount", l_PrintOption)) - time.sleep(0.25) - - l_PrintOption = False - if (l_PrevServer != l_ActiveServer or l_PrevValue != l_Value): - l_PrintOption = True - l_PrevServer = l_ActiveServer - l_PrevValue = l_Value - l_Attempts = 0 - l_Attempts += 1 - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def ListHandles(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - bb.getHandles() - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def ListTransfers(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - l_Handles = bb.getHandles() - for l_Handle in l_Handles: - try: - BB_GetTransferInfo(l_Handle) - except BBError as error: - error.handleError() - - except BBError as error: - error.handleError() - rc = error.rc - - return rc - - -# -# Interfaces directly invoked from the command line -# These can be must be run uder the root profile -# - -def CloseServer(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - l_Name = pEnv["name"] - - BB_CloseServer(l_Name) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def CreateLogicalVolume_Direct(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - BB_CreateLogicalVolume(pEnv["MOUNT"], pEnv["SIZE"]) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def GetServer(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - l_Type = pEnv["type"] - - BB_GetServer(l_Type) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def OpenServer(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - l_Name = pEnv["name"] - - BB_OpenServer(l_Name) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def RetrieveTransfers(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - l_HostName = pEnv["hostname"] - l_Handle = pEnv["handle"] - l_Flags = DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS - if pEnv.has_key("FLAGS"): - l_Flags = BB_RTV_TRANSFERDEFS_FLAGS.get(pEnv["FLAGS"], DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS) - - (l_NumberOfTranferDefs, l_TransferDefs, l_BytesForTransferDefs) = BB_RetrieveTransfers(l_HostName, l_Handle, l_Flags) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def RestartTransfers(pEnv): - rc = 0 - l_ResumeCN_Host = False - l_NumStoppedTransferDefs = 0 - - try: - bb.initEnv(pEnv) - l_HostName = pEnv["hostname"] - l_Handle = pEnv["handle"] - l_Flags = DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS - if pEnv.has_key("FLAGS"): - l_Flags = BB_RTV_TRANSFERDEFS_FLAGS.get(pEnv["FLAGS"], DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS) - - l_ActiveServer = BB_GetServer("active") - if (pEnv["IO_FAILOVER"] == 1): - # New ESS, set up that environment - l_PrimaryServer = BB_GetServer("primary") - l_BackupServer = BB_GetServer("backup") - - if (l_ActiveServer == "" or l_ActiveServer == l_PrimaryServer): - l_NewServer = l_BackupServer - else: - l_NewServer = l_PrimaryServer - - if (l_NewServer.upper() in ("NONE",)): - raise BBError(rc=-1, text="There is no definied backup for this bbServer") - - ''' - # Don't do this as it causes a window... 'activate' makes the swap atomically between the two servers... - try: - BB_SetServer("offline", l_ActiveServer) - except BBError as error: - if not error.handleError(): - raise - ''' - - try: - BB_OpenServer(l_NewServer) - except BBError as error: - if not error.handleError(): - raise - -# bb.flushWaiters(l_ActiveServer) - BB_SetServer("activate", l_NewServer) - - # NOTE: Try to let start transfers to complete their second volley - # before suspending the connection(s) -# bb.flushWaiters(l_ActiveServer) - BB_Suspend(l_HostName) - l_ResumeCN_Host = True - - # NOTE: We do not want to close the connection to the previouly active server until we have - # stopped any transfers that may still be running on that server - (l_NumberOfTranferDefs, l_TransferDefs, l_BytesForTransferDefs) = BB_RetrieveTransfers(l_HostName, l_Handle, l_Flags) - if (l_NumberOfTranferDefs > 0): - if (not (l_Flags == BB_RTV_TRANSFERDEFS_FLAGS["ONLY_DEFINITIONS_WITH_STOPPED_FILES"])): - l_NumStoppedTransferDefs = BB_StopTransfers(l_HostName, l_Handle, l_TransferDefs, l_BytesForTransferDefs) - print "%d transfer definition(s) were found already stopped or were stopped by the previous operation" % (l_NumStoppedTransferDefs) - else: - l_NumStoppedTransferDefs = l_NumberOfTranferDefs - print "Additional transfer definition(s) are not being stopped because the option passed on the retrieve transfer operation indicated %s" % (BB_RTV_TRANSFERDEFS_FLAGS[l_Flags]) - else: - print "No transfer definition(s) were found given the provided input criteria. An operation to stop transfers will not be attempted." - - # NOTE: Even if no transfer definitions were stopped, we still need to issue the restart. - # The stop transfer(s) could have actually been performed on other bbServers via - # the async request file... - if (l_NumberOfTranferDefs > 0): - # Restart the transfers - l_NumRestartedTransferDefs = BB_RestartTransfers(l_HostName, l_Handle, l_TransferDefs, l_BytesForTransferDefs) - else: - print "%sNo transfer definition(s) were found given the provided input criteria. An operation to restart transfers will not be attempted." % (os.linesep) - - if (l_ResumeCN_Host): - l_ResumeCN_Host = False - try: - # NOTE: This resume will ensure that the new - # bbServer is 'resumed' from any prior - # suspend activity. If the new bbServer - # is not suspended, it is a tolerated exception. - # This resume will also, via the async request file, - # resume the old bbServer for this CN hostname. - # However, any transfers on the old bbServer - # have been stopped, and thus, have been removed - # from any transfer queues. - BB_Resume(l_HostName) - except BBError as error: - if not error.handleError(): - raise - - if (pEnv["IO_FAILOVER"] == 1): - # Now, close the connection to the previously active bbServer - try: - # First, make sure we have no waiters... - # bb.flushWaiters(l_ActiveServer) - - # Close the connection to the 'old' server - if (l_ActiveServer != ""): - BB_CloseServer(l_ActiveServer) - except BBError as error: - if not error.handleError(): - raise - - elif (0==1 and pEnv["CN_FAILOVER"]): - # New CN, set up that environment - # Not ready for prime time... - - l_Mountpoint = pEnv["MOUNT"] - l_Owner = pEnv.get("OWNER", None) - l_Group = pEnv.get("GROUP", None) - l_Mode = int(pEnv.get("MODE", 0755)) - l_LVSize = pEnv.get("SIZE", "1G") - sudo_CreateDirectory(pEnv, l_Mountpoint) - sudo_ChangeOwner(pEnv, l_Mountpoint, l_Owner, l_Group) - sudo_ChangeMode(pEnv, l_Mountpoint, l_Mode) - sudo_CreateLogicalVolume(pEnv, l_Mountpoint, l_LVSize) - - l_ActiveServer = BB_GetServer("active") - - except BBError as error: - error.handleError() - rc = error.rc - - # If we need to resume the active bbServer, do so now... - if (l_ResumeCN_Host): - l_ResumeCN_Host = False - BB_Resume(l_HostName) - - return rc - -def Resume(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - l_HostName = pEnv["hostname"] - - BB_Resume(l_HostName) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def SetServer(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - l_Type = pEnv["type"] - l_Name = pEnv["name"] - - BB_SetServer(l_Type, l_Name) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def StopTransfers(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - l_HostName = pEnv["hostname"] - l_Handle = pEnv["handle"] - l_Flags = DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS - if pEnv.has_key("FLAGS"): - l_Flags = BB_RTV_TRANSFERDEFS_FLAGS.get(pEnv["FLAGS"], DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS) - - (l_NumberOfTranferDefs, l_TransferDefs, l_BytesForTransferDefs) = BB_RetrieveTransfers(l_HostName, l_Handle, l_Flags) - if (l_NumberOfTranferDefs > 0): - l_NumStoppedTransferdDefs = BB_StopTransfers(l_HostName, l_Handle, l_TransferDefs, l_BytesForTransferDefs) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def Suspend(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - l_HostName = pEnv["hostname"] - - BB_Suspend(l_HostName) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - - -# -# Interfaces invoked via bbapiAdminProcs -# (i.e., --procedure_args contains the necessary arguments...) -# -# These implementations invoke another instance of the python -# interpreter with sudo... -# - -def ChangeMode(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - l_Args = pEnv["procedure_args"].split(",") - BB_ChangeMode(l_Args[0], int(l_Args[1])) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def ChangeOwner(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - l_Args = pEnv["procedure_args"].split(",") - BB_ChangeOwner(l_Args[0], l_Args[1], l_Args[2]) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def CreateDirectory(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - l_Args = pEnv["procedure_args"].split(",") - BB_CreateDirectory(l_Args[0]) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def CreateLogicalVolume(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - l_Args = pEnv["procedure_args"].split(",") - BB_CreateLogicalVolume(l_Args[0], l_Args[1], int(l_Args[2])) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def RemoveDirectory(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - l_Args = pEnv["procedure_args"].split(",") - BB_RemoveDirectory(l_Args[0]) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def ResizeMountPoint(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - l_Args = pEnv["procedure_args"].split(",") - BB_ResizeMountPoint(l_Args[0], l_Args[1], int(l_Args[2])) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -# NOTE: Currently, a test command... -def StageOutStart(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - l_Args = pEnv["procedure_args"].split(",") - Coral_StageOutStart(l_Args[0]) - except BBError as error: - error.handleError() - rc = error.rc - - return rc - - -# -# Interfaces invoked via either the command line or bbapiAdminProcs. -# (When via the command line, must be invoked using root. Otherwise, -# sudo is used for the python interpreter running the procedure.) -# -def RemoveJobInfo(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - - BB_RemoveJobInfo() - except BBError as error: - error.handleError() - rc = error.rc - - return rc - -def RemoveLogicalVolume(pEnv): - rc = 0 - - try: - bb.initEnv(pEnv) - if (pEnv["procedure_args"] == ""): - l_Mountpoint = pEnv["MOUNT"] - else: - l_Mountpoint = pEnv["procedure_args"] - - BB_RemoveLogicalVolume(l_Mountpoint) - except BBError as error: - error.handleError() - rc = error.rc - - return rc +#!/usr/bin/python +#################################################### +# Procedures.py +# +# Copyright IBM Corporation 2017,2017. All Rights Reserved +# +# This program is licensed under the terms of the Eclipse Public License +# v1.0 as published by the Eclipse Foundation and available at +# http://www.eclipse.org/legal/epl-v10.html +# +# U.S. Government Users Restricted Rights: Use, duplication or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. +################################################### + +""" + Common Burst Buffer Procedures +""" + +import time + +import bb +from bberror import * +from bbapi import * +from bbapiAdmin import * +from bbapiTest import * + +# +# Interfaces directly invoked from the command line +# These can be run under the user's profile +# + +def CancelTransfers(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + l_TargetHandle = pEnv["handle"] + l_CancelScope = pEnv["cancelscope"] + l_Handles = bb.getHandles() + for l_Handle in l_Handles: + try: + if (l_TargetHandle == bb.NO_HANDLE or l_TargetHandle == l_Handle): + BB_CancelTransfer(l_Handle, l_CancelScope) + except BBError as error: + error.handleError() + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def GetHandle(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + l_Contrib = pEnv["contrib"] + l_Handle = BB_GetTransferHandle(pEnv["tag"], len(l_Contrib), l_Contrib) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def GetTransferInfo(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + l_Handle = pEnv["handle"] + l_Info = BB_GetTransferInfo(l_Handle) + bb.printStruct(l_Info) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def GetUsage(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + l_Mount = pEnv["MOUNT"] + BB_GetUsage(l_Mount) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def GetWaitForReplyCount(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + l_Attempts = 0 + l_Continue = 1 + l_PrevServer = None + l_PrevValue = None + l_PrintOption = False; + while (l_Continue > 0): + l_ActiveServer = BB_GetServer("active", False) + if (l_ActiveServer != ""): + if (l_Attempts % 120 == 0): + l_PrintOption = True + l_Value = int(BB_GetServerByName(l_ActiveServer, "waitforreplycount", l_PrintOption)) + time.sleep(0.25) + + l_PrintOption = False + if (l_PrevServer != l_ActiveServer or l_PrevValue != l_Value): + l_PrintOption = True + l_PrevServer = l_ActiveServer + l_PrevValue = l_Value + l_Attempts = 0 + l_Attempts += 1 + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def ListHandles(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + bb.getHandles() + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def ListTransfers(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + l_Handles = bb.getHandles() + for l_Handle in l_Handles: + try: + BB_GetTransferInfo(l_Handle) + except BBError as error: + error.handleError() + + except BBError as error: + error.handleError() + rc = error.rc + + return rc + + +# +# Interfaces directly invoked from the command line +# These can be must be run uder the root profile +# + +def CloseServer(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + l_Name = pEnv["name"] + + BB_CloseServer(l_Name) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def CreateLogicalVolume_Direct(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + BB_CreateLogicalVolume(pEnv["MOUNT"], pEnv["SIZE"]) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def GetServer(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + l_Type = pEnv["type"] + + BB_GetServer(l_Type) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def OpenServer(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + l_Name = pEnv["name"] + + BB_OpenServer(l_Name) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def RetrieveTransfers(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + l_HostName = pEnv["hostname"] + l_Handle = pEnv["handle"] + l_Flags = DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS + if "FLAGS" in pEnv: + l_Flags = BB_RTV_TRANSFERDEFS_FLAGS.get(pEnv["FLAGS"], DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS) + + (l_NumberOfTranferDefs, l_TransferDefs, l_BytesForTransferDefs) = BB_RetrieveTransfers(l_HostName, l_Handle, l_Flags) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def RestartTransfers(pEnv): + rc = 0 + l_ResumeCN_Host = False + l_NumStoppedTransferDefs = 0 + + try: + bb.initEnv(pEnv) + l_HostName = pEnv["hostname"] + l_Handle = pEnv["handle"] + l_Flags = DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS + if "FLAGS" in pEnv: + l_Flags = BB_RTV_TRANSFERDEFS_FLAGS.get(pEnv["FLAGS"], DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS) + + l_ActiveServer = BB_GetServer("active") + if (pEnv["IO_FAILOVER"] == 1): + # New ESS, set up that environment + l_PrimaryServer = BB_GetServer("primary") + l_BackupServer = BB_GetServer("backup") + + if (l_ActiveServer == "" or l_ActiveServer == l_PrimaryServer): + l_NewServer = l_BackupServer + else: + l_NewServer = l_PrimaryServer + + if (l_NewServer.upper() in ("NONE",)): + raise BBError(rc=-1, text="There is no definied backup for this bbServer") + + ''' + # Don't do this as it causes a window... 'activate' makes the swap atomically between the two servers... + try: + BB_SetServer("offline", l_ActiveServer) + except BBError as error: + if not error.handleError(): + raise + ''' + + try: + BB_OpenServer(l_NewServer) + except BBError as error: + if not error.handleError(): + raise + +# bb.flushWaiters(l_ActiveServer) + BB_SetServer("activate", l_NewServer) + + # NOTE: Try to let start transfers to complete their second volley + # before suspending the connection(s) +# bb.flushWaiters(l_ActiveServer) + BB_Suspend(l_HostName) + l_ResumeCN_Host = True + + # NOTE: We do not want to close the connection to the previouly active server until we have + # stopped any transfers that may still be running on that server + (l_NumberOfTranferDefs, l_TransferDefs, l_BytesForTransferDefs) = BB_RetrieveTransfers(l_HostName, l_Handle, l_Flags) + if (l_NumberOfTranferDefs > 0): + if (not (l_Flags == BB_RTV_TRANSFERDEFS_FLAGS["ONLY_DEFINITIONS_WITH_STOPPED_FILES"])): + l_NumStoppedTransferDefs = BB_StopTransfers(l_HostName, l_Handle, l_TransferDefs, l_BytesForTransferDefs) + print("%d transfer definition(s) were found already stopped or were stopped by the previous operation" % (l_NumStoppedTransferDefs)) + else: + l_NumStoppedTransferDefs = l_NumberOfTranferDefs + print("Additional transfer definition(s) are not being stopped because the option passed on the retrieve transfer operation indicated %s" % (BB_RTV_TRANSFERDEFS_FLAGS[l_Flags])) + else: + print("No transfer definition(s) were found given the provided input criteria. An operation to stop transfers will not be attempted.") + + # NOTE: Even if no transfer definitions were stopped, we still need to issue the restart. + # The stop transfer(s) could have actually been performed on other bbServers via + # the async request file... + if (l_NumberOfTranferDefs > 0): + # Restart the transfers + l_NumRestartedTransferDefs = BB_RestartTransfers(l_HostName, l_Handle, l_TransferDefs, l_BytesForTransferDefs) + else: + print("%sNo transfer definition(s) were found given the provided input criteria. An operation to restart transfers will not be attempted." % (os.linesep)) + + if (l_ResumeCN_Host): + l_ResumeCN_Host = False + try: + # NOTE: This resume will ensure that the new + # bbServer is 'resumed' from any prior + # suspend activity. If the new bbServer + # is not suspended, it is a tolerated exception. + # This resume will also, via the async request file, + # resume the old bbServer for this CN hostname. + # However, any transfers on the old bbServer + # have been stopped, and thus, have been removed + # from any transfer queues. + BB_Resume(l_HostName) + except BBError as error: + if not error.handleError(): + raise + + if (pEnv["IO_FAILOVER"] == 1): + # Now, close the connection to the previously active bbServer + try: + # First, make sure we have no waiters... + # bb.flushWaiters(l_ActiveServer) + + # Close the connection to the 'old' server + if (l_ActiveServer != ""): + BB_CloseServer(l_ActiveServer) + except BBError as error: + if not error.handleError(): + raise + + elif (0==1 and pEnv["CN_FAILOVER"]): + # New CN, set up that environment + # Not ready for prime time... + + l_Mountpoint = pEnv["MOUNT"] + l_Owner = pEnv.get("OWNER", None) + l_Group = pEnv.get("GROUP", None) + l_Mode = int(pEnv.get("MODE", 0o755)) + l_LVSize = pEnv.get("SIZE", "1G") + sudo_CreateDirectory(pEnv, l_Mountpoint) + sudo_ChangeOwner(pEnv, l_Mountpoint, l_Owner, l_Group) + sudo_ChangeMode(pEnv, l_Mountpoint, l_Mode) + sudo_CreateLogicalVolume(pEnv, l_Mountpoint, l_LVSize) + + l_ActiveServer = BB_GetServer("active") + + except BBError as error: + error.handleError() + rc = error.rc + + # If we need to resume the active bbServer, do so now... + if (l_ResumeCN_Host): + l_ResumeCN_Host = False + BB_Resume(l_HostName) + + return rc + +def Resume(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + l_HostName = pEnv["hostname"] + + BB_Resume(l_HostName) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def SetServer(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + l_Type = pEnv["type"] + l_Name = pEnv["name"] + + BB_SetServer(l_Type, l_Name) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def StopTransfers(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + l_HostName = pEnv["hostname"] + l_Handle = pEnv["handle"] + l_Flags = DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS + if "FLAGS" in pEnv: + l_Flags = BB_RTV_TRANSFERDEFS_FLAGS.get(pEnv["FLAGS"], DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS) + + (l_NumberOfTranferDefs, l_TransferDefs, l_BytesForTransferDefs) = BB_RetrieveTransfers(l_HostName, l_Handle, l_Flags) + if (l_NumberOfTranferDefs > 0): + l_NumStoppedTransferdDefs = BB_StopTransfers(l_HostName, l_Handle, l_TransferDefs, l_BytesForTransferDefs) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def Suspend(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + l_HostName = pEnv["hostname"] + + BB_Suspend(l_HostName) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + + +# +# Interfaces invoked via bbapiAdminProcs +# (i.e., --procedure_args contains the necessary arguments...) +# +# These implementations invoke another instance of the python +# interpreter with sudo... +# + +def ChangeMode(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + l_Args = pEnv["procedure_args"].split(",") + BB_ChangeMode(l_Args[0], int(l_Args[1])) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def ChangeOwner(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + l_Args = pEnv["procedure_args"].split(",") + BB_ChangeOwner(l_Args[0], l_Args[1], l_Args[2]) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def CreateDirectory(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + l_Args = pEnv["procedure_args"].split(",") + BB_CreateDirectory(l_Args[0]) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def CreateLogicalVolume(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + l_Args = pEnv["procedure_args"].split(",") + BB_CreateLogicalVolume(l_Args[0], l_Args[1], int(l_Args[2])) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def RemoveDirectory(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + l_Args = pEnv["procedure_args"].split(",") + BB_RemoveDirectory(l_Args[0]) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def ResizeMountPoint(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + l_Args = pEnv["procedure_args"].split(",") + BB_ResizeMountPoint(l_Args[0], l_Args[1], int(l_Args[2])) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +# NOTE: Currently, a test command... +def StageOutStart(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + l_Args = pEnv["procedure_args"].split(",") + Coral_StageOutStart(l_Args[0]) + except BBError as error: + error.handleError() + rc = error.rc + + return rc + + +# +# Interfaces invoked via either the command line or bbapiAdminProcs. +# (When via the command line, must be invoked using root. Otherwise, +# sudo is used for the python interpreter running the procedure.) +# +def RemoveJobInfo(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + + BB_RemoveJobInfo() + except BBError as error: + error.handleError() + rc = error.rc + + return rc + +def RemoveLogicalVolume(pEnv): + rc = 0 + + try: + bb.initEnv(pEnv) + if (pEnv["procedure_args"] == ""): + l_Mountpoint = pEnv["MOUNT"] + else: + l_Mountpoint = pEnv["procedure_args"] + + BB_RemoveLogicalVolume(l_Mountpoint) + except BBError as error: + error.handleError() + rc = error.rc + + return rc diff --git a/bb/wrappers/RunProcedure.py b/bb/wrappers/RunProcedure.py index d9d0f7513..eac9a8fcb 100644 --- a/bb/wrappers/RunProcedure.py +++ b/bb/wrappers/RunProcedure.py @@ -1,80 +1,80 @@ -#!/usr/bin/python -#################################################### -# RunProcedure.py -# -# Copyright IBM Corporation 2017,2017. All Rights Reserved -# -# This program is licensed under the terms of the Eclipse Public License -# v1.0 as published by the Eclipse Foundation and available at -# http://www.eclipse.org/legal/epl-v10.html -# -# U.S. Government Users Restricted Rights: Use, duplication or disclosure -# restricted by GSA ADP Schedule Contract with IBM Corp. -################################################### - -""" - Common Burst Buffer Procedures - - Example invocations with bluecoral as your current working directory: - python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers - --testcase RunProcedure --procedure CancelTransfers --jobid 20 --jobstepid 0 - - python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers - --testcase RunProcedure --procedure ListTransfers --jobid 20 --jobstepid 0 - - python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers - --testcase RunProcedure --procedure ListHandles --jobid 20 --jobstepid 0 - - python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers - --testcase RunProcedure --procedure ListTransfers --jobid 20 --jobstepid 0 - - python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers - --testcase RunProcedure --procedure Resume - - NOTE: The following does a retrieve, followed by a stop, and then a restart - python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers - --testcase RunProcedure --procedure RestartTransfers --jobid 20 --jobstepid 0 - - python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers - --testcase RunProcedure --procedure RetrieveTransfers --jobid 20 --jobstepid 0 - - NOTE: The following does a retrieve, followed by a stop - python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers - --testcase RunProcedure --procedure StopTransfers --jobid 20 --jobstepid 0 - - python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers - --testcase RunProcedure --procedure Suspend -""" - -import sys - -import bb -import Procedures as procs - -from bberror import * -from bbapi import * -from bbapiAdmin import * - -def main(pEnv): - rc = 0; - - if pEnv.has_key("procedure"): - try: - rc = getattr(procs, pEnv["procedure"])(pEnv) - print "%sProcedure -> %s, rc = %d" % (os.linesep, os.path.splitext(os.path.basename(pEnv["procedure"]))[0], rc) - except: - rc = -1 - print "When attempting to invoke procedure %s, unexpected error: %s" % (pEnv["procedure"], sys.exc_info()[0]) - else: - rc = -1 - print "Option -w or --procedure must be specified" - - return rc - - -# -# Invoke main routine -# - -if __name__ == '__main__': - main(sys.argv[1]) +#!/usr/bin/python +#################################################### +# RunProcedure.py +# +# Copyright IBM Corporation 2017,2017. All Rights Reserved +# +# This program is licensed under the terms of the Eclipse Public License +# v1.0 as published by the Eclipse Foundation and available at +# http://www.eclipse.org/legal/epl-v10.html +# +# U.S. Government Users Restricted Rights: Use, duplication or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. +################################################### + +""" + Common Burst Buffer Procedures + + Example invocations with bluecoral as your current working directory: + python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers + --testcase RunProcedure --procedure CancelTransfers --jobid 20 --jobstepid 0 + + python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers + --testcase RunProcedure --procedure ListTransfers --jobid 20 --jobstepid 0 + + python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers + --testcase RunProcedure --procedure ListHandles --jobid 20 --jobstepid 0 + + python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers + --testcase RunProcedure --procedure ListTransfers --jobid 20 --jobstepid 0 + + python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers + --testcase RunProcedure --procedure Resume + + NOTE: The following does a retrieve, followed by a stop, and then a restart + python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers + --testcase RunProcedure --procedure RestartTransfers --jobid 20 --jobstepid 0 + + python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers + --testcase RunProcedure --procedure RetrieveTransfers --jobid 20 --jobstepid 0 + + NOTE: The following does a retrieve, followed by a stop + python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers + --testcase RunProcedure --procedure StopTransfers --jobid 20 --jobstepid 0 + + python ./bb/wrappers/bbapi_main.py --libpath ./work/bb/lib --testpath ./bb/wrappers + --testcase RunProcedure --procedure Suspend +""" + +import sys + +import bb +import Procedures as procs + +from bberror import * +from bbapi import * +from bbapiAdmin import * + +def main(pEnv): + rc = 0; + + if "procedure" in pEnv: + try: + rc = getattr(procs, pEnv["procedure"])(pEnv) + print("%sProcedure -> %s, rc = %d" % (os.linesep, os.path.splitext(os.path.basename(pEnv["procedure"]))[0], rc)) + except: + rc = -1 + print("When attempting to invoke procedure %s, unexpected error: %s" % (pEnv["procedure"], sys.exc_info()[0])) + else: + rc = -1 + print("Option -w or --procedure must be specified") + + return rc + + +# +# Invoke main routine +# + +if __name__ == '__main__': + main(sys.argv[1]) diff --git a/bb/wrappers/bb.py b/bb/wrappers/bb.py index a4c3789e8..e242b1213 100644 --- a/bb/wrappers/bb.py +++ b/bb/wrappers/bb.py @@ -1,440 +1,448 @@ -#!/usr/bin/python -#################################################### -# bb.py -# -# Copyright IBM Corporation 2017,2017. All Rights Reserved -# -# This program is licensed under the terms of the Eclipse Public License -# v1.0 as published by the Eclipse Foundation and available at -# http://www.eclipse.org/legal/epl-v10.html -# -# U.S. Government Users Restricted Rights: Use, duplication or disclosure -# restricted by GSA ADP Schedule Contract with IBM Corp. -################################################### - -""" - Common include to bbapi functions. -""" - -import ctypes -import collections -import ctypes -import itertools -import os -import random -import pprint -import sys -import time -import threading - -from copy import deepcopy - -from bbapi import * -from bbapiAdmin import BB_GetServerByName -from bbapiAdminProcs import * -from bbapiTest import * -from datetime import datetime - -# NOTE: All of the following "NO_" values -# correspond to the "UNDEFINED_" variables -# in bbinternal.h -NO_HOSTNAME = "%%_NuLl_HoStNaMe_%%" -NO_JOBID = 0 -NO_JOBSTEPID = 0 -NO_HANDLE = 0 -# NOTE: The following is a value that Coral_G/SetVar can handle... -NO_CONTRIBID = 999999999 - - -DO_NOT_PRINT_VALUE = False -DEFAULT_REMOVE_JOB_INFO_DELAY_FOR_ISSUER = 3 -DEFAULT_REMOVE_JOB_INFO_DELAY_FOR_NON_ISSUER = 60 -# DEFAULT_GET_TRANSFERINFO_DELAY = 5 -DEFAULT_GET_TRANSFERINFO_DELAY = 1.5 -# DEFAULT_WAIT_FOR_COMPLETION_ATTEMPTS = 360 # 30 minutes -DEFAULT_WAIT_FOR_COMPLETION_ATTEMPTS = 1440 # 2 hours - -Vars = {"buffer":ctypes.create_string_buffer, - "cancelscope":ctypes.c_int32, - "contribid":ctypes.c_uint32, - "devicenum":ctypes.c_uint32, - "flags":ctypes.c_int64, - "errformat":ctypes.c_int32, - "group":ctypes.create_string_buffer, - "handle":ctypes.c_uint64, - "hostname":ctypes.create_string_buffer, - "jobid":ctypes.c_uint64, - "key":ctypes.create_string_buffer, - "mode":ctypes.c_uint32, - "mountpoint":ctypes.create_string_buffer, - "name":ctypes.create_string_buffer, - "newpathname":ctypes.create_string_buffer, - "numavailhandles":ctypes.c_uint64, - "numbytesavailable":ctypes.c_uint64, - "numcontrib":ctypes.c_uint32, - "numhandles":ctypes.c_uint64, - "numtransferdefs":ctypes.c_uint32, - "option":ctypes.create_string_buffer, - "owner":ctypes.create_string_buffer, - "pathname":ctypes.create_string_buffer, - "rate":ctypes.c_uint64, - "size":ctypes.c_uint64, - "size_str":ctypes.create_string_buffer, - "source":ctypes.create_string_buffer, - "status":ctypes.c_int64, - "tag":ctypes.c_ulong, - "target":ctypes.create_string_buffer, - "transferdefs":ctypes.create_string_buffer, - "type":ctypes.create_string_buffer, - "value":ctypes.create_string_buffer, - "variable":ctypes.create_string_buffer, - } - -api = None - - -# -# Helper routines -# - -# Override the format() method for pprint.PrettyPrinter -# so that any unicode data is converted to utf8 -class bbpprint(pprint.PrettyPrinter): - def format(self, pObject, pContext, pMaxLevels, pLevel): - if isinstance(pObject, unicode): - return (pObject.encode('utf8'), True, False) - else: - return pprint.PrettyPrinter.format(self, pObject, pContext, pMaxLevels, pLevel) - -def cvar(pVar, pValue): - try: - if (type(pValue) == dict): - return Vars[pVar](pValue[pVar]) - else: - return Vars[pVar](pValue) - except: - print "Unexpected error when attempting to build a cvar: Type(pVar)=", type(pVar), ", type(pValue)=", type(pValue), ", pVar=", pVar, ", pValue=", pValue, "error=", sys.exc_info()[0] - return None - -def checkMD5SumFile(pMD5SumFile): - l_RC = 0 - - l_SourceCheckSum = "" - l_Result = [] - for l_Line in open(pMD5SumFile, "r"): - l_Result = l_Line.strip().split() - if len(l_SourceCheckSum) == 0: - l_SourceCheckSum, l_SourceFile = l_Result - else: - if l_SourceCheckSum == l_Result[0]: - print "** SUCCESS ** File %s with checksum of %s matches the checksum for source file %s" % (l_Result[1], l_Result[0], l_SourceFile) - else: - l_RC = -1 - print "** ERROR ** File %s with checksum of %s mismatches source file %s with checksum of %s" % (l_Result[1], l_Result[0], l_SourceFile, l_SourceCheckSum) - - return l_RC - -def checkFiles(pSourceFiles, pTargetFiles): - l_RC = 0 - - l_MD5SumFile = "/tmp/md5sum%s" % (random.choice(xrange(1000000))) - if type(pSourceFiles) in (tuple, list,): - for i in xrange(len(pSourceFiles)): - if type(pSourceFiles[i]) in (tuple, list,): - for j in xrange(len(pSourceFiles[i])): - if type(pSourceFiles[i][j]) not in (list, tuple): - runCmd("md5sum %s %s > %s" % (pSourceFiles[i][j], pTargetFiles[i][j], l_MD5SumFile)) - l_RC = checkMD5SumFile(l_MD5SumFile) - else: - for k in xrange(len(pSourceFiles[i][j])): - runCmd("md5sum %s %s > %s" % (pSourceFiles[i][j][k], pTargetFiles[i][j][k], l_MD5SumFile)) - l_RC = checkMD5SumFile(l_MD5SumFile) - else: - runCmd("md5sum %s %s > %s" % (pSourceFiles[i], pTargetFiles[i], l_MD5SumFile)) - l_RC = checkMD5SumFile(l_MD5SumFile) - else: - runCmd("md5sum %s %s > %s" % (pSourceFiles, pTargetFiles, l_MD5SumFile)) - l_RC = checkMD5SumFile(l_MD5SumFile) - - runCmd("rm %s" % (l_MD5SumFile)) - - if l_RC: - raise bberror.BBError(rc=l_RC, text="MD5Sum check failure") - - return - -def createRandomFile(pEnv, pFile, pSize): - l_RandFilePgm = os.path.abspath(os.path.join(os.path.dirname(pEnv["LIBPATH"]), ".", "tools/randfile")) - l_RemoveFilePgm = "rm" - l_CreateFilePgm = "touch" - if pSize > 0: - runCmd("%s --file %s --size %d" % (l_RandFilePgm, pFile, pSize)) - else: - runCmd("%s %s" % (l_RemoveFilePgm, pFile)) - runCmd("%s %s" % (l_CreateFilePgm, pFile)) - - return - -def flushWaiters(pActiveServer): - # Flush the waiters for the input active server - - if (pActiveServer != ""): - l_StartingValue = 5 - - if (int(BB_GetServerByName(pActiveServer, "waitforreplycount", False)) == 0): - l_Continue = l_StartingValue - 1; - else: - l_Continue = l_StartingValue - l_Attempts = 1 - - while (l_Continue > 0): - time.sleep(0.05) - l_PrintOption = False; - if (l_Attempts % 200 == 0): - l_PrintOption = True - if (int(BB_GetServerByName(pActiveServer, "waitforreplycount", l_PrintOption)) == 0): - l_Continue -= 1; - else: - l_Continue = l_StartingValue - l_Attempts += 1 - - return; - -def getContribId(pPrintValue=DO_NOT_PRINT_VALUE): - return Coral_GetVar('contribid', pPrintValue) - -def getHandles(pStatus=BBSTATUS["BBALL"]): - l_NumAvailHandles = 0 - l_NumHandles = -1 - while (l_NumAvailHandles > l_NumHandles): - l_NumHandles = l_NumAvailHandles - (l_NumAvailHandles, l_Handles) = BB_GetTransferList(pStatus, l_NumHandles) - - print "getHandles: jobid=%d, jobstepid=%d, contribid=%d, l_NumAvailHandles=%d, l_Handles=%s" % ( - (getJobId(), getJobStepId(), getContribId(), l_NumAvailHandles, `l_Handles`)) - - return l_Handles - -def getJobId(pPrintValue=DO_NOT_PRINT_VALUE): - return Coral_GetVar('jobid', pPrintValue) - -def getJobStepId(pPrintValue=DO_NOT_PRINT_VALUE): - return Coral_GetVar('jobstepid', pPrintValue) - -def incrJobStepId(pValue=1): - l_JobStepId = getJobStepId() - l_JobStepId = l_JobStepId + pValue - setJobStepId(l_JobStepId) - - return - -def initEnv(pEnv, pMountpoints=None, pDirectories=None): - # Cleanup from prior variations... - - l_RC = 0 - l_Continue = True; - -# print -# print ">>>> Start: Cleanup..." - - l_Suffix = "Tolerated exception and processing continues..." - l_SuffixEnd = "Non-tolerated exception and processing is ending..." - - # NOTE: The initialization must occur first so that if removejobinfo is - # issued below, the removejobinfo is issued for the correct jobid. - if (l_Continue): -# print ">>>> Start: Initialize environment..." - - l_Iteration = pEnv.get("iteration", 0) - l_JobIdBump = pEnv.get("jobid_bump", 0) - setJobId(pEnv["jobid"] + (l_Iteration*10) + l_JobIdBump) - setJobStepId(pEnv['jobstepid']) - setContribId(pEnv['contribid']) - -# print ">>>> End: Initialize environment..." - else: - l_Continue = False - - if ((pMountpoints is not None) or (pDirectories is not None)): - if l_Continue: - # If single contributor or contribid is 0, - # issue the removejobinfo request. - # - # Whether or not the request was issued, delay - # for a bit waiting for the removejobinfo to be - # processed, and possibly propagated, amongst all - # bbServers. - if len(pEnv['contrib']) == 1 or pEnv['contribid'] == 0: - try: - sudo_RemoveJobInfo(pEnv) - except BBError as error: - l_Continue = error.handleError() - time.sleep(DEFAULT_REMOVE_JOB_INFO_DELAY_FOR_ISSUER) - else: - time.sleep(DEFAULT_REMOVE_JOB_INFO_DELAY_FOR_NON_ISSUER) - - if l_Continue: - if pDirectories is not None: - if type(pDirectories) not in (tuple, list): - try: - l_Directory = pDirectories - sudo_RemoveDirectory(pEnv, l_Directory) - except BBError as error: - l_Continue = error.handleError() - else: - for l_Directory in pDirectories: - if l_Continue: - try: - sudo_RemoveDirectory(pEnv, l_Directory) - except BBError as error: - l_Continue = error.handleError() - if l_Continue: - if pMountpoints is not None: - if type(pMountpoints) not in (tuple, list): - try: - l_Mountpoint = pMountpoints - sudo_RemoveLogicalVolume(pEnv, l_Mountpoint) - except BBError as error: - l_Continue = error.handleError() - if l_Continue: - try: - sudo_RemoveDirectory(pEnv, l_Mountpoint) - except BBError as error: - l_Continue = error.handleError() - else: - for l_Mountpoint in pMountpoints: - if l_Continue: - try: - sudo_RemoveLogicalVolume(pEnv, l_Mountpoint) - except BBError as error: - l_Continue = error.handleError() - if l_Continue: - try: - sudo_RemoveDirectory(pEnv, l_Mountpoint) - except BBError as error: - l_Continue = error.handleError() - if not l_Continue: - l_RC = -1 - else: - l_RC = -1 - else: - l_RC = -1 - else: - l_RC = -1 - -# print ">>>> End: Cleanup..." - - return l_RC - -def printLastErrorDetailsSummary(): - print datetime.now().strftime("Current date/time: %Y-%m-%d %H:%M:%S") - dummy = BBError() - print dummy.getLastErrorDetailsSummary() - - return - -def printStruct(pStruct): - print "Start: Print of struct %s" % (pStruct) - for l_FieldName, l_FieldType in pStruct._fields_: - print "%s%s = %s" % (len("Start: ")*" ", l_FieldName, getattr(pStruct, l_FieldName)) - print " End: Print of struct %s" % (pStruct) - - return - -def printEnv(pEnv): - print "Start: Print of environment" - bbpprint().pprint(pEnv) - print " End: Print of environment" - - return - -def runCmd(pCmd): - print pCmd - os.system(pCmd) - - return - -# Setting the jobid from a program is not necessary if bbapi_main is invoked -# to run a testcase. Changing the jobid in the middle of a bbapi testcase -# is 'cheating', as the same connection is being used. But, it works to test -# most scenarios... @DLH -def setJobId(pValue): - Coral_SetVar('jobid', `pValue`) - - return - -def setJobStepId(pValue): - Coral_SetVar('jobstepid', `pValue`) - - return - -# Setting the contribid from a program is not necessary if bbapi_main is invoked -# to run a testcase. Changing the contribid in the middle of a bbapi testcase -# is 'cheating', as the same connection is being used. But, it works to test -# most scenarios... @DLH -def setContribId(pValue): - Coral_SetVar('contribid', `pValue`) - - return - -def waitForCompletion(pEnv, pHandles, pAttempts=DEFAULT_WAIT_FOR_COMPLETION_ATTEMPTS): - l_Complete = False - l_Continue = True - l_AllFullSuccess = True; - l_Attempts = 1 - l_LastTotalTransferSize = 0 - l_Status = [] - l_TransferSize = [] - - l_CopyHandles = deepcopy(pHandles) - while (l_Continue and (not l_Complete) and l_Attempts < pAttempts): - l_Complete = True - l_Handles = deepcopy(l_CopyHandles) - for l_Handle in l_Handles: - try: - l_Info = BB_GetTransferInfo(l_Handle) - # NOTE: BBSTOPPED is not included below because we want to 'spin' on a stopped transfer definition, - # waiting for it to progress to another status. - # NOTE: BBPARTIALSUCCESS is not included below because we want to 'spin' on a it also, waiting for - # it to progress back to BBINPROGRESS. It is possible in the restart scenarios if all of the - # extents are processed, but the restart processing hasn't finished yet. - if BBSTATUS[l_Info.status] not in ("BBFULLSUCCESS", "BBCANCELED",): - l_Complete = False - if l_LastTotalTransferSize != l_Info.totalTransferSize: - # Making progress... Reset the number of attempts... - l_LastTotalTransferSize = l_Info.totalTransferSize - l_Attempts = 0 - if len(pEnv["contrib"]) == 1 or bb.getContribId() == 0: - l_Delay = DEFAULT_GET_TRANSFERINFO_DELAY - else: - l_Delay = DEFAULT_GET_TRANSFERINFO_DELAY*2 - time.sleep(l_Delay) - break - else: - if BBSTATUS[l_Info.status] not in ("BBFULLSUCCESS",) or BBSTATUS[l_Info.localstatus] not in ("BBFULLSUCCESS",): - l_AllFullSuccess = False - l_Status.append((BBSTATUS[l_Info.localstatus],BBSTATUS[l_Info.status])) - l_TransferSize.append((l_Info.localTransferSize,l_Info.totalTransferSize)) - l_CopyHandles.remove(l_Handle) - l_Attempts = 0 - except BBError as error: - l_Continue = error.handleError() - if (not l_Continue): - break - l_Attempts += 1 - - if (l_Continue): - if (l_Complete): - if (len(pHandles) > 1): - print - - for i in xrange(len(l_Status)): - print " *FINAL* Handle: %12s -> Status (Local:Overall) (%13s:%13s) Transfer Size in bytes (Local:Total) (%s : %s)" % (pHandles[i], l_Status[i][0], l_Status[i][1], '{:,}'.format(l_TransferSize[i][0]), '{:,}'.format(l_TransferSize[i][1])) - else: - l_AllFullSuccess = False - print "Exceeded the maximum number of attempts to have all handles reach a status of BBFULLSUCCESS. %d attempts were made." % (pAttempts) - else: - l_AllFullSuccess = False - print "Error occurred while waiting for all handles reach a status of BBFULLSUCCESS." - - return l_AllFullSuccess +#!/usr/bin/python +#################################################### +# bb.py +# +# Copyright IBM Corporation 2017,2017. All Rights Reserved +# +# This program is licensed under the terms of the Eclipse Public License +# v1.0 as published by the Eclipse Foundation and available at +# http://www.eclipse.org/legal/epl-v10.html +# +# U.S. Government Users Restricted Rights: Use, duplication or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. +################################################### + +""" + Common include to bbapi functions. +""" + +import collections +import ctypes +import itertools +import os +import random +import pprint +import sys +import time +import threading + +from copy import deepcopy + +from bbapi import * +from bbapiAdmin import BB_GetServerByName +from bbapiAdminProcs import * +from bbapiTest import * +from datetime import datetime + +# NOTE: All of the following "NO_" values +# correspond to the "UNDEFINED_" variables +# in bbinternal.h +NO_HOSTNAME = "%%_NuLl_HoStNaMe_%%" +NO_JOBID = 0 +NO_JOBSTEPID = 0 +NO_HANDLE = 0 +# NOTE: The following is a value that Coral_G/SetVar can handle... +NO_CONTRIBID = 999999999 + + +DO_NOT_PRINT_VALUE = False +DEFAULT_REMOVE_JOB_INFO_DELAY_FOR_ISSUER = 3 +DEFAULT_REMOVE_JOB_INFO_DELAY_FOR_NON_ISSUER = 60 +# DEFAULT_GET_TRANSFERINFO_DELAY = 5 +DEFAULT_GET_TRANSFERINFO_DELAY = 1.5 +# DEFAULT_WAIT_FOR_COMPLETION_ATTEMPTS = 360 # 30 minutes +DEFAULT_WAIT_FOR_COMPLETION_ATTEMPTS = 1440 # 2 hours + +Vars = {"buffer":ctypes.create_string_buffer, + "cancelscope":ctypes.c_int32, + "contribid":ctypes.c_uint32, + "devicenum":ctypes.c_uint32, + "flags":ctypes.c_int64, + "errformat":ctypes.c_int32, + "group":ctypes.create_string_buffer, + "handle":ctypes.c_uint64, + "hostname":ctypes.create_string_buffer, + "jobid":ctypes.c_uint64, + "key":ctypes.create_string_buffer, + "mode":ctypes.c_uint32, + "mountpoint":ctypes.create_string_buffer, + "name":ctypes.create_string_buffer, + "newpathname":ctypes.create_string_buffer, + "numavailhandles":ctypes.c_uint64, + "numbytesavailable":ctypes.c_uint64, + "numcontrib":ctypes.c_uint32, + "numhandles":ctypes.c_uint64, + "numtransferdefs":ctypes.c_uint32, + "option":ctypes.create_string_buffer, + "owner":ctypes.create_string_buffer, + "pathname":ctypes.create_string_buffer, + "rate":ctypes.c_uint64, + "size":ctypes.c_uint64, + "size_str":ctypes.create_string_buffer, + "source":ctypes.create_string_buffer, + "status":ctypes.c_int64, + "tag":ctypes.c_ulong, + "target":ctypes.create_string_buffer, + "transferdefs":ctypes.create_string_buffer, + "type":ctypes.create_string_buffer, + "value":ctypes.create_string_buffer, + "variable":ctypes.create_string_buffer, + } + +api = None + + +# +# Helper routines +# + +# Override the format() method for pprint.PrettyPrinter +# so that any unicode data is converted to utf8 +# NOTE: For python3, PrettyPrinter does not handle 'bytes'. +# Therefore, we do not encode type(str). +class bbpprint(pprint.PrettyPrinter): + def format(self, pObject, pContext, pMaxLevels, pLevel): + if False and isinstance(pObject, str): + return (pObject.encode('utf8'), True, False) + else: + return pprint.PrettyPrinter.format(self, pObject, pContext, pMaxLevels, pLevel) + +# NOTE: For python3, ctypes requires 'bytes' for type(str). +# Therefore, type(str) must be encoded. +def cvar(pVar, pValue): + try: + if (type(pValue) == dict): + return Vars[pVar](pValue[pVar]) + elif(type(pValue) == str): + return Vars[pVar](pValue.encode('utf8')) + else: + return Vars[pVar](pValue) + except: + print("Unexpected error when attempting to build a cvar: Type(pVar)=", type(pVar), ", type(pValue)=", type(pValue), ", pVar=", pVar, ", pValue=", pValue, "error=", sys.exc_info()[0]) + return None + +def checkMD5SumFile(pMD5SumFile): + l_RC = 0 + + l_SourceCheckSum = "" + l_Result = [] + for l_Line in open(pMD5SumFile, "r"): + l_Result = l_Line.strip().split() + if len(l_SourceCheckSum) == 0: + l_SourceCheckSum, l_SourceFile = l_Result + else: + if l_SourceCheckSum == l_Result[0]: + print("** SUCCESS ** File %s with checksum of %s matches the checksum for source file %s" % (l_Result[1], l_Result[0], l_SourceFile)) + else: + l_RC = -1 + print("** ERROR ** File %s with checksum of %s mismatches source file %s with checksum of %s" % (l_Result[1], l_Result[0], l_SourceFile, l_SourceCheckSum)) + + return l_RC + +def checkFiles(pSourceFiles, pTargetFiles): + l_RC = 0 + + l_MD5SumFile = "/tmp/md5sum%s" % (random.choice(range(1000000))) + if type(pSourceFiles) in (tuple, list,): + for i in range(len(pSourceFiles)): + if type(pSourceFiles[i]) in (tuple, list,): + for j in range(len(pSourceFiles[i])): + if type(pSourceFiles[i][j]) not in (list, tuple): + runCmd("md5sum %s %s > %s" % (pSourceFiles[i][j], pTargetFiles[i][j], l_MD5SumFile)) + l_RC = checkMD5SumFile(l_MD5SumFile) + else: + for k in range(len(pSourceFiles[i][j])): + runCmd("md5sum %s %s > %s" % (pSourceFiles[i][j][k], pTargetFiles[i][j][k], l_MD5SumFile)) + l_RC = checkMD5SumFile(l_MD5SumFile) + else: + runCmd("md5sum %s %s > %s" % (pSourceFiles[i], pTargetFiles[i], l_MD5SumFile)) + l_RC = checkMD5SumFile(l_MD5SumFile) + else: + runCmd("md5sum %s %s > %s" % (pSourceFiles, pTargetFiles, l_MD5SumFile)) + l_RC = checkMD5SumFile(l_MD5SumFile) + + runCmd("rm %s" % (l_MD5SumFile)) + + if l_RC: + raise bberror.BBError(rc=l_RC, text="MD5Sum check failure") + + return + +def createRandomFile(pEnv, pFile, pSize): + l_RandFilePgm = os.path.abspath(os.path.join(os.path.dirname(pEnv["LIBPATH"]), ".", "tools/randfile")) + l_RemoveFilePgm = "rm" + l_CreateFilePgm = "touch" + if pSize > 0: + runCmd("%s --file %s --size %d" % (l_RandFilePgm, pFile, pSize)) + else: + runCmd("%s %s" % (l_RemoveFilePgm, pFile)) + runCmd("%s %s" % (l_CreateFilePgm, pFile)) + + return + +def flushWaiters(pActiveServer): + # Flush the waiters for the input active server + + if (pActiveServer != ""): + l_StartingValue = 5 + + if (int(BB_GetServerByName(pActiveServer, "waitforreplycount", False)) == 0): + l_Continue = l_StartingValue - 1; + else: + l_Continue = l_StartingValue + l_Attempts = 1 + + while (l_Continue > 0): + time.sleep(0.05) + l_PrintOption = False; + if (l_Attempts % 200 == 0): + l_PrintOption = True + if (int(BB_GetServerByName(pActiveServer, "waitforreplycount", l_PrintOption)) == 0): + l_Continue -= 1; + else: + l_Continue = l_StartingValue + l_Attempts += 1 + + return; + +def getContribId(pPrintValue=DO_NOT_PRINT_VALUE): + return Coral_GetVar('contribid', pPrintValue) + +def getHandles(pStatus=BBSTATUS["BBALL"]): + l_NumAvailHandles = 0 + l_NumHandles = -1 + while (l_NumAvailHandles > l_NumHandles): + l_NumHandles = l_NumAvailHandles + (l_NumAvailHandles, l_Handles) = BB_GetTransferList(pStatus, l_NumHandles) + + print("type(l_NumAvailHandles)=%s, l_NumAvailHandles=%d, type(l_Handles)=%s, l_Handles=%s" % ( + type(l_NumAvailHandles), l_NumAvailHandles, type(l_Handles), l_Handles)) + + print("getHandles: jobid=%d, jobstepid=%d, contribid=%d, l_NumAvailHandles=%d, l_Handles=%s" % ( + getJobId(), getJobStepId(), getContribId(), l_NumAvailHandles, repr(l_Handles))) + + return l_Handles + +def getJobId(pPrintValue=DO_NOT_PRINT_VALUE): + return Coral_GetVar('jobid', pPrintValue) + +def getJobStepId(pPrintValue=DO_NOT_PRINT_VALUE): + return Coral_GetVar('jobstepid', pPrintValue) + +def incrJobStepId(pValue=1): + l_JobStepId = getJobStepId() + l_JobStepId = l_JobStepId + pValue + setJobStepId(l_JobStepId) + + return + +def initEnv(pEnv, pMountpoints=None, pDirectories=None): + # Cleanup from prior variations... + + l_RC = 0 + l_Continue = True; + +# print +# print ">>>> Start: Cleanup..." + + l_Suffix = "Tolerated exception and processing continues..." + l_SuffixEnd = "Non-tolerated exception and processing is ending..." + + # NOTE: The initialization must occur first so that if removejobinfo is + # issued below, the removejobinfo is issued for the correct jobid. + if (l_Continue): +# print ">>>> Start: Initialize environment..." + + l_Iteration = pEnv.get("iteration", 0) + l_JobIdBump = pEnv.get("jobid_bump", 0) + setJobId(pEnv["jobid"] + (l_Iteration*10) + l_JobIdBump) + setJobStepId(pEnv['jobstepid']) + setContribId(pEnv['contribid']) + +# print ">>>> End: Initialize environment..." + else: + l_Continue = False + + if ((pMountpoints is not None) or (pDirectories is not None)): + if l_Continue: + # If single contributor or contribid is 0, + # issue the removejobinfo request. + # + # Whether or not the request was issued, delay + # for a bit waiting for the removejobinfo to be + # processed, and possibly propagated, amongst all + # bbServers. + if len(pEnv['contrib']) == 1 or pEnv['contribid'] == 0: + try: + sudo_RemoveJobInfo(pEnv) + except BBError as error: + l_Continue = error.handleError() + time.sleep(DEFAULT_REMOVE_JOB_INFO_DELAY_FOR_ISSUER) + else: + time.sleep(DEFAULT_REMOVE_JOB_INFO_DELAY_FOR_NON_ISSUER) + + if l_Continue: + if pDirectories is not None: + if type(pDirectories) not in (tuple, list): + try: + l_Directory = pDirectories + sudo_RemoveDirectory(pEnv, l_Directory) + except BBError as error: + l_Continue = error.handleError() + else: + for l_Directory in pDirectories: + if l_Continue: + try: + sudo_RemoveDirectory(pEnv, l_Directory) + except BBError as error: + l_Continue = error.handleError() + if l_Continue: + if pMountpoints is not None: + if type(pMountpoints) not in (tuple, list): + try: + l_Mountpoint = pMountpoints + sudo_RemoveLogicalVolume(pEnv, l_Mountpoint) + except BBError as error: + l_Continue = error.handleError() + if l_Continue: + try: + sudo_RemoveDirectory(pEnv, l_Mountpoint) + except BBError as error: + l_Continue = error.handleError() + else: + for l_Mountpoint in pMountpoints: + if l_Continue: + try: + sudo_RemoveLogicalVolume(pEnv, l_Mountpoint) + except BBError as error: + l_Continue = error.handleError() + if l_Continue: + try: + sudo_RemoveDirectory(pEnv, l_Mountpoint) + except BBError as error: + l_Continue = error.handleError() + if not l_Continue: + l_RC = -1 + else: + l_RC = -1 + else: + l_RC = -1 + else: + l_RC = -1 + +# print ">>>> End: Cleanup..." + + return l_RC + +def printLastErrorDetailsSummary(): + print(datetime.now().strftime("Current date/time: %Y-%m-%d %H:%M:%S")) + dummy = BBError() + print(dummy.getLastErrorDetailsSummary()) + + return + +def printStruct(pStruct): + print("Start: Print of struct %s" % (pStruct)) + for l_FieldName, l_FieldType in pStruct._fields_: + print("%s%s = %s" % (len("Start: ")*" ", l_FieldName, getattr(pStruct, l_FieldName))) + print(" End: Print of struct %s" % (pStruct)) + + return + +def printEnv(pEnv): + print("Start: Print of environment") + bbpprint().pprint(pEnv) + print(" End: Print of environment") + + return + +def runCmd(pCmd): + print(pCmd) + os.system(pCmd) + + return + +# Setting the jobid from a program is not necessary if bbapi_main is invoked +# to run a testcase. Changing the jobid in the middle of a bbapi testcase +# is 'cheating', as the same connection is being used. But, it works to test +# most scenarios... @DLH +def setJobId(pValue): + Coral_SetVar('jobid', repr(pValue)) + + return + +def setJobStepId(pValue): + Coral_SetVar('jobstepid', repr(pValue)) + + return + +# Setting the contribid from a program is not necessary if bbapi_main is invoked +# to run a testcase. Changing the contribid in the middle of a bbapi testcase +# is 'cheating', as the same connection is being used. But, it works to test +# most scenarios... @DLH +def setContribId(pValue): + Coral_SetVar('contribid', repr(pValue)) + + return + +def waitForCompletion(pEnv, pHandles, pAttempts=DEFAULT_WAIT_FOR_COMPLETION_ATTEMPTS): + l_Complete = False + l_Continue = True + l_AllFullSuccess = True; + l_Attempts = 1 + l_LastTotalTransferSize = 0 + l_Status = [] + l_TransferSize = [] + + l_CopyHandles = deepcopy(pHandles) + while (l_Continue and (not l_Complete) and l_Attempts < pAttempts): + l_Complete = True + l_Handles = deepcopy(l_CopyHandles) + for l_Handle in l_Handles: + try: + l_Info = BB_GetTransferInfo(l_Handle) + # NOTE: BBSTOPPED is not included below because we want to 'spin' on a stopped transfer definition, + # waiting for it to progress to another status. + # NOTE: BBPARTIALSUCCESS is not included below because we want to 'spin' on a it also, waiting for + # it to progress back to BBINPROGRESS. It is possible in the restart scenarios if all of the + # extents are processed, but the restart processing hasn't finished yet. + if BBSTATUS[l_Info.status] not in ("BBFULLSUCCESS", "BBCANCELED",): + l_Complete = False + if l_LastTotalTransferSize != l_Info.totalTransferSize: + # Making progress... Reset the number of attempts... + l_LastTotalTransferSize = l_Info.totalTransferSize + l_Attempts = 0 + if len(pEnv["contrib"]) == 1 or bb.getContribId() == 0: + l_Delay = DEFAULT_GET_TRANSFERINFO_DELAY + else: + l_Delay = DEFAULT_GET_TRANSFERINFO_DELAY*2 + time.sleep(l_Delay) + break + else: + if BBSTATUS[l_Info.status] not in ("BBFULLSUCCESS",) or BBSTATUS[l_Info.localstatus] not in ("BBFULLSUCCESS",): + l_AllFullSuccess = False + l_Status.append((BBSTATUS[l_Info.localstatus],BBSTATUS[l_Info.status])) + l_TransferSize.append((l_Info.localTransferSize,l_Info.totalTransferSize)) + l_CopyHandles.remove(l_Handle) + l_Attempts = 0 + except BBError as error: + l_Continue = error.handleError() + if (not l_Continue): + break + l_Attempts += 1 + + if (l_Continue): + if (l_Complete): + if (len(pHandles) > 1): + print() + + for i in range(len(l_Status)): + print(" *FINAL* Handle: %12s -> Status (Local:Overall) (%13s:%13s) Transfer Size in bytes (Local:Total) (%s : %s)" % (pHandles[i], l_Status[i][0], l_Status[i][1], '{:,}'.format(l_TransferSize[i][0]), '{:,}'.format(l_TransferSize[i][1]))) + else: + l_AllFullSuccess = False + print("Exceeded the maximum number of attempts to have all handles reach a status of BBFULLSUCCESS. %d attempts were made." % (pAttempts)) + else: + l_AllFullSuccess = False + print("Error occurred while waiting for all handles reach a status of BBFULLSUCCESS.") + + return l_AllFullSuccess diff --git a/bb/wrappers/bbapi.py b/bb/wrappers/bbapi.py index f509f0bef..0753cb357 100644 --- a/bb/wrappers/bbapi.py +++ b/bb/wrappers/bbapi.py @@ -1,561 +1,586 @@ -#!/usr/bin/python -#################################################### -# bbapi.py -# -# Copyright IBM Corporation 2017,2017. All Rights Reserved -# -# This program is licensed under the terms of the Eclipse Public License -# v1.0 as published by the Eclipse Foundation and available at -# http://www.eclipse.org/legal/epl-v10.html -# -# U.S. Government Users Restricted Rights: Use, duplication or disclosure -# restricted by GSA ADP Schedule Contract with IBM Corp. -################################################### - -""" - This module provides wrappers to the following bbapi functions. - -int BB_AddFiles(BBTransferDef_t* transfer, const char* source, const char* target, BBFILEFLAGS flags) -int BB_AddKeys(BBTransferDef_t* transfer, const char* key, const char* value) -int BB_CancelTransfer(BBTransferHandle_t pHandle, BBCANCELSCOPE pScope) -int BB_CreateTransferDef(BBTransferDef_t** transfer) -int BB_FreeTransferDef(BBTransferDef_t* transfer) -int BB_GetDeviceUsage(uint32_t devicenum, BBDeviceUsage_t* usage) -int BB_GetThrottleRate(const char* mountpoint, uint64_t* rate) -int BB_GetTransferHandle(BBTAG pTag, uint64_t pNumContrib, uint32_t pContrib[], BBTransferHandle_t* pHandle) -int BB_GetTransferInfo(BBTransferHandle_t pHandle, BBTransferInfo_t* pInfo) -int BB_GetTransferKeys(BBTransferHandle_t handle, size_t* buffersize, char* bufferForKeyData) -int BB_GetTransferList(BBSTATUS pMatchStatus, uint64_t* pNumHandles, BBTransferHandle_t pHandles[], uint64_t* pNumAvailHandles) -int BB_GetUsage(const char* mountpoint, BBUsage_t* usage) -int BB_SetThrottleRate(const char* mountpoint, uint64_t rate) -int BB_SetUsageLimit(const char* mountpoint, BBUsage_t* usage) -int BB_StartTransfer(BBTransferDef_t* pTransfer, BBTransferHandle_t pHandle) -""" - -from ctypes import * -# import json - -import bb -from bberror import * -import re -import time - - -# -# Flags/constants for bbapi interfaces -# - -# -# BB_AddFiles BBFILEFLAGS -# -BBFILEFLAGS = { "None" : 0x0000, #### No specification - "BBRecursive" : 0x0001, #### Recursively transfer all files and subdirectories - "BBTestStageIn" : 0x0004, #### Test value to force stagein for the file pair - "BBTestStageOut" : 0x0008, #### Test value to force stageout for the file pair - "BB_BSCFS_Transfer" : 0x0010, #### File transfer type -- BSCFS - "INVALID_BBFILEFLAG" : 0x270F, #### Invalid test value for 'normal' file - "INVALID_BSCFS_BBFILEFLAG" : 0x2710, #### Invalid test value for 'BSCFS' file - 0x0000 : "None" , #### No specification - 0x0001 : "BBRecursive" , #### Recursively transfer all files and subdirectories - 0x0004 : "BBTestStageIn" , #### Test value to force stagein for the file pair - 0x0008 : "BBTestStageOut" , #### Test value to force stageout for the file pair - 0x0010 : "BB_BSCFS_Transfer" , #### Test value to force stagein for the file pair - 0x270F : "INVALID_BBFILEFLAG" , #### Invalid test value for 'normal' file - 0x2710 : "INVALID_BSCFS_BBFILEFLAG" , #### Invalid test value for 'BSCFS' file - } -DEFAULT_BBFILEFLAG = BBFILEFLAGS["None"] -INVALID_BBFILEFLAG = BBFILEFLAGS["INVALID_BBFILEFLAG"] -INVALID_BSCFS_BBFILEFLAG = BBFILEFLAGS["INVALID_BSCFS_BBFILEFLAG"] - -BBFileBSCFS = 0x0010 #### Test value for file transfer order of 0 (highest) -BBFileTransferOrder_0 = 0x0000 #### Test value for file transfer order of 0 (highest) -BBFileTransferOrder_1 = 0x0040 #### Test value for file transfer order of 1 -BBFileTransferOrder_2 = 0x0080 #### Test value for file transfer order of 2 -BBFileTransferOrder_3 = 0x00C0 #### Test value for file transfer order of 3 (lowest) -BBFileBundleId_1 = 0x0100 #### Test value for file bundle id 1 -BBFileBundleId_2 = 0x0200 #### Test value for file bundle id 2 -BBFileBundleId_3 = 0x0300 #### Test value for file bundle id 3 -BBFileBundleId_4 = 0x0400 #### Test value for file bundle id 4 -BBFileBundleId_5 = 0x0500 #### Test value for file bundle id 5 -BBFileBundleId_6 = 0x0600 #### Test value for file bundle id 6 -BBFileBundleId_7 = 0x0700 #### Test value for file bundle id 7 -BBFileBundleId_8 = 0x0800 #### Test value for file bundle id 8 -BBFileBundleId_9 = 0x0900 #### Test value for file bundle id 9 - -# -# BB_CancelTransfer BBCANCELSCOPE -# -BBCANCELSCOPE = {"BBSCOPETRANSFER" : 0x0001, #### Cancel only impacts the local contribution - "BBSCOPETAG" : 0x0002, #### Cancel impacts all transfers associated by the tag - "INVALID" : 0x270F, #### Invalid value - 0x0001 : "BBSCOPETRANSFER", #### Cancel only impacts the local contribution - 0x0002 : "BBSCOPETAG" , #### Cancel impacts all transfers associated by the tag - 0x270F : "INVALID" , #### Invalid value - } -DEFAULT_BBCANCELSCOPE = BBCANCELSCOPE["BBSCOPETAG"] -INVALID_BBCANCELSCOPE = BBCANCELSCOPE["INVALID"] - -# -# BB_GetTransferList BBSTATUS -# -BBSTATUS = {"BBNONE" : 0x0000000000000000, #### No status (used by BB_GetTransferInfo) - "BBNOTSTARTED" : 0x0000000000000001, #### The tag has been defined to the system - #### but none of the contributors have yet - #### reported. - "BBINPROGRESS" : 0x0000000000000002, #### The tag has been defined to the system - #### and data for the associated transfer - #### definitions is actively being transferred. - "BBPARTIALSUCCESS" : 0x0000000000000004, #### Less than the full number of contributors - #### reported for the tag, but all of the - #### data for those reported transfer definitions - #### has been transferred successfully. However, - #### some contributors never reported, and - #### therefore, not all data was transferred. - "BBFULLSUCCESS" : 0x0000000000000008, #### All contributors have reported for the tag - #### and all data for all of the associated transfer - #### definitions has been transferred successfully. - "BBCANCELED" : 0x0000000000000010, #### All data transfers associated with the tag - #### have been cancelled by the user. No or - #### some data may have been transferred. - "BBFAILED" : 0x0000000000000020, #### One or more data transfers have failed for - #### one or more of the associated transfer - #### definitions for the tag. No or some data - #### may have been transferred successfully. - "BBSTOPPED" : 0x0000000000000040, #### One or more data transfers have been stopped for - #### one or more of the associated transfer - #### definitions for the tag. No or some data - #### may have been transferred successfully. - "BBNOTACONTRIB" : 0x0000000000000100, #### The contributor making the API request is - #### NOT in the array of contributors for this tag. - #### Therefore, localStatus and localTransferSize - #### cannot be returned. - "BBNOTREPORTED" : 0x0000000000000200, #### The contributor making the API request is - #### in the array of contributors for this tag. - #### However, it has not yet reported in with it's - #### transfer definition. - "BBALL" : 0xFFFFFFFFFFFFFFFF, #### All status values (used by BB_GetTransferList) - 0x0000000000000000 : "BBNONE" , #### No status (used by BB_GetTransferInfo) - 0x0000000000000001 : "BBNOTSTARTED" , #### The tag has been defined to the system - #### but none of the contributors have yet - #### reported. - 0x0000000000000002 : "BBINPROGRESS" , #### The tag has been defined to the system - #### and data for the associated transfer - #### definitions is actively being transferred. - 0x0000000000000004 : "BBPARTIALSUCCESS", #### Less than the full number of contributors - #### reported for the tag, but all of the - #### data for those reported transfer definitions - #### has been transferred successfully. However, - #### some contributors never reported, and - #### therefore, not all data was transferred. - 0x0000000000000008 : "BBFULLSUCCESS" , #### All contributors have reported for the tag - #### and all data for all of the associated transfer - #### definitions has been transferred successfully. - 0x0000000000000010 : "BBCANCELED" , #### All data transfers associated with the tag - #### have been cancelled by the user. No or - #### some data may have been transferred. - 0x0000000000000020 : "BBFAILED" , #### One or more data transfers have failed for - #### one or more of the associated transfer - #### definitions for the tag. No or some data - #### may have been transferred successfully. - 0x0000000000000040 : "BBSTOPPED" , #### One or more data transfers have been stopped for - #### one or more of the associated transfer - #### definitions for the tag. No or some data - #### may have been transferred successfully. - 0x0000000000000100 : "BBNOTACONTRIB" , #### The contributor making the API request is - #### NOT in the array of contributors for this tag. - #### Therefore, localStatus and localTransferSize - #### cannot be returned. - 0x0000000000000200 : "BBNOTREPORTED" , #### The contributor making the API request is - #### in the array of contributors for this tag. - #### However, it has not yet reported in with it's - #### transfer definition. - 0xFFFFFFFFFFFFFFFF : "BBALL" , #### All status values (used by BB_GetTransferList) - } - - -# -# Flags/constants for bbapiAdmin interfaces -# - -# -# BB_CreateLogicalVolume BBCREATEFLAGS -# -BBCREATEFLAGS = {"BBXFS" : 0x0001, #### Create xfs logical volume - "BBEXT4" : 0x0002, #### Create ext4 logical volume - "BBFSCUSTOM1" : 0x0100, #### Site custom logical value 1 - "BBFSCUSTOM2" : 0x0200, #### Site custom logical value 2 - "BBFSCUSTOM3" : 0x0400, #### Site custom logical value 3 - "BBFSCUSTOM4" : 0x0800, #### Site custom logical value 4 - "INVALID" : 0x270F, #### Invalid value - 0x0001 : "BBXFS" , #### Create xfs logical volume - 0x0002 : "BBEXT4" , #### Create ext4 logical volume - 0x0100 : "BBFSCUSTOM1", #### Site custom logical value 1 - 0x0200 : "BBFSCUSTOM2", #### Site custom logical value 2 - 0x0400 : "BBFSCUSTOM3", #### Site custom logical value 3 - 0x0800 : "BBFSCUSTOM4", #### Site custom logical value 4 - 0x270F : "INVALID" , #### Invalid value - } -DEFAULT_BBCREATEFLAGS = BBCREATEFLAGS["BBXFS"] -INVALID_BBCREATEFLAGS = BBCREATEFLAGS["INVALID"] - -# -# BB_ResizeMountPoint BBRESIZEFLAGS -# -BBRESIZEFLAGS = {"BB_NONE" : 0x0000, #### No resize flag option - "BB_DO_NOT_PRESERVE_FS" : 0x0001, #### Do not preserve file system - "INVALID" : 0x270F, #### Invalid value - 0x0000 : "BB_NONE" , #### No resize flag option - 0x0001 : "BB_DO_NOT_PRESERVE_FS", #### Do not preserve file system - 0x270F : "INVALID" , #### Invalid value - } -DEFAULT_BBRESIZEFLAGS = BBRESIZEFLAGS["BB_NONE"] -INVALID_BBRESIZEFLAGS = BBRESIZEFLAGS["INVALID"] - -# -# BB_RetrieveTransfers -# -BB_RTV_TRANSFERDEFS_FLAGS = {"ALL_DEFINITIONS" : 0x0001, #### All definitions - "ONLY_DEFINITIONS_WITH_UNFINISHED_FILES" : 0x0002, #### Only definitions with unfinished files - "ONLY_DEFINITIONS_WITH_STOPPED_FILES" : 0x0003, #### Only definitions with stopped files - "INVALID" : 0x270F, #### Invalid value - 0x0001 : "ALL_DEFINITIONS", #### All definitions - 0x0002 : "ONLY_DEFINITIONS_WITH_UNFINISHED_FILES", #### Only rebuilt definitions with unfinished files - 0x0003 : "ONLY_DEFINITIONS_WITH_STOPPED_FILES", #### Only rebuilt definitions with stopped files - 0x270F : "INVALID" , #### Invalid value - } -DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS = BB_RTV_TRANSFERDEFS_FLAGS["ONLY_DEFINITIONS_WITH_UNFINISHED_FILES"] -INVALID_BB_RTV_TRANSFERDEFS_FLAGS = BB_RTV_TRANSFERDEFS_FLAGS["INVALID"] - - -# -# Structs -# -class BBUsage_t(Structure): - _fields_ = [("totalBytesRead", c_uint64), #### Number of bytes written to the logical volume - ("totalBytesWritten", c_uint64), #### Number of bytes read from the logical volume - ("localBytesRead", c_uint64), #### Number of bytes written to the logical volume via compute node - ("localBytesWritten", c_uint64), #### Number of bytes read from the logical volume via compute node - ("burstBytesRead", c_uint64), #### Number of bytes written to the logical volume via burst buffer transfers - ("burstBytesWritten", c_uint64), #### Number of bytes read from the logical volume via burst buffer transfers - ("localReadCount", c_uint64), #### Number of read operations to the logical volume - ("localWriteCount", c_uint64),] #### Number of write operations to the logical volume - -class BBDeviceUsage_t(Structure): - _fields_ = [("critical_warning", c_uint64), #### Number of bytes written to the logical volume - ("temperature", c_double), #### Temperature of the SSD in degrees C - ("available_spare", c_double), #### Amount of SSD capacity over-provisioning that remains - ("percentage_used", c_double), #### Estimate of the amount of SSD life consumed. - ("data_read", c_uint64), #### Number of bytes read from the SSD over the life of the device. - ("data_written", c_uint64), #### Number of bytes written to the SSD over the life of the device. - ("num_read_commands", c_uint64), #### Number of I/O read commands received from the compute node. - ("num_write_commands", c_uint64), #### Number of completed I/O write commands from the compute node. - ("busy_time", c_uint64), #### Amount of time that the I/O controller was handling I/O requests. - ("power_cycles", c_uint64), #### Number of power on events for the SSD. - ("power_on_hours", c_uint64), #### Number of hours that the SSD has power. - ("unsafe_shutdowns", c_uint64), #### Number of unexpected power OFF events. - ("media_errors", c_uint64), #### Count of all data unrecoverable events. - ("num_err_log_entries", c_uint64),] #### Number of error log entries available. - -class BBTransferInfo_t(Structure): - _fields_ = [("handle", c_uint64), #### Transfer handle - ("contribid", c_uint32), #### Contributor Id - ("jobid", c_uint64), #### Job Id - ("jobstepid", c_uint64), #### Jobstep Id - ("tag", c_ulong), #### User specified tag - ("numcontrib", c_uint64), #### Number of contributors - ("status", c_int64), #### Current status of the transfer - ("numreportingcontribs", c_uint64), #### Number of reporting contributors - ("totalTransferKeyLength", c_uint64), #### Total number of bytes required to retrieve all - #### key:value pairs using BB_GetTransferKeys() - ("totalTransferSize", c_uint64), #### Total number of bytes for the files associated with - #### all transfer definitions - ("localstatus", c_int64), #### Current status of the transfer for the requesting contributor - ("localTransferSize", c_uint64),] #### Total number of bytes for the files associated with - #### the transfer definition for the requesting contributor - - -# -# Miscellaneous -# - - -# -# Wrappers for official bbapi calls -# - -def BB_AddFiles(pTransferDef, pSource, pTarget, pFlags=DEFAULT_BBFILEFLAG): - l_Source = bb.cvar("source", pSource) - l_Target = bb.cvar("target", pTarget) - l_Flags = bb.cvar("flags", pFlags) - - print "%sBB_AddFiles issued to add source file %s and target file %s with flag %s to transfer definition %s" % (os.linesep, pSource, pTarget, BBFILEFLAGS[l_Flags.value], `pTransferDef`) - rc = bb.api.BB_AddFiles(pTransferDef, l_Source, l_Target, l_Flags) - if (rc): - raise BB_AddFilesError(rc) - - bb.printLastErrorDetailsSummary() - print "Source file %s and target file %s with flag %s added to transfer definition %s" % (pSource, pTarget, BBFILEFLAGS[l_Flags.value], `pTransferDef`) - - return - -def BB_AddKeys(pTransferDef, pKey, pValue): - l_Key = bb.cvar("key", pKey) - l_Value = bb.cvar("value", pValue) - - print "%sBB_AddKeys issued to add key %s, value %s to transfer definition %s" % (os.linesep, pKey, pValue, `pTransferDef`) - rc = bb.api.BB_AddKeys(pTransferDef, l_Key, l_Value) - if (rc): - raise BB_AddKeysError(rc) - - bb.printLastErrorDetailsSummary() - print "Key '%s' with keyvalue '%s' added to transfer definition %s" % (pKey, pValue, `pTransferDef`) - - return - -def BB_CancelTransfer(pHandle, pCancelScope=DEFAULT_BBCANCELSCOPE): - l_NormalRCs = BB_CancelTransferError(BBError(Exception())).getNormalRCs() - l_ToleratedErrorRCs = BB_CancelTransferError(BBError(Exception())).getToleratedErrorRCs() - - l_Handle = bb.cvar("handle", pHandle) - l_CancelScope = bb.cvar("cancelscope", pCancelScope) - - print "%sBB_CancelTransfer issued to initiate cancel for handle %s, with cancel scope of %s" % (os.linesep, pHandle, BBCANCELSCOPE[l_CancelScope.value]) - rc = bb.api.BB_CancelTransfer(l_Handle, l_CancelScope) - if ((rc not in l_NormalRCs) and (rc not in l_ToleratedErrorRCs)): - dummy = BBError() - FIND_INCORRECT_BBSERVER = re.compile(".*A cancel request for an individual transfer definition must be directed to the bbServer servicing that jobid and contribid") - l_ErrorSummary = dummy.getLastErrorDetailsSummary() - l_Success = FIND_INCORRECT_BBSERVER.search(l_ErrorSummary) - if ((not l_Success) or (pCancelScope == DEFAULT_BBCANCELSCOPE)): - raise BB_CancelTransferError(rc) - else: - # NOTE: This could be a 'normal' case where the cancel operation is running simultaneously with a failover operation - # to a new bbServer. Retry the cancel operation... - print "Cancel operation with a cancel scope of BBSCOPETRANSFER was issued to the incorrect bbServer due to concurrent failover processing. Exception was tolerated and cancel operation will not be retried." - rc = -2 - - bb.printLastErrorDetailsSummary() - if (rc == 0): - print "Cancel initiated for handle %s, with cancel scope of %s" % (pHandle, l_CancelScope.value) - - return - -def BB_CreateTransferDef(): - l_TransferDef = POINTER(c_uint32)() - - print "%sBB_CreateTransferDef issued" % (os.linesep) - rc = bb.api.BB_CreateTransferDef(byref(l_TransferDef)) - if (rc): - raise BB_CreateTransferDefError(rc) - - bb.printLastErrorDetailsSummary() - print "Transfer definition %s created" % (`l_TransferDef`) - - return l_TransferDef - -def BB_FreeTransferDef(pTransferDef): - print "%sBB_FreeTransferDef issued for %s" % (os.linesep, `pTransferDef`) - rc = bb.api.BB_FreeTransferDef(pTransferDef) - if (rc): - raise BB_FreeTransferDefError(rc) - - bb.printLastErrorDetailsSummary() - print "Transfer definition %s freed" % (`pTransferDef`) - - return - -def BB_GetDeviceUsage(pDeviceNum): - l_DeviceNum = bb.cvar("devicenum", pDeviceNum) - l_DeviceUsage = create_string_buffer(sizeof(BBDeviceUsage_t)) - - print "%sBB_GetDeviceUsage issued for device number %d" % (os.linesep, pDeviceNum) - rc = bb.api.BB_GetDeviceUsage(l_DeviceNum, byref(l_DeviceUsage)) - if (rc): - raise BB_GetDeviceUsageError(rc) - l_ReturnStruct = BBDeviceUsage_t.from_buffer_copy(l_DeviceUsage) - - bb.printLastErrorDetailsSummary() - print "Estimated percentage of life used for device number %d is %d%%" % (pDeviceNum, getattr(l_ReturnStruct, "percentage_used")) - - return l_ReturnStruct - -def BB_GetThrottleRate(pMountpoint): - l_Mountpoint = bb.cvar("mountpoint", pMountpoint) - l_Rate = bb.cvar("rate", 0) - - l_NormalRCs = BB_GetThrottleRateError(BBError(Exception())).getNormalRCs() - l_ToleratedErrorRCs = BB_GetThrottleRateError(BBError(Exception())).getToleratedErrorRCs() - - print "%sBB_GetThrottleRate issued for mountpoint %s" % (os.linesep, pMountpoint) - while (True): - rc = bb.api.BB_GetThrottleRate(l_Mountpoint, byref(l_Rate)) - if (rc in (l_NormalRCs + l_ToleratedErrorRCs)): - if (rc in l_ToleratedErrorRCs): - print "Retrieving the throttle rate for mountpoint %s cannot be completed at this time. Operation will be retried again in 10 seconds." % (pMountpoint) - time.sleep(10) - else: - break - else: - raise BB_GetThrottleRateError(rc) - - bb.printLastErrorDetailsSummary() - print "Throttle rate for mountpoint %s is %d bytes/sec" % (pMountpoint, l_Rate.value) - - return l_Rate.value - -def BB_GetTransferHandle(pTag, pNumContrib, pContrib): - l_Tag = bb.cvar("tag", pTag) - l_NumContrib = bb.cvar("numcontrib", pNumContrib) - Contrib = c_uint32 * pNumContrib - l_Contrib = Contrib(*pContrib) - l_Handle = bb.cvar("handle", 0) - - print "%sBB_GetTransferHandle issued for job (%d,%d), tag %d, numcontrib %d, contrib %s" % (os.linesep, bb.getJobId(), bb.getJobStepId(), pTag, pNumContrib, `pContrib`) - rc = bb.api.BB_GetTransferHandle(l_Tag, l_NumContrib, l_Contrib, byref(l_Handle)) - if (rc): - raise BB_GetTransferHandleError(rc) - - bb.printLastErrorDetailsSummary() - print "Transfer handle %s obtained, for job (%d,%d), tag %d, numcontrib %d, contrib %s" % (l_Handle.value, bb.getJobId(), bb.getJobStepId(), pTag, pNumContrib, `pContrib`) - - return l_Handle.value - -def BB_GetTransferInfo(pHandle): - l_Handle = bb.cvar("handle", pHandle) - l_Info = create_string_buffer(sizeof(BBTransferInfo_t)) - - print "%sBB_GetTransferInfo issued for handle %s" % (os.linesep, pHandle) - rc = bb.api.BB_GetTransferInfo(l_Handle, byref(l_Info)) - if (rc): - raise BB_GetTransferInfoError(rc) - l_ReturnStruct = BBTransferInfo_t.from_buffer_copy(l_Info) - - bb.printLastErrorDetailsSummary() - print "BB_GetTransferInfo completed%s Handle: %12d -> Status (Local:Overall) (%13s:%13s) Transfer Size in bytes (Local:Total) (%s : %s)" % (os.linesep, l_ReturnStruct.handle, BBSTATUS[l_ReturnStruct.localstatus], BBSTATUS[l_ReturnStruct.status], '{:,}'.format(l_ReturnStruct.localTransferSize), '{:,}'.format(l_ReturnStruct.totalTransferSize)) - - return l_ReturnStruct - -def BB_GetTransferKeys(pHandle, pSize): - l_Handle = bb.cvar("handle", pHandle) - l_Size = bb.cvar("size", pSize) - l_Buffer = bb.cvar("buffer", pSize) - - print "%sBB_GetTransferKeys issued for handle %s, buffer size %d" % (os.linesep, pHandle, pSize) - rc = bb.api.BB_GetTransferKeys(l_Handle, l_Size, byref(l_Buffer)) - if (rc): - raise BB_GetTransferKeysError(rc) - - bb.printLastErrorDetailsSummary() -# print json.dumps(l_Buffer, sort_keys=True, indent=4, separators=(',', ': ')) - - l_Temp = [] - for i in xrange(l_Size.value): - if l_Buffer[i] != '\x00': - l_Temp.append(l_Buffer[i]) - else: - break - l_Output = "".join(l_Temp) - print "Transfer keys: %s" % (l_Output) - - return l_Output - -def BB_GetTransferList(pMatchStatus, pNumHandles): - l_MatchStatus = bb.cvar("status", pMatchStatus) - l_NumHandles = bb.cvar("numhandles", pNumHandles) - l_Handles = create_string_buffer(sizeof(c_uint64*pNumHandles)) - l_NumAvailHandles = bb.cvar("numavailhandles", 0) - l_MatchStatusStr = BBSTATUS.get(pMatchStatus, `pMatchStatus`) - - print "%sBB_GetTransferList issued with match status of %s" % (os.linesep, l_MatchStatusStr) - rc = bb.api.BB_GetTransferList(l_MatchStatus, byref(l_NumHandles), byref(l_Handles), byref(l_NumAvailHandles)) - if (rc): - raise BB_GetTransferListError(rc) - - l_HandleCount = min(pNumHandles, l_NumAvailHandles.value) - l_HandleArray = (c_ulonglong*l_HandleCount).from_address(addressof(l_Handles)) - l_Output = map(int, l_HandleArray) -# for i in xrange(l_HandleCount): -# l_Output.append(l_HandleArray[i]) - - bb.printLastErrorDetailsSummary() - print "BB_GetTransferList completed, %d handle(s) available, %s" % (l_NumAvailHandles.value, l_Output) - - return (l_NumAvailHandles.value, l_Output) - -def BB_GetUsage(pMountpoint): - l_Mountpoint = bb.cvar("mountpoint", pMountpoint) - l_Usage = create_string_buffer(sizeof(BBUsage_t)) - - print "%sBB_GetUsage issued for mountpoint %s" % (os.linesep, pMountpoint) - rc = bb.api.BB_GetUsage(l_Mountpoint, byref(l_Usage)) - if (rc): - raise BB_GetUsageError(rc) - - bb.printLastErrorDetailsSummary() - l_ReturnStruct = BBUsage_t.from_buffer_copy(l_Usage) - print "For mount point %s, the following are the current usage values:" % (pMountpoint) - print " Total bytes read = %d, Total bytes written = %d" % (getattr(l_ReturnStruct, "totalBytesRead"), getattr(l_ReturnStruct, "totalBytesWritten")) - print " Local bytes read = %d, Local bytes written = %d" % (getattr(l_ReturnStruct, "localBytesRead"), getattr(l_ReturnStruct, "localBytesWritten")) - print " Burst bytes read = %d, Burst bytes written = %d" % (getattr(l_ReturnStruct, "burstBytesRead"), getattr(l_ReturnStruct, "burstBytesWritten")) - - return l_ReturnStruct - -def BB_SetThrottleRate(pMountpoint, pRate): - l_Mountpoint = bb.cvar("mountpoint", pMountpoint) - l_Rate = bb.cvar("rate", pRate) - - print "%sBB_SetThrottleRate issued to set the throttle rate for mountpoint %s to %d bytes/sec" % (os.linesep, pMountpoint, pRate) - rc = bb.api.BB_SetThrottleRate(l_Mountpoint, l_Rate) - if (rc): - raise BB_SetThrottleRateError(rc) - - bb.printLastErrorDetailsSummary() - print "Throttle rate for mountpoint %s changed to %d bytes/sec" % (pMountpoint, pRate) - - return - -def BB_SetUsageLimit(pMountpoint, pUsage): - l_Mountpoint = bb.cvar("mountpoint", pMountpoint) - - print "%sBB_SetUsageLimit issued to set the usage limit for mountpoint %s to %d" % (os.linesep, pMountpoint, pUsage) - rc = bb.api.BB_SetUsage(l_Mountpoint, byref(pUsage)) - if (rc): - raise BB_SetUsageError(rc) - - bb.printLastErrorDetailsSummary() - print "For mount point %s, the following usage limits were set:" % (pMountpoint) - print " Total bytes read = %d, Total bytes written = %d" % (getattr(pUsage, "totalBytesRead"), getattr(pUsage, "totalBytesWritten")) - print " Local bytes read = %d, Local bytes written = %d" % (getattr(pUsage, "localBytesRead"), getattr(pUsage, "localBytesWritten")) - print " Burst bytes read = %d, Burst bytes written = %d" % (getattr(pUsage, "burstBytesRead"), getattr(pUsage, "burstBytesWritten")) - - return - -def BB_StartTransfer(pTransferDef, pHandle): - l_NormalRCs = BB_StartTransferError(BBError(Exception())).getNormalRCs() - l_ToleratedErrorRCs = BB_StartTransferError(BBError(Exception())).getToleratedErrorRCs() - - l_Handle = bb.cvar("handle", pHandle) - - print "%sBB_StartTransfer issued to start the transfer %s for handle %s" % (os.linesep, `pTransferDef`, pHandle) - - while (True): - rc = bb.api.BB_StartTransfer(pTransferDef, l_Handle) - if (rc in (l_NormalRCs + l_ToleratedErrorRCs)): - if (rc in l_ToleratedErrorRCs): - dummy = BBError() - if ("Attempt to retry" in dummy.getLastErrorDetailsSummary()): - print "Transfer %s cannot be started for handle %s because of a suspended condition. This start transfer request will be attempted again in 15 seconds." % (`pTransferDef`, pHandle) - time.sleep(15) - else: - print "Transfer %s cannot be started for handle %s because of a suspended condition. Restart logic will resubmit this start transfer operation." % (`pTransferDef`, pHandle) - break - else: - break - else: - raise BB_StartTransferError(rc) - - bb.printLastErrorDetailsSummary() - if (rc == 0): - print "Transfer %s started for handle %s" % (`pTransferDef`, pHandle) - - return +#!/usr/bin/python +#################################################### +# bbapi.py +# +# Copyright IBM Corporation 2017,2017. All Rights Reserved +# +# This program is licensed under the terms of the Eclipse Public License +# v1.0 as published by the Eclipse Foundation and available at +# http://www.eclipse.org/legal/epl-v10.html +# +# U.S. Government Users Restricted Rights: Use, duplication or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. +################################################### + +""" + This module provides wrappers to the following bbapi functions. + +int BB_AddFiles(BBTransferDef_t* transfer, const char* source, const char* target, BBFILEFLAGS flags) +int BB_AddKeys(BBTransferDef_t* transfer, const char* key, const char* value) +int BB_CancelTransfer(BBTransferHandle_t pHandle, BBCANCELSCOPE pScope) +int BB_CreateTransferDef(BBTransferDef_t** transfer) +int BB_FreeTransferDef(BBTransferDef_t* transfer) +int BB_GetDeviceUsage(uint32_t devicenum, BBDeviceUsage_t* usage) +int BB_GetThrottleRate(const char* mountpoint, uint64_t* rate) +int BB_GetTransferHandle(BBTAG pTag, uint64_t pNumContrib, uint32_t pContrib[], BBTransferHandle_t* pHandle) +int BB_GetTransferInfo(BBTransferHandle_t pHandle, BBTransferInfo_t* pInfo) +int BB_GetTransferKeys(BBTransferHandle_t handle, size_t* buffersize, char* bufferForKeyData) +int BB_GetTransferList(BBSTATUS pMatchStatus, uint64_t* pNumHandles, BBTransferHandle_t pHandles[], uint64_t* pNumAvailHandles) +int BB_GetUsage(const char* mountpoint, BBUsage_t* usage) +int BB_SetThrottleRate(const char* mountpoint, uint64_t rate) +int BB_SetUsageLimit(const char* mountpoint, BBUsage_t* usage) +int BB_StartTransfer(BBTransferDef_t* pTransfer, BBTransferHandle_t pHandle) +""" + +from ctypes import * +# import json + +import bb +from bberror import * +import re +import time + + +# +# Flags/constants for bbapi interfaces +# + +# +# BB_AddFiles BBFILEFLAGS +# +BBFILEFLAGS = { "None" : 0x0000, #### No specification + "BBRecursive" : 0x0001, #### Recursively transfer all files and subdirectories + "BBTestStageIn" : 0x0004, #### Test value to force stagein for the file pair + "BBTestStageOut" : 0x0008, #### Test value to force stageout for the file pair + "BB_BSCFS_Transfer" : 0x0010, #### File transfer type -- BSCFS + "INVALID_BBFILEFLAG" : 0x270F, #### Invalid test value for 'normal' file + "INVALID_BSCFS_BBFILEFLAG" : 0x2710, #### Invalid test value for 'BSCFS' file + 0x0000 : "None" , #### No specification + 0x0001 : "BBRecursive" , #### Recursively transfer all files and subdirectories + 0x0004 : "BBTestStageIn" , #### Test value to force stagein for the file pair + 0x0008 : "BBTestStageOut" , #### Test value to force stageout for the file pair + 0x0010 : "BB_BSCFS_Transfer" , #### Test value to force stagein for the file pair + 0x270F : "INVALID_BBFILEFLAG" , #### Invalid test value for 'normal' file + 0x2710 : "INVALID_BSCFS_BBFILEFLAG" , #### Invalid test value for 'BSCFS' file + } +DEFAULT_BBFILEFLAG = BBFILEFLAGS["None"] +INVALID_BBFILEFLAG = BBFILEFLAGS["INVALID_BBFILEFLAG"] +INVALID_BSCFS_BBFILEFLAG = BBFILEFLAGS["INVALID_BSCFS_BBFILEFLAG"] + +BBFileBSCFS = 0x0010 #### Test value for file transfer order of 0 (highest) +BBFileTransferOrder_0 = 0x0000 #### Test value for file transfer order of 0 (highest) +BBFileTransferOrder_1 = 0x0040 #### Test value for file transfer order of 1 +BBFileTransferOrder_2 = 0x0080 #### Test value for file transfer order of 2 +BBFileTransferOrder_3 = 0x00C0 #### Test value for file transfer order of 3 (lowest) +BBFileBundleId_1 = 0x0100 #### Test value for file bundle id 1 +BBFileBundleId_2 = 0x0200 #### Test value for file bundle id 2 +BBFileBundleId_3 = 0x0300 #### Test value for file bundle id 3 +BBFileBundleId_4 = 0x0400 #### Test value for file bundle id 4 +BBFileBundleId_5 = 0x0500 #### Test value for file bundle id 5 +BBFileBundleId_6 = 0x0600 #### Test value for file bundle id 6 +BBFileBundleId_7 = 0x0700 #### Test value for file bundle id 7 +BBFileBundleId_8 = 0x0800 #### Test value for file bundle id 8 +BBFileBundleId_9 = 0x0900 #### Test value for file bundle id 9 + +# +# BB_CancelTransfer BBCANCELSCOPE +# +BBCANCELSCOPE = {"BBSCOPETRANSFER" : 0x0001, #### Cancel only impacts the local contribution + "BBSCOPETAG" : 0x0002, #### Cancel impacts all transfers associated by the tag + "INVALID" : 0x270F, #### Invalid value + 0x0001 : "BBSCOPETRANSFER", #### Cancel only impacts the local contribution + 0x0002 : "BBSCOPETAG" , #### Cancel impacts all transfers associated by the tag + 0x270F : "INVALID" , #### Invalid value + } +DEFAULT_BBCANCELSCOPE = BBCANCELSCOPE["BBSCOPETAG"] +INVALID_BBCANCELSCOPE = BBCANCELSCOPE["INVALID"] + +# +# BB_GetTransferList BBSTATUS +# +BBSTATUS = {"BBNONE" : 0x0000000000000000, #### No status (used by BB_GetTransferInfo) + "BBNOTSTARTED" : 0x0000000000000001, #### The tag has been defined to the system + #### but none of the contributors have yet + #### reported. + "BBINPROGRESS" : 0x0000000000000002, #### The tag has been defined to the system + #### and data for the associated transfer + #### definitions is actively being transferred. + "BBPARTIALSUCCESS" : 0x0000000000000004, #### Less than the full number of contributors + #### reported for the tag, but all of the + #### data for those reported transfer definitions + #### has been transferred successfully. However, + #### some contributors never reported, and + #### therefore, not all data was transferred. + "BBFULLSUCCESS" : 0x0000000000000008, #### All contributors have reported for the tag + #### and all data for all of the associated transfer + #### definitions has been transferred successfully. + "BBCANCELED" : 0x0000000000000010, #### All data transfers associated with the tag + #### have been cancelled by the user. No or + #### some data may have been transferred. + "BBFAILED" : 0x0000000000000020, #### One or more data transfers have failed for + #### one or more of the associated transfer + #### definitions for the tag. No or some data + #### may have been transferred successfully. + "BBSTOPPED" : 0x0000000000000040, #### One or more data transfers have been stopped for + #### one or more of the associated transfer + #### definitions for the tag. No or some data + #### may have been transferred successfully. + "BBNOTACONTRIB" : 0x0000000000000100, #### The contributor making the API request is + #### NOT in the array of contributors for this tag. + #### Therefore, localStatus and localTransferSize + #### cannot be returned. + "BBNOTREPORTED" : 0x0000000000000200, #### The contributor making the API request is + #### in the array of contributors for this tag. + #### However, it has not yet reported in with it's + #### transfer definition. + "BBALL" : 0xFFFFFFFFFFFFFFFF, #### All status values (used by BB_GetTransferList) + 0x0000000000000000 : "BBNONE" , #### No status (used by BB_GetTransferInfo) + 0x0000000000000001 : "BBNOTSTARTED" , #### The tag has been defined to the system + #### but none of the contributors have yet + #### reported. + 0x0000000000000002 : "BBINPROGRESS" , #### The tag has been defined to the system + #### and data for the associated transfer + #### definitions is actively being transferred. + 0x0000000000000004 : "BBPARTIALSUCCESS", #### Less than the full number of contributors + #### reported for the tag, but all of the + #### data for those reported transfer definitions + #### has been transferred successfully. However, + #### some contributors never reported, and + #### therefore, not all data was transferred. + 0x0000000000000008 : "BBFULLSUCCESS" , #### All contributors have reported for the tag + #### and all data for all of the associated transfer + #### definitions has been transferred successfully. + 0x0000000000000010 : "BBCANCELED" , #### All data transfers associated with the tag + #### have been cancelled by the user. No or + #### some data may have been transferred. + 0x0000000000000020 : "BBFAILED" , #### One or more data transfers have failed for + #### one or more of the associated transfer + #### definitions for the tag. No or some data + #### may have been transferred successfully. + 0x0000000000000040 : "BBSTOPPED" , #### One or more data transfers have been stopped for + #### one or more of the associated transfer + #### definitions for the tag. No or some data + #### may have been transferred successfully. + 0x0000000000000100 : "BBNOTACONTRIB" , #### The contributor making the API request is + #### NOT in the array of contributors for this tag. + #### Therefore, localStatus and localTransferSize + #### cannot be returned. + 0x0000000000000200 : "BBNOTREPORTED" , #### The contributor making the API request is + #### in the array of contributors for this tag. + #### However, it has not yet reported in with it's + #### transfer definition. + 0xFFFFFFFFFFFFFFFF : "BBALL" , #### All status values (used by BB_GetTransferList) + } + + +# +# Flags/constants for bbapiAdmin interfaces +# + +# +# BB_CreateLogicalVolume BBCREATEFLAGS +# +BBCREATEFLAGS = {"BBXFS" : 0x0001, #### Create xfs logical volume + "BBEXT4" : 0x0002, #### Create ext4 logical volume + "BBFSCUSTOM1" : 0x0100, #### Site custom logical value 1 + "BBFSCUSTOM2" : 0x0200, #### Site custom logical value 2 + "BBFSCUSTOM3" : 0x0400, #### Site custom logical value 3 + "BBFSCUSTOM4" : 0x0800, #### Site custom logical value 4 + "INVALID" : 0x270F, #### Invalid value + 0x0001 : "BBXFS" , #### Create xfs logical volume + 0x0002 : "BBEXT4" , #### Create ext4 logical volume + 0x0100 : "BBFSCUSTOM1", #### Site custom logical value 1 + 0x0200 : "BBFSCUSTOM2", #### Site custom logical value 2 + 0x0400 : "BBFSCUSTOM3", #### Site custom logical value 3 + 0x0800 : "BBFSCUSTOM4", #### Site custom logical value 4 + 0x270F : "INVALID" , #### Invalid value + } +DEFAULT_BBCREATEFLAGS = BBCREATEFLAGS["BBXFS"] +INVALID_BBCREATEFLAGS = BBCREATEFLAGS["INVALID"] + +# +# BB_ResizeMountPoint BBRESIZEFLAGS +# +BBRESIZEFLAGS = {"BB_NONE" : 0x0000, #### No resize flag option + "BB_DO_NOT_PRESERVE_FS" : 0x0001, #### Do not preserve file system + "INVALID" : 0x270F, #### Invalid value + 0x0000 : "BB_NONE" , #### No resize flag option + 0x0001 : "BB_DO_NOT_PRESERVE_FS", #### Do not preserve file system + 0x270F : "INVALID" , #### Invalid value + } +DEFAULT_BBRESIZEFLAGS = BBRESIZEFLAGS["BB_NONE"] +INVALID_BBRESIZEFLAGS = BBRESIZEFLAGS["INVALID"] + +# +# BB_RetrieveTransfers +# +BB_RTV_TRANSFERDEFS_FLAGS = {"ALL_DEFINITIONS" : 0x0001, #### All definitions + "ONLY_DEFINITIONS_WITH_UNFINISHED_FILES" : 0x0002, #### Only definitions with unfinished files + "ONLY_DEFINITIONS_WITH_STOPPED_FILES" : 0x0003, #### Only definitions with stopped files + "INVALID" : 0x270F, #### Invalid value + 0x0001 : "ALL_DEFINITIONS", #### All definitions + 0x0002 : "ONLY_DEFINITIONS_WITH_UNFINISHED_FILES", #### Only rebuilt definitions with unfinished files + 0x0003 : "ONLY_DEFINITIONS_WITH_STOPPED_FILES", #### Only rebuilt definitions with stopped files + 0x270F : "INVALID" , #### Invalid value + } +DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS = BB_RTV_TRANSFERDEFS_FLAGS["ONLY_DEFINITIONS_WITH_UNFINISHED_FILES"] +INVALID_BB_RTV_TRANSFERDEFS_FLAGS = BB_RTV_TRANSFERDEFS_FLAGS["INVALID"] + + +# +# Structs +# +class BBUsage_t(Structure): + _fields_ = [("totalBytesRead", c_uint64), #### Number of bytes written to the logical volume + ("totalBytesWritten", c_uint64), #### Number of bytes read from the logical volume + ("localBytesRead", c_uint64), #### Number of bytes written to the logical volume via compute node + ("localBytesWritten", c_uint64), #### Number of bytes read from the logical volume via compute node + ("burstBytesRead", c_uint64), #### Number of bytes written to the logical volume via burst buffer transfers + ("burstBytesWritten", c_uint64), #### Number of bytes read from the logical volume via burst buffer transfers + ("localReadCount", c_uint64), #### Number of read operations to the logical volume + ("localWriteCount", c_uint64),] #### Number of write operations to the logical volume + +class BBDeviceUsage_t(Structure): + _fields_ = [("critical_warning", c_uint64), #### Number of bytes written to the logical volume + ("temperature", c_double), #### Temperature of the SSD in degrees C + ("available_spare", c_double), #### Amount of SSD capacity over-provisioning that remains + ("percentage_used", c_double), #### Estimate of the amount of SSD life consumed. + ("data_read", c_uint64), #### Number of bytes read from the SSD over the life of the device. + ("data_written", c_uint64), #### Number of bytes written to the SSD over the life of the device. + ("num_read_commands", c_uint64), #### Number of I/O read commands received from the compute node. + ("num_write_commands", c_uint64), #### Number of completed I/O write commands from the compute node. + ("busy_time", c_uint64), #### Amount of time that the I/O controller was handling I/O requests. + ("power_cycles", c_uint64), #### Number of power on events for the SSD. + ("power_on_hours", c_uint64), #### Number of hours that the SSD has power. + ("unsafe_shutdowns", c_uint64), #### Number of unexpected power OFF events. + ("media_errors", c_uint64), #### Count of all data unrecoverable events. + ("num_err_log_entries", c_uint64),] #### Number of error log entries available. + +class BBTransferInfo_t(Structure): + _fields_ = [("handle", c_uint64), #### Transfer handle + ("contribid", c_uint32), #### Contributor Id + ("jobid", c_uint64), #### Job Id + ("jobstepid", c_uint64), #### Jobstep Id + ("tag", c_ulong), #### User specified tag + ("numcontrib", c_uint64), #### Number of contributors + ("status", c_int64), #### Current status of the transfer + ("numreportingcontribs", c_uint64), #### Number of reporting contributors + ("totalTransferKeyLength", c_uint64), #### Total number of bytes required to retrieve all + #### key:value pairs using BB_GetTransferKeys() + ("totalTransferSize", c_uint64), #### Total number of bytes for the files associated with + #### all transfer definitions + ("localstatus", c_int64), #### Current status of the transfer for the requesting contributor + ("localTransferSize", c_uint64),] #### Total number of bytes for the files associated with + #### the transfer definition for the requesting contributor + + +# +# Miscellaneous +# + + +# +# Wrappers for official bbapi calls +# + +def BB_AddFiles(pTransferDef, pSource, pTarget, pFlags=DEFAULT_BBFILEFLAG): + l_Source = bb.cvar("source", pSource) + l_Target = bb.cvar("target", pTarget) + l_Flags = bb.cvar("flags", pFlags) + + print("%sBB_AddFiles issued to add source file %s and target file %s with flag %s to transfer definition %s" % (os.linesep, pSource, pTarget, BBFILEFLAGS[l_Flags.value], repr(pTransferDef))) + rc = bb.api.BB_AddFiles(pTransferDef, l_Source, l_Target, l_Flags) + if (rc): + raise BB_AddFilesError(rc) + + bb.printLastErrorDetailsSummary() + print("Source file %s and target file %s with flag %s added to transfer definition %s" % (pSource, pTarget, BBFILEFLAGS[l_Flags.value], repr(pTransferDef))) + + return + +def BB_AddKeys(pTransferDef, pKey, pValue): + l_Key = bb.cvar("key", pKey) + l_Value = bb.cvar("value", pValue) + + print("%sBB_AddKeys issued to add key %s, value %s to transfer definition %s" % (os.linesep, pKey, pValue, repr(pTransferDef))) + rc = bb.api.BB_AddKeys(pTransferDef, l_Key, l_Value) + if (rc): + raise BB_AddKeysError(rc) + + bb.printLastErrorDetailsSummary() + print("Key '%s' with keyvalue '%s' added to transfer definition %s" % (pKey, pValue, repr(pTransferDef))) + + return + +def BB_CancelTransfer(pHandle, pCancelScope=DEFAULT_BBCANCELSCOPE): + l_NormalRCs = BB_CancelTransferError(BBError(Exception())).getNormalRCs() + l_ToleratedErrorRCs = BB_CancelTransferError(BBError(Exception())).getToleratedErrorRCs() + + l_Handle = bb.cvar("handle", pHandle) + l_CancelScope = bb.cvar("cancelscope", pCancelScope) + + print("%sBB_CancelTransfer issued to initiate cancel for handle %s, with cancel scope of %s" % (os.linesep, pHandle, BBCANCELSCOPE[l_CancelScope.value])) + rc = bb.api.BB_CancelTransfer(l_Handle, l_CancelScope) + if ((rc not in l_NormalRCs) and (rc not in l_ToleratedErrorRCs)): + dummy = BBError() + FIND_INCORRECT_BBSERVER = re.compile(".*A cancel request for an individual transfer definition must be directed to the bbServer servicing that jobid and contribid") + l_ErrorSummary = dummy.getLastErrorDetailsSummary() + l_Success = FIND_INCORRECT_BBSERVER.search(l_ErrorSummary) + if ((not l_Success) or (pCancelScope == DEFAULT_BBCANCELSCOPE)): + raise BB_CancelTransferError(rc) + else: + # NOTE: This could be a 'normal' case where the cancel operation is running simultaneously with a failover operation + # to a new bbServer. Retry the cancel operation... + print("Cancel operation with a cancel scope of BBSCOPETRANSFER was issued to the incorrect bbServer due to concurrent failover processing. Exception was tolerated and cancel operation will not be retried.") + rc = -2 + + bb.printLastErrorDetailsSummary() + if (rc == 0): + print("Cancel initiated for handle %s, with cancel scope of %s" % (pHandle, l_CancelScope.value)) + + return + +def BB_CreateTransferDef(): + l_TransferDef = POINTER(c_uint32)() + + print("%sBB_CreateTransferDef issued" % (os.linesep)) + rc = bb.api.BB_CreateTransferDef(byref(l_TransferDef)) + if (rc): + raise BB_CreateTransferDefError(rc) + + bb.printLastErrorDetailsSummary() + print("Transfer definition %s created" % (repr(l_TransferDef))) + + return l_TransferDef + +def BB_FreeTransferDef(pTransferDef): + print("%sBB_FreeTransferDef issued for %s" % (os.linesep, repr(pTransferDef))) + rc = bb.api.BB_FreeTransferDef(pTransferDef) + if (rc): + raise BB_FreeTransferDefError(rc) + + bb.printLastErrorDetailsSummary() + print("Transfer definition %s freed" % (repr(pTransferDef))) + + return + +def BB_GetDeviceUsage(pDeviceNum): + l_DeviceNum = bb.cvar("devicenum", pDeviceNum) + l_DeviceUsage = create_string_buffer(sizeof(BBDeviceUsage_t)) + + print("%sBB_GetDeviceUsage issued for device number %d" % (os.linesep, pDeviceNum)) + rc = bb.api.BB_GetDeviceUsage(l_DeviceNum, byref(l_DeviceUsage)) + if (rc): + raise BB_GetDeviceUsageError(rc) + l_ReturnStruct = BBDeviceUsage_t.from_buffer_copy(l_DeviceUsage) + + bb.printLastErrorDetailsSummary() + print("Estimated percentage of life used for device number %d is %d%%" % (pDeviceNum, getattr(l_ReturnStruct, "percentage_used"))) + + return l_ReturnStruct + +def BB_GetThrottleRate(pMountpoint): + l_Mountpoint = bb.cvar("mountpoint", pMountpoint) + l_Rate = bb.cvar("rate", 0) + + l_NormalRCs = BB_GetThrottleRateError(BBError(Exception())).getNormalRCs() + l_ToleratedErrorRCs = BB_GetThrottleRateError(BBError(Exception())).getToleratedErrorRCs() + + print("%sBB_GetThrottleRate issued for mountpoint %s" % (os.linesep, pMountpoint)) + while (True): + rc = bb.api.BB_GetThrottleRate(l_Mountpoint, byref(l_Rate)) + if (rc in (l_NormalRCs + l_ToleratedErrorRCs)): + if (rc in l_ToleratedErrorRCs): + print("Retrieving the throttle rate for mountpoint %s cannot be completed at this time. Operation will be retried again in 10 seconds." % (pMountpoint)) + time.sleep(10) + else: + break + else: + raise BB_GetThrottleRateError(rc) + + bb.printLastErrorDetailsSummary() + print("Throttle rate for mountpoint %s is %d bytes/sec" % (pMountpoint, l_Rate.value)) + + return l_Rate.value + +def BB_GetTransferHandle(pTag, pNumContrib, pContrib): + l_Tag = bb.cvar("tag", pTag) + l_NumContrib = bb.cvar("numcontrib", pNumContrib) + Contrib = c_uint32 * pNumContrib + l_Contrib = Contrib(*pContrib) + l_Handle = bb.cvar("handle", 0) + + print("%sBB_GetTransferHandle issued for job (%d,%d), tag %d, numcontrib %d, contrib %s" % (os.linesep, bb.getJobId(), bb.getJobStepId(), pTag, pNumContrib, repr(pContrib))) + rc = bb.api.BB_GetTransferHandle(l_Tag, l_NumContrib, l_Contrib, byref(l_Handle)) + if (rc): + raise BB_GetTransferHandleError(rc) + + bb.printLastErrorDetailsSummary() + print("Transfer handle %s obtained, for job (%d,%d), tag %d, numcontrib %d, contrib %s" % (l_Handle.value, bb.getJobId(), bb.getJobStepId(), pTag, pNumContrib, repr(pContrib))) + + return l_Handle.value + +def BB_GetTransferInfo(pHandle): + l_Handle = bb.cvar("handle", pHandle) + l_Info = create_string_buffer(sizeof(BBTransferInfo_t)) + + print("%sBB_GetTransferInfo issued for handle %s" % (os.linesep, pHandle)) + rc = bb.api.BB_GetTransferInfo(l_Handle, byref(l_Info)) + if (rc): + raise BB_GetTransferInfoError(rc) + l_ReturnStruct = BBTransferInfo_t.from_buffer_copy(l_Info) + + bb.printLastErrorDetailsSummary() + print("BB_GetTransferInfo completed%s Handle: %12d -> Status (Local:Overall) (%13s:%13s) Transfer Size in bytes (Local:Total) (%s : %s)" % (os.linesep, l_ReturnStruct.handle, BBSTATUS[l_ReturnStruct.localstatus], BBSTATUS[l_ReturnStruct.status], '{:,}'.format(l_ReturnStruct.localTransferSize), '{:,}'.format(l_ReturnStruct.totalTransferSize))) + + return l_ReturnStruct + +def BB_GetTransferKeys(pHandle, pSize): + l_Handle = bb.cvar("handle", pHandle) + l_Size = bb.cvar("size", pSize) + l_Buffer = bb.cvar("buffer", pSize) + + print("%sBB_GetTransferKeys issued for handle %s, buffer size %d" % (os.linesep, pHandle, pSize)) + rc = bb.api.BB_GetTransferKeys(l_Handle, l_Size, byref(l_Buffer)) + if (rc): + raise BB_GetTransferKeysError(rc) + + bb.printLastErrorDetailsSummary() +# print json.dumps(l_Buffer, sort_keys=True, indent=4, separators=(',', ': ')) + + l_Temp = [] + for i in range(l_Size.value): + if l_Buffer[i] != b'\0': + l_Temp.append(l_Buffer[i]) + else: + break + l_Output = (b"".join(l_Temp)).decode() + print("Transfer keys: %s" % (l_Output)) + + return l_Output + +def BB_GetTransferList(pMatchStatus, pNumHandles): + l_MatchStatus = bb.cvar("status", pMatchStatus) + l_NumHandles = bb.cvar("numhandles", pNumHandles) +# l_Handles = create_string_buffer(sizeof(c_uint64)*pNumHandles) + l_Handles = create_string_buffer(int(((max((sizeof(c_uint64)*pNumHandles),1))+63)/64)*64) + l_NumAvailHandles = bb.cvar("numavailhandles", 0) + l_MatchStatusStr = BBSTATUS.get(pMatchStatus, repr(pMatchStatus)) + + print("%sBB_GetTransferList issued with match status of %s" % (os.linesep, l_MatchStatusStr)) + rc = bb.api.BB_GetTransferList(l_MatchStatus, byref(l_NumHandles), byref(l_Handles), byref(l_NumAvailHandles)) + if (rc): + raise BB_GetTransferListError(rc) + +# NOTE: We are failing atttempting to get the handle values from the string buffer. +# For now, we get the handle values from BBError... + ''' + l_HandleCount = min(pNumHandles, l_NumAvailHandles.value) + l_HandleArray = (c_ulonglong*l_HandleCount).from_address(addressof(l_Handles)) + l_Output = [l_HandleArray[i] for i in range(l_HandleCount)] + ''' +# l_Output = list(map(int, l_HandleArray)) +# print(l_Output) +# for i in range(l_HandleCount): +# l_Output.append(l_HandleArray[i]) + + bb.printLastErrorDetailsSummary() + + l_Output = [] + if pNumHandles > 0: + l_BBError = BBError() + l_BBErrorData = l_BBError.getLastErrorDetails() + if "out" in l_BBErrorData.keys(): + for l_Key, l_Value in l_BBErrorData["out"].items(): + if l_Key[0:7] == "handle_": + if l_Value.isdigit(): + l_Output.append(int(l_Value)) + else: + raise BB_GetTransferListError(rc=-1, text="Handle value %s from BBError is invalid" % (l_Value)) + else: + raise BB_GetTransferListError(rc=-1, text="Key 'out' not found in BBError") + + if l_NumAvailHandles.value != len(l_Output): + raise BB_GetTransferListError(rc=-1, text="Wrong number of handle values returned in BBError") + + print("BB_GetTransferList completed, %d handle(s) available, %s" % (l_NumAvailHandles.value, l_Output)) + + return (l_NumAvailHandles.value, l_Output) + +def BB_GetUsage(pMountpoint): + l_Mountpoint = bb.cvar("mountpoint", pMountpoint) + l_Usage = create_string_buffer(sizeof(BBUsage_t)) + + print("%sBB_GetUsage issued for mountpoint %s" % (os.linesep, pMountpoint)) + rc = bb.api.BB_GetUsage(l_Mountpoint, byref(l_Usage)) + if (rc): + raise BB_GetUsageError(rc) + + bb.printLastErrorDetailsSummary() + l_ReturnStruct = BBUsage_t.from_buffer_copy(l_Usage) + print("For mount point %s, the following are the current usage values:" % (pMountpoint)) + print(" Total bytes read = %d, Total bytes written = %d" % (getattr(l_ReturnStruct, "totalBytesRead"), getattr(l_ReturnStruct, "totalBytesWritten"))) + print(" Local bytes read = %d, Local bytes written = %d" % (getattr(l_ReturnStruct, "localBytesRead"), getattr(l_ReturnStruct, "localBytesWritten"))) + print(" Burst bytes read = %d, Burst bytes written = %d" % (getattr(l_ReturnStruct, "burstBytesRead"), getattr(l_ReturnStruct, "burstBytesWritten"))) + + return l_ReturnStruct + +def BB_SetThrottleRate(pMountpoint, pRate): + l_Mountpoint = bb.cvar("mountpoint", pMountpoint) + l_Rate = bb.cvar("rate", pRate) + + print("%sBB_SetThrottleRate issued to set the throttle rate for mountpoint %s to %d bytes/sec" % (os.linesep, pMountpoint, pRate)) + rc = bb.api.BB_SetThrottleRate(l_Mountpoint, l_Rate) + if (rc): + raise BB_SetThrottleRateError(rc) + + bb.printLastErrorDetailsSummary() + print("Throttle rate for mountpoint %s changed to %d bytes/sec" % (pMountpoint, pRate)) + + return + +def BB_SetUsageLimit(pMountpoint, pUsage): + l_Mountpoint = bb.cvar("mountpoint", pMountpoint) + + print("%sBB_SetUsageLimit issued to set the usage limit for mountpoint %s to %d" % (os.linesep, pMountpoint, pUsage)) + rc = bb.api.BB_SetUsage(l_Mountpoint, byref(pUsage)) + if (rc): + raise BB_SetUsageError(rc) + + bb.printLastErrorDetailsSummary() + print("For mount point %s, the following usage limits were set:" % (pMountpoint)) + print(" Total bytes read = %d, Total bytes written = %d" % (getattr(pUsage, "totalBytesRead"), getattr(pUsage, "totalBytesWritten"))) + print(" Local bytes read = %d, Local bytes written = %d" % (getattr(pUsage, "localBytesRead"), getattr(pUsage, "localBytesWritten"))) + print(" Burst bytes read = %d, Burst bytes written = %d" % (getattr(pUsage, "burstBytesRead"), getattr(pUsage, "burstBytesWritten"))) + + return + +def BB_StartTransfer(pTransferDef, pHandle): + l_NormalRCs = BB_StartTransferError(BBError(Exception())).getNormalRCs() + l_ToleratedErrorRCs = BB_StartTransferError(BBError(Exception())).getToleratedErrorRCs() + + l_Handle = bb.cvar("handle", pHandle) + + print("%sBB_StartTransfer issued to start the transfer %s for handle %s" % (os.linesep, repr(pTransferDef), pHandle)) + + while (True): + rc = bb.api.BB_StartTransfer(pTransferDef, l_Handle) + if (rc in (l_NormalRCs + l_ToleratedErrorRCs)): + if (rc in l_ToleratedErrorRCs): + dummy = BBError() + if ("Attempt to retry" in dummy.getLastErrorDetailsSummary()): + print("Transfer %s cannot be started for handle %s because of a suspended condition. This start transfer request will be attempted again in 15 seconds." % (repr(pTransferDef), pHandle)) + time.sleep(15) + else: + print("Transfer %s cannot be started for handle %s because of a suspended condition. Restart logic will resubmit this start transfer operation." % (repr(pTransferDef), pHandle)) + break + else: + break + else: + raise BB_StartTransferError(rc) + + bb.printLastErrorDetailsSummary() + if (rc == 0): + print("Transfer %s started for handle %s" % (repr(pTransferDef), pHandle)) + + return diff --git a/bb/wrappers/bbapiAdmin.py b/bb/wrappers/bbapiAdmin.py index e4af16e5b..ab52d0faf 100644 --- a/bb/wrappers/bbapiAdmin.py +++ b/bb/wrappers/bbapiAdmin.py @@ -1,411 +1,411 @@ -#!/usr/bin/python -#################################################### -# bbapiAdmin.py -# -# Copyright IBM Corporation 2017,2017. All Rights Reserved -# -# This program is licensed under the terms of the Eclipse Public License -# v1.0 as published by the Eclipse Foundation and available at -# http://www.eclipse.org/legal/epl-v10.html -# -# U.S. Government Users Restricted Rights: Use, duplication or disclosure -# restricted by GSA ADP Schedule Contract with IBM Corp. -################################################### - -""" - This module provides wrappers to the following bbapi functions. - -int BB_ChangeMode(const char* pathname, mode_t mode) -int BB_ChangeOwner(const char* pathname, const char* owner, const char* group) -int BB_CloseServer(const char* name); -int BB_CreateDirectory(const char* newpathname) -int BB_CreateLogicalVolume(const char* mountpoint, const char* size, BBCREATEFLAGS flags=BBXFS) -int BB_GetServer(const char* type, size_t bufsize, char* buffer); -int BB_GetServerByName(const char* bbserverName, const char* type, size_t bufsize, char* buffer); -int BB_OpenServer(const char* name); -int BB_RemoveDirectory(const char* pathname) -int BB_RemoveJobInfo() -int BB_RemoveLogicalVolume(const char* mountpoint) -int BB_ResizeMountPoint(const char* mountpoint, char** logicalvolume, const char* size, BBRESIZEFLAGS flags) -int BB_RestartTransfers(const char* pHostName, const uint64_t pHandle, const char* pTransferDefs, const size_t pTransferDefsSize); -int BB_Resume(const char* pHostName); -int BB_RetrieveTransfers(const char* pHostName, const uint64_t pHandle, const BB_RTV_TRANSFERDEFS_FLAGS pFlags, size_t* pNumBytesAvail, const size_t pBufferSize, char* pBuffer); -int BB_SetServer(const char* type, const char* name); -int BB_StopTransfers(const char* pHostName, const uint64_t pHandle, const char* pTransferDefs, const size_t pTransferDefsSize); -int BB_Suspend(const char* pHostName); - - Not implemented - -int BB_QueryServer(enum BBServerQuery, size_t bufsize, char* buffer); -int BB_SwitchServer(const char* buffer); -""" - -from ctypes import * -import subprocess - -import bb -from bbapi import * -import bbapiAdminProcs as adminProcs -from bberror import * -import time - -PRINT_VALUE = True - - -# -# Miscellaneous -# - -ValueMap = {"%%_NuLl_HoStNaMe_%%":"*", "":'""'} - -LIBPATH = "/u/dlherms/git/bluecoral/bb/wrappers" - - -# -# Wrappers for official bbapiAdmin calls -# - -def BB_ChangeMode(pPathName, pMode): - l_PathName = bb.cvar("pathname", pPathName) - l_Mode = bb.cvar("mode", pMode) - - print "%sBB_ChangeMode issued to change the mode for path %s to %s" % (os.linesep, pPathName, oct(pMode)) - rc = bb.api.BB_ChangeMode(l_PathName, l_Mode) - if (rc): - raise BB_ChangeModeError(rc) - - bb.printLastErrorDetailsSummary() - print "Mode for path %s changed to %s" % (pPathName, oct(pMode)) - - return - -def BB_ChangeOwner(pPathName, pOwner, pGroup): - l_PathName = bb.cvar("pathname", pPathName) - l_Owner = bb.cvar("owner", pOwner) - l_Group = bb.cvar("group", pGroup) - - print '%sBB_ChangeOwner issued to change the ownership of path %s to user "%s", group "%s"' % (os.linesep, pPathName, pOwner, pGroup) - rc = bb.api.BB_ChangeOwner(l_PathName, l_Owner, l_Group) - if (rc): - raise BB_ChangeOwnerError(rc) - - bb.printLastErrorDetailsSummary() - print 'Ownership of path %s changed to user "%s", group "%s"' % (pPathName, pOwner, pGroup) - - return - -def BB_CloseServer(pName): - l_Name = bb.cvar("name", pName) - - print "%sBB_CloseServer issued for name %s" % (os.linesep, pName) - rc = bb.api.BB_CloseServer(l_Name) - if (rc): - raise BB_CloseServerError(rc) - - bb.printLastErrorDetailsSummary() - - print "%sBB_CloseServer completed for name %s" % (os.linesep, pName) - - return - -def BB_CreateDirectory(pNewPathName): - l_NewPathName = bb.cvar("newpathname", pNewPathName) - - print "%sBB_CreateDirectory issued to create directory %s" % (os.linesep, pNewPathName) - rc = bb.api.BB_CreateDirectory(l_NewPathName) - if (rc): - raise BB_CreateDirectoryError(rc) - - bb.printLastErrorDetailsSummary() - print "Directory %s created" % (pNewPathName) - - return - -def BB_CreateLogicalVolume(pMountpoint, pSize, pFlags=DEFAULT_BBCREATEFLAGS): - l_NormalRCs = BB_CreateLogicalVolumeError(BBError(Exception())).getNormalRCs() - l_ToleratedErrorRCs = BB_CreateLogicalVolumeError(BBError(Exception())).getToleratedErrorRCs() - rc = l_ToleratedErrorRCs[0] - - l_Mountpoint = bb.cvar("mountpoint", pMountpoint) - l_Size = bb.cvar("size_str", pSize) - l_Flags = bb.cvar("flags", pFlags) - - print "%sBB_CreateLogicalVolume issued to create logical volume with size %s, directory %s mounted, file system flag %s" % (os.linesep, pSize, pMountpoint, BBCREATEFLAGS[l_Flags.value]) - - while (True): - rc = bb.api.BB_CreateLogicalVolume(l_Mountpoint, l_Size, l_Flags) - if (rc in (l_NormalRCs + l_ToleratedErrorRCs)): - if (rc in l_ToleratedErrorRCs): - dummy = BBError() - if ("Attempt to retry" in dummy.getLastErrorDetailsSummary()): - print "Logical volume cannot be created because of a suspended condition. This create logical volume request will be attempted again in 15 seconds." - time.sleep(15) - else: - print "Logical volume cannot be created now, rc %d. See error details." % (rc) - break - else: - break - else: - raise BB_CreateLogicalVolumeError(rc) - - bb.printLastErrorDetailsSummary() - if (rc in l_NormalRCs): - print "Logical volume created with size %s, directory %s mounted, file system flag %s" % (pSize, pMountpoint, BBCREATEFLAGS[l_Flags.value]) - - return - -def BB_GetServer(pType, pPrintOption=True): - l_BufferSize = 512 - - l_Type = bb.cvar("type", pType) - l_Size = bb.cvar("size", l_BufferSize) - l_Buffer = bb.cvar("buffer", l_BufferSize) - - if (pPrintOption): - print "%sBB_GetServer issued for type %s" % (os.linesep, pType) - rc = bb.api.BB_GetServer(l_Type, l_Size, byref(l_Buffer)) - if (rc): - raise BB_GetServerError(rc) - - if (pPrintOption): - bb.printLastErrorDetailsSummary() - - l_Temp = [] - for i in xrange(l_Size.value): - if l_Buffer[i] != '\x00': - l_Temp.append(l_Buffer[i]) - else: - break - l_Output = "".join(l_Temp) - l_TypePrt = list(pType) - l_TypePrt[0] = l_TypePrt[0].upper() - if (pPrintOption): - print "%s Server: %s" % ("".join(l_TypePrt), l_Output) - - return l_Output - -def BB_GetServerByName(pName, pType, pPrintOption=True): - l_BufferSize = 512 - - l_Name = bb.cvar("type", pName) - l_Type = bb.cvar("type", pType) - l_Size = bb.cvar("size", l_BufferSize) - l_Buffer = bb.cvar("buffer", l_BufferSize) - - if (pPrintOption): - print "%sBB_GetServerByName issued for name %s, type %s" % (os.linesep, pName, pType) - rc = bb.api.BB_GetServerByName(l_Name, l_Type, l_Size, byref(l_Buffer)) - if (rc): - raise BB_GetServerByNameError(rc) - - if (pPrintOption): - bb.printLastErrorDetailsSummary() - - l_Temp = [] - for i in xrange(l_Size.value): - if l_Buffer[i] != '\x00': - l_Temp.append(l_Buffer[i]) - else: - break - l_Output = "".join(l_Temp) - if (pPrintOption): - print "%s Server: %s Option: %s Result: %s" % (os.linesep, pName, pType, l_Output) - - return l_Output - -def BB_OpenServer(pName): - l_Name = bb.cvar("name", pName) - - print "%sBB_OpenServer issued for name %s" % (os.linesep, pName) - rc = bb.api.BB_OpenServer(l_Name) - if (rc): - raise BB_OpenServerError(rc) - - bb.printLastErrorDetailsSummary() - - print "%sBB_OpenServer completed for name %s" % (os.linesep, pName) - - return - -def BB_RemoveDirectory(pPathName): - l_PathName = bb.cvar("pathname", pPathName) - - print "%sBB_RemoveDirectory issued to remove directory %s" % (os.linesep, pPathName) - rc = bb.api.BB_RemoveDirectory(l_PathName) - if (rc): - raise BB_RemoveDirectoryError(rc) - - bb.printLastErrorDetailsSummary() - print "Directory %s removed" % (pPathName) - - return - -def BB_RemoveJobInfo(): - print "%sBB_RemoveJobInfo issued" % (os.linesep) - - rc = bb.api.BB_RemoveJobInfo() - if (rc): - raise BB_RemoveJobInfoError(rc) - - bb.printLastErrorDetailsSummary() - print "BB_RemoveJobInfo completed" - - return - -def BB_RemoveLogicalVolume(pMountpoint): - l_NormalRCs = BB_RemoveLogicalVolumeError(BBError(Exception())).getNormalRCs() - l_ToleratedErrorRCs = BB_RemoveLogicalVolumeError(BBError(Exception())).getToleratedErrorRCs() - - l_Mountpoint = bb.cvar("mountpoint", pMountpoint) - - print "%sBB_RemoveLogicalVolume issued to remove the logical volume associated with mountpoint %s" % (os.linesep, pMountpoint) - - rc = bb.api.BB_RemoveLogicalVolume(l_Mountpoint) - while ((rc not in l_NormalRCs) and (rc not in l_ToleratedErrorRCs)): - dummy = BBError() - if ("Device or resource busy" not in dummy.getLastErrorDetailsSummary()): - raise BB_RemoveLogicalVolumeError(rc) - else: - # NOTE: This could be a 'normal' case where restart transfer definition has not completed before remove logical volume - # is attempted in the normal flow of operations from the original start transfer request. - # The restart transfer may take 'minutes' attempting to determine if the transfer definition is in a stopped state. - print "Device or resource busy. The remove logical volume for mountpoint %s request will be re-attempted in 30 seconds." % (pMountpoint) - time.sleep(30) - rc = bb.api.BB_RemoveLogicalVolume(l_Mountpoint) - - bb.printLastErrorDetailsSummary() - if (rc in l_NormalRCs): - print "Logical volume associated with mountpoint %s removed" % (pMountpoint) - - return - -def BB_ResizeMountPoint(pMountpoint, pSize, pFlags=DEFAULT_BBRESIZEFLAGS): - l_Mountpoint = bb.cvar("mountpoint", pMountpoint) - l_Size = bb.cvar("size_str", pSize) - l_Flags = bb.cvar("flags", pFlags) - - print "%sBB_ResizeMountPoint issued to have mountpoint %s resized with a size specification of %s and flags %s" % (os.linesep, pMountpoint, pSize, BBRESIZEFLAGS[l_Flags.value]) - rc = bb.api.BB_ResizeMountPoint(l_Mountpoint, l_Size, l_Flags) - if (rc): - raise BB_ResizeMountPointError(rc) - - bb.printLastErrorDetailsSummary() - print "Mountpoint %s resized with a size specification of %s and flags %s" % (pMountpoint, pSize, BBRESIZEFLAGS[l_Flags.value]) - - return - -def BB_RestartTransfers(pHostName, pHandle, pTransferDefs, pTransferDefsSize): - l_NormalRCs = BB_RestartTransfersError(BBError(Exception())).getNormalRCs() - l_ToleratedErrorRCs = BB_RestartTransfersError(BBError(Exception())).getToleratedErrorRCs() - rc = l_ToleratedErrorRCs[0] - - l_HostName = bb.cvar("hostname", pHostName) - l_Handle = bb.cvar("handle", pHandle) - l_NumberOfRestartedTransferDefs = bb.cvar("numtransferdefs", 0) - - print '%sBB_RestartTransfers issued to restart transfer definitions using this criteria: hostname %s, handle %d, transferdefs size %d, transferdefs %s' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value), l_Handle.value, pTransferDefsSize.value, pTransferDefs.value) - - while (True): - rc = bb.api.BB_RestartTransfers(l_HostName, l_Handle, byref(l_NumberOfRestartedTransferDefs), byref(pTransferDefs), pTransferDefsSize) - if (rc in (l_NormalRCs + l_ToleratedErrorRCs)): - if (rc in l_ToleratedErrorRCs): - dummy = BBError() - if ("Attempt to retry" in dummy.getLastErrorDetailsSummary()): - print "Restart transfers cannot be performed for handle %s because of a suspended condition. This restart transfers request will be attempted again in 15 seconds." % (l_Handle) - time.sleep(15) - else: - print "Restart transfers cannot be performed for handle %s. See error details." % (l_Handle) - break - else: - break - else: - raise BB_RestartTransfersError(rc) - - - bb.printLastErrorDetailsSummary() - print '%sBB_RestartTransfers completed' % (os.linesep) - - if (rc in l_NormalRCs): - return l_NumberOfRestartedTransferDefs.value - -def BB_Resume(pHostHame): - l_HostName = bb.cvar("hostname", pHostHame) - - print '%sBB_Resume issued for hostname %s' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value)) - - rc = bb.api.BB_Resume(l_HostName) - if (rc): - raise BB_ResumeError(rc) - - bb.printLastErrorDetailsSummary() - print '%sBB_Resume completed for hostname %s' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value)) - - return - -def BB_RetrieveTransfers(pHostHame, pHandle, pFlags=DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS): - l_HostName = bb.cvar("hostname", pHostHame) - l_Handle = bb.cvar("handle", pHandle) - l_Flags = bb.cvar("flags", pFlags) - l_BufferSizeIncr = 16*1024 - - l_NumTransferDefs = bb.cvar("numtransferdefs", 0) - l_NumBytesAvailable = bb.cvar("numbytesavailable", l_BufferSizeIncr) - l_BufferSize = bb.cvar("size", 0) - while (l_NumBytesAvailable.value > l_BufferSize.value): - l_BufferSize = bb.cvar("size", ((l_NumBytesAvailable.value + (l_BufferSizeIncr - 1)) / l_BufferSizeIncr) * l_BufferSizeIncr) - l_Buffer = bb.cvar("buffer", l_BufferSize.value) - print '%sBB_RetrieveTransfers issued to retrieve transfer definitions using this criteria: hostname %s, handle %d, flags %s, bytesavailable %d, bytesprovided %d' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value), l_Handle.value, BB_RTV_TRANSFERDEFS_FLAGS[l_Flags.value], l_NumBytesAvailable.value, l_BufferSize.value) - rc = bb.api.BB_RetrieveTransfers(l_HostName, l_Handle, l_Flags, byref(l_NumTransferDefs), byref(l_NumBytesAvailable), l_BufferSize, byref(l_Buffer)) - if (rc): - raise BB_RetrieveTransfersError(rc) - - bb.printLastErrorDetailsSummary() -# print '%sBB_RetrieveTransfers completed the retrieval of transfer definitions using this criteria: hostname %s, handle %d, flags %s, number of transferdefs %d, length of transferdefs %d, transferdefs |%s|' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value), l_Handle.value, BB_RTV_TRANSFERDEFS_FLAGS[l_Flags.value], l_NumTransferDefs.value, l_NumBytesAvailable.value, l_Buffer.value) - print '%sBB_RetrieveTransfers completed the retrieval of transfer definitions using this criteria: hostname %s, handle %d, flags %s, number of transferdefs %d.' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value), l_Handle.value, BB_RTV_TRANSFERDEFS_FLAGS[l_Flags.value], l_NumTransferDefs.value) - - return (l_NumTransferDefs.value, l_Buffer, l_NumBytesAvailable) - -def BB_SetServer(pType, pName): - l_Type = bb.cvar("type", pType) - l_Name = bb.cvar("name", pName) - - print "%sBB_SetServer issued for type %s, name %s" % (os.linesep, pType, pName) - rc = bb.api.BB_SetServer(l_Type, l_Name) - if (rc): - raise BB_SetServerError(rc) - - bb.printLastErrorDetailsSummary() - - print "%sBB_SetServer completed for type %s, name %s" % (os.linesep, pType, pName) - - return - -def BB_StopTransfers(pHostName, pHandle, pTransferDefs, pTransferDefsSize): - l_HostName = bb.cvar("hostname", pHostName) - l_Handle = bb.cvar("handle", pHandle) - l_NumStoppedTransferDefs = bb.cvar("numtransferdefs", 0) - - print '%sBB_StopTransfers issued to retrieve transfer definitions using this criteria: hostname %s, handle %d, transferdefs size %d, transferdefs %s' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value), l_Handle.value, pTransferDefsSize.value, pTransferDefs.value) - - rc = bb.api.BB_StopTransfers(l_HostName, l_Handle, byref(l_NumStoppedTransferDefs), byref(pTransferDefs), pTransferDefsSize) - if (rc): - raise BB_StopTransfersError(rc) - - bb.printLastErrorDetailsSummary() - print '%sBB_StopTransfers completed' % (os.linesep) - - return l_NumStoppedTransferDefs.value - -def BB_Suspend(pHostHame): - l_HostName = bb.cvar("hostname", pHostHame) - - print '%sBB_Suspend issued for hostname %s' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value)) - - rc = bb.api.BB_Suspend(l_HostName) - if (rc): - raise BB_SuspendError(rc) - - bb.printLastErrorDetailsSummary() - print '%sBB_Suspend completed for hostname %s' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value)) - - return +#!/usr/bin/python +#################################################### +# bbapiAdmin.py +# +# Copyright IBM Corporation 2017,2017. All Rights Reserved +# +# This program is licensed under the terms of the Eclipse Public License +# v1.0 as published by the Eclipse Foundation and available at +# http://www.eclipse.org/legal/epl-v10.html +# +# U.S. Government Users Restricted Rights: Use, duplication or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. +################################################### + +""" + This module provides wrappers to the following bbapi functions. + +int BB_ChangeMode(const char* pathname, mode_t mode) +int BB_ChangeOwner(const char* pathname, const char* owner, const char* group) +int BB_CloseServer(const char* name); +int BB_CreateDirectory(const char* newpathname) +int BB_CreateLogicalVolume(const char* mountpoint, const char* size, BBCREATEFLAGS flags=BBXFS) +int BB_GetServer(const char* type, size_t bufsize, char* buffer); +int BB_GetServerByName(const char* bbserverName, const char* type, size_t bufsize, char* buffer); +int BB_OpenServer(const char* name); +int BB_RemoveDirectory(const char* pathname) +int BB_RemoveJobInfo() +int BB_RemoveLogicalVolume(const char* mountpoint) +int BB_ResizeMountPoint(const char* mountpoint, char** logicalvolume, const char* size, BBRESIZEFLAGS flags) +int BB_RestartTransfers(const char* pHostName, const uint64_t pHandle, const char* pTransferDefs, const size_t pTransferDefsSize); +int BB_Resume(const char* pHostName); +int BB_RetrieveTransfers(const char* pHostName, const uint64_t pHandle, const BB_RTV_TRANSFERDEFS_FLAGS pFlags, size_t* pNumBytesAvail, const size_t pBufferSize, char* pBuffer); +int BB_SetServer(const char* type, const char* name); +int BB_StopTransfers(const char* pHostName, const uint64_t pHandle, const char* pTransferDefs, const size_t pTransferDefsSize); +int BB_Suspend(const char* pHostName); + + Not implemented + +int BB_QueryServer(enum BBServerQuery, size_t bufsize, char* buffer); +int BB_SwitchServer(const char* buffer); +""" + +from ctypes import * +import subprocess + +import bb +from bbapi import * +import bbapiAdminProcs as adminProcs +from bberror import * +import time + +PRINT_VALUE = True + + +# +# Miscellaneous +# + +ValueMap = {"%%_NuLl_HoStNaMe_%%":"*", "":'""'} + +LIBPATH = "/u/dlherms/git/bluecoral/bb/wrappers" + + +# +# Wrappers for official bbapiAdmin calls +# + +def BB_ChangeMode(pPathName, pMode): + l_PathName = bb.cvar("pathname", pPathName) + l_Mode = bb.cvar("mode", pMode) + + print("%sBB_ChangeMode issued to change the mode for path %s to %s" % (os.linesep, pPathName, oct(pMode))) + rc = bb.api.BB_ChangeMode(l_PathName, l_Mode) + if (rc): + raise BB_ChangeModeError(rc) + + bb.printLastErrorDetailsSummary() + print("Mode for path %s changed to %s" % (pPathName, oct(pMode))) + + return + +def BB_ChangeOwner(pPathName, pOwner, pGroup): + l_PathName = bb.cvar("pathname", pPathName) + l_Owner = bb.cvar("owner", pOwner) + l_Group = bb.cvar("group", pGroup) + + print('%sBB_ChangeOwner issued to change the ownership of path %s to user "%s", group "%s"' % (os.linesep, pPathName, pOwner, pGroup)) + rc = bb.api.BB_ChangeOwner(l_PathName, l_Owner, l_Group) + if (rc): + raise BB_ChangeOwnerError(rc) + + bb.printLastErrorDetailsSummary() + print('Ownership of path %s changed to user "%s", group "%s"' % (pPathName, pOwner, pGroup)) + + return + +def BB_CloseServer(pName): + l_Name = bb.cvar("name", pName) + + print("%sBB_CloseServer issued for name %s" % (os.linesep, pName)) + rc = bb.api.BB_CloseServer(l_Name) + if (rc): + raise BB_CloseServerError(rc) + + bb.printLastErrorDetailsSummary() + + print("%sBB_CloseServer completed for name %s" % (os.linesep, pName)) + + return + +def BB_CreateDirectory(pNewPathName): + l_NewPathName = bb.cvar("newpathname", pNewPathName) + + print("%sBB_CreateDirectory issued to create directory %s" % (os.linesep, pNewPathName)) + rc = bb.api.BB_CreateDirectory(l_NewPathName) + if (rc): + raise BB_CreateDirectoryError(rc) + + bb.printLastErrorDetailsSummary() + print("Directory %s created" % (pNewPathName)) + + return + +def BB_CreateLogicalVolume(pMountpoint, pSize, pFlags=DEFAULT_BBCREATEFLAGS): + l_NormalRCs = BB_CreateLogicalVolumeError(BBError(Exception())).getNormalRCs() + l_ToleratedErrorRCs = BB_CreateLogicalVolumeError(BBError(Exception())).getToleratedErrorRCs() + rc = l_ToleratedErrorRCs[0] + + l_Mountpoint = bb.cvar("mountpoint", pMountpoint) + l_Size = bb.cvar("size_str", pSize) + l_Flags = bb.cvar("flags", pFlags) + + print("%sBB_CreateLogicalVolume issued to create logical volume with size %s, directory %s mounted, file system flag %s" % (os.linesep, pSize, pMountpoint, BBCREATEFLAGS[l_Flags.value])) + + while (True): + rc = bb.api.BB_CreateLogicalVolume(l_Mountpoint, l_Size, l_Flags) + if (rc in (l_NormalRCs + l_ToleratedErrorRCs)): + if (rc in l_ToleratedErrorRCs): + dummy = BBError() + if ("Attempt to retry" in dummy.getLastErrorDetailsSummary()): + print("Logical volume cannot be created because of a suspended condition. This create logical volume request will be attempted again in 15 seconds.") + time.sleep(15) + else: + print("Logical volume cannot be created now, rc %d. See error details." % (rc)) + break + else: + break + else: + raise BB_CreateLogicalVolumeError(rc) + + bb.printLastErrorDetailsSummary() + if (rc in l_NormalRCs): + print("Logical volume created with size %s, directory %s mounted, file system flag %s" % (pSize, pMountpoint, BBCREATEFLAGS[l_Flags.value])) + + return + +def BB_GetServer(pType, pPrintOption=True): + l_BufferSize = 512 + + l_Type = bb.cvar("type", pType) + l_Size = bb.cvar("size", l_BufferSize) + l_Buffer = bb.cvar("buffer", l_BufferSize) + + if (pPrintOption): + print("%sBB_GetServer issued for type %s" % (os.linesep, pType)) + rc = bb.api.BB_GetServer(l_Type, l_Size, byref(l_Buffer)) + if (rc): + raise BB_GetServerError(rc) + + if (pPrintOption): + bb.printLastErrorDetailsSummary() + + l_Temp = [] + for i in range(l_Size.value): + if l_Buffer[i] != b'\0': + l_Temp.append(l_Buffer[i]) + else: + break + l_Output = (b"".join(l_Temp)).decode() + l_TypePrt = list(pType) + l_TypePrt[0] = l_TypePrt[0].upper() + if (pPrintOption): + print("%s Server: %s" % ("".join(l_TypePrt), l_Output)) + + return l_Output + +def BB_GetServerByName(pName, pType, pPrintOption=True): + l_BufferSize = 512 + + l_Name = bb.cvar("type", pName) + l_Type = bb.cvar("type", pType) + l_Size = bb.cvar("size", l_BufferSize) + l_Buffer = bb.cvar("buffer", l_BufferSize) + + if (pPrintOption): + print("%sBB_GetServerByName issued for name %s, type %s" % (os.linesep, pName, pType)) + rc = bb.api.BB_GetServerByName(l_Name, l_Type, l_Size, byref(l_Buffer)) + if (rc): + raise BB_GetServerByNameError(rc) + + if (pPrintOption): + bb.printLastErrorDetailsSummary() + + l_Temp = [] + for i in range(l_Size.value): + if l_Buffer[i] != b'\0': + l_Temp.append(l_Buffer[i]) + else: + break + l_Output = (b"".join(l_Temp)).decode() + if (pPrintOption): + print("%s Server: %s Option: %s Result: %s" % (os.linesep, pName, pType, l_Output)) + + return l_Output + +def BB_OpenServer(pName): + l_Name = bb.cvar("name", pName) + + print("%sBB_OpenServer issued for name %s" % (os.linesep, pName)) + rc = bb.api.BB_OpenServer(l_Name) + if (rc): + raise BB_OpenServerError(rc) + + bb.printLastErrorDetailsSummary() + + print("%sBB_OpenServer completed for name %s" % (os.linesep, pName)) + + return + +def BB_RemoveDirectory(pPathName): + l_PathName = bb.cvar("pathname", pPathName) + + print("%sBB_RemoveDirectory issued to remove directory %s" % (os.linesep, pPathName)) + rc = bb.api.BB_RemoveDirectory(l_PathName) + if (rc): + raise BB_RemoveDirectoryError(rc) + + bb.printLastErrorDetailsSummary() + print("Directory %s removed" % (pPathName)) + + return + +def BB_RemoveJobInfo(): + print("%sBB_RemoveJobInfo issued" % (os.linesep)) + + rc = bb.api.BB_RemoveJobInfo() + if (rc): + raise BB_RemoveJobInfoError(rc) + + bb.printLastErrorDetailsSummary() + print("BB_RemoveJobInfo completed") + + return + +def BB_RemoveLogicalVolume(pMountpoint): + l_NormalRCs = BB_RemoveLogicalVolumeError(BBError(Exception())).getNormalRCs() + l_ToleratedErrorRCs = BB_RemoveLogicalVolumeError(BBError(Exception())).getToleratedErrorRCs() + + l_Mountpoint = bb.cvar("mountpoint", pMountpoint.encode()) + + print("%sBB_RemoveLogicalVolume issued to remove the logical volume associated with mountpoint %s" % (os.linesep, pMountpoint)) + + rc = bb.api.BB_RemoveLogicalVolume(l_Mountpoint) + while ((rc not in l_NormalRCs) and (rc not in l_ToleratedErrorRCs)): + dummy = BBError() + if ("Device or resource busy" not in dummy.getLastErrorDetailsSummary()): + raise BB_RemoveLogicalVolumeError(rc) + else: + # NOTE: This could be a 'normal' case where restart transfer definition has not completed before remove logical volume + # is attempted in the normal flow of operations from the original start transfer request. + # The restart transfer may take 'minutes' attempting to determine if the transfer definition is in a stopped state. + print("Device or resource busy. The remove logical volume for mountpoint %s request will be re-attempted in 30 seconds." % (pMountpoint)) + time.sleep(30) + rc = bb.api.BB_RemoveLogicalVolume(l_Mountpoint) + + bb.printLastErrorDetailsSummary() + if (rc in l_NormalRCs): + print("Logical volume associated with mountpoint %s removed" % (pMountpoint)) + + return + +def BB_ResizeMountPoint(pMountpoint, pSize, pFlags=DEFAULT_BBRESIZEFLAGS): + l_Mountpoint = bb.cvar("mountpoint", pMountpoint) + l_Size = bb.cvar("size_str", pSize) + l_Flags = bb.cvar("flags", pFlags) + + print("%sBB_ResizeMountPoint issued to have mountpoint %s resized with a size specification of %s and flags %s" % (os.linesep, pMountpoint, pSize, BBRESIZEFLAGS[l_Flags.value])) + rc = bb.api.BB_ResizeMountPoint(l_Mountpoint, l_Size, l_Flags) + if (rc): + raise BB_ResizeMountPointError(rc) + + bb.printLastErrorDetailsSummary() + print("Mountpoint %s resized with a size specification of %s and flags %s" % (pMountpoint, pSize, BBRESIZEFLAGS[l_Flags.value])) + + return + +def BB_RestartTransfers(pHostName, pHandle, pTransferDefs, pTransferDefsSize): + l_NormalRCs = BB_RestartTransfersError(BBError(Exception())).getNormalRCs() + l_ToleratedErrorRCs = BB_RestartTransfersError(BBError(Exception())).getToleratedErrorRCs() + rc = l_ToleratedErrorRCs[0] + + l_HostName = bb.cvar("hostname", pHostName) + l_Handle = bb.cvar("handle", pHandle) + l_NumberOfRestartedTransferDefs = bb.cvar("numtransferdefs", 0) + + print('%sBB_RestartTransfers issued to restart transfer definitions using this criteria: hostname %s, handle %d, transferdefs size %d, transferdefs %s' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value), l_Handle.value, pTransferDefsSize.value, pTransferDefs.value)) + + while (True): + rc = bb.api.BB_RestartTransfers(l_HostName, l_Handle, byref(l_NumberOfRestartedTransferDefs), byref(pTransferDefs), pTransferDefsSize) + if (rc in (l_NormalRCs + l_ToleratedErrorRCs)): + if (rc in l_ToleratedErrorRCs): + dummy = BBError() + if ("Attempt to retry" in dummy.getLastErrorDetailsSummary()): + print("Restart transfers cannot be performed for handle %s because of a suspended condition. This restart transfers request will be attempted again in 15 seconds." % (l_Handle)) + time.sleep(15) + else: + print("Restart transfers cannot be performed for handle %s. See error details." % (l_Handle)) + break + else: + break + else: + raise BB_RestartTransfersError(rc) + + + bb.printLastErrorDetailsSummary() + print('%sBB_RestartTransfers completed' % (os.linesep)) + + if (rc in l_NormalRCs): + return l_NumberOfRestartedTransferDefs.value + +def BB_Resume(pHostHame): + l_HostName = bb.cvar("hostname", pHostHame) + + print('%sBB_Resume issued for hostname %s' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value))) + + rc = bb.api.BB_Resume(l_HostName) + if (rc): + raise BB_ResumeError(rc) + + bb.printLastErrorDetailsSummary() + print('%sBB_Resume completed for hostname %s' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value))) + + return + +def BB_RetrieveTransfers(pHostHame, pHandle, pFlags=DEFAULT_BB_RTV_TRANSFERDEFS_FLAGS): + l_HostName = bb.cvar("hostname", pHostHame) + l_Handle = bb.cvar("handle", pHandle) + l_Flags = bb.cvar("flags", pFlags) + l_BufferSizeIncr = 16*1024 + + l_NumTransferDefs = bb.cvar("numtransferdefs", 0) + l_NumBytesAvailable = bb.cvar("numbytesavailable", l_BufferSizeIncr) + l_BufferSize = bb.cvar("size", 0) + while (l_NumBytesAvailable.value > l_BufferSize.value): + l_BufferSize = bb.cvar("size", ((l_NumBytesAvailable.value + (l_BufferSizeIncr - 1)) / l_BufferSizeIncr) * l_BufferSizeIncr) + l_Buffer = bb.cvar("buffer", l_BufferSize.value) + print('%sBB_RetrieveTransfers issued to retrieve transfer definitions using this criteria: hostname %s, handle %d, flags %s, bytesavailable %d, bytesprovided %d' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value), l_Handle.value, BB_RTV_TRANSFERDEFS_FLAGS[l_Flags.value], l_NumBytesAvailable.value, l_BufferSize.value)) + rc = bb.api.BB_RetrieveTransfers(l_HostName, l_Handle, l_Flags, byref(l_NumTransferDefs), byref(l_NumBytesAvailable), l_BufferSize, byref(l_Buffer)) + if (rc): + raise BB_RetrieveTransfersError(rc) + + bb.printLastErrorDetailsSummary() +# print '%sBB_RetrieveTransfers completed the retrieval of transfer definitions using this criteria: hostname %s, handle %d, flags %s, number of transferdefs %d, length of transferdefs %d, transferdefs |%s|' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value), l_Handle.value, BB_RTV_TRANSFERDEFS_FLAGS[l_Flags.value], l_NumTransferDefs.value, l_NumBytesAvailable.value, l_Buffer.value) + print('%sBB_RetrieveTransfers completed the retrieval of transfer definitions using this criteria: hostname %s, handle %d, flags %s, number of transferdefs %d.' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value), l_Handle.value, BB_RTV_TRANSFERDEFS_FLAGS[l_Flags.value], l_NumTransferDefs.value)) + + return (l_NumTransferDefs.value, l_Buffer, l_NumBytesAvailable) + +def BB_SetServer(pType, pName): + l_Type = bb.cvar("type", pType) + l_Name = bb.cvar("name", pName) + + print("%sBB_SetServer issued for type %s, name %s" % (os.linesep, pType, pName)) + rc = bb.api.BB_SetServer(l_Type, l_Name) + if (rc): + raise BB_SetServerError(rc) + + bb.printLastErrorDetailsSummary() + + print("%sBB_SetServer completed for type %s, name %s" % (os.linesep, pType, pName)) + + return + +def BB_StopTransfers(pHostName, pHandle, pTransferDefs, pTransferDefsSize): + l_HostName = bb.cvar("hostname", pHostName) + l_Handle = bb.cvar("handle", pHandle) + l_NumStoppedTransferDefs = bb.cvar("numtransferdefs", 0) + + print('%sBB_StopTransfers issued to retrieve transfer definitions using this criteria: hostname %s, handle %d, transferdefs size %d, transferdefs %s' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value), l_Handle.value, pTransferDefsSize.value, pTransferDefs.value)) + + rc = bb.api.BB_StopTransfers(l_HostName, l_Handle, byref(l_NumStoppedTransferDefs), byref(pTransferDefs), pTransferDefsSize) + if (rc): + raise BB_StopTransfersError(rc) + + bb.printLastErrorDetailsSummary() + print('%sBB_StopTransfers completed' % (os.linesep)) + + return l_NumStoppedTransferDefs.value + +def BB_Suspend(pHostHame): + l_HostName = bb.cvar("hostname", pHostHame) + + print('%sBB_Suspend issued for hostname %s' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value))) + + rc = bb.api.BB_Suspend(l_HostName) + if (rc): + raise BB_SuspendError(rc) + + bb.printLastErrorDetailsSummary() + print('%sBB_Suspend completed for hostname %s' % (os.linesep, ValueMap.get(l_HostName.value, l_HostName.value))) + + return diff --git a/bb/wrappers/bbapiAdminProcs.py b/bb/wrappers/bbapiAdminProcs.py index 8a587e297..4a09f43e3 100644 --- a/bb/wrappers/bbapiAdminProcs.py +++ b/bb/wrappers/bbapiAdminProcs.py @@ -1,152 +1,185 @@ -#!/usr/bin/python -#################################################### -# bbapiAdminProcs.py -# -# Copyright IBM Corporation 2017,2017. All Rights Reserved -# -# This program is licensed under the terms of the Eclipse Public License -# v1.0 as published by the Eclipse Foundation and available at -# http://www.eclipse.org/legal/epl-v10.html -# -# U.S. Government Users Restricted Rights: Use, duplication or disclosure -# restricted by GSA ADP Schedule Contract with IBM Corp. -################################################### - -import subprocess -import time - -from bbapi import * -import bberror - - -# -# Helper routines -# - -def IssueCmd(pCmd): - l_RC = -1 - - l_Attempts = 1 - while (l_RC in (-1, )): - l_RC = 0 - try: - subprocess.check_call(pCmd, stderr=subprocess.STDOUT, shell=True) - except subprocess.CalledProcessError as error: - # NOTE: When invoking APIs via the subprocess module, - # we simply treat 0 and -2 as the only tolerated - # return codes. We do not tie into the extra - # capabilities of the return code processing - # of BBError for each of the APIs. - # Maybe someday, but not needed now... - # NOTE: If we receive a return code of -2, we still - # want to pass that back to our invoker. - # They may or may not tolerate the rc, but - # we will not signal the BBError exception. - l_RC = -1 - print "Failure when attempting to invoke subprocess.check_call() with command of |", pCmd, "|" - - l_Attempts += 1 - if (l_Attempts > 25): - l_RC = error.returncode - print "IssueCmd(): Throwing BBError, l_RC=", l_RC - raise bberror.BBError(rc=l_RC, text=pCmd) - else: - print "Attempting to re-submit the command, attempt number ", l_Attempts - time.sleep(12) - - return l_RC - - -# NOTE: Convention below is to ALWAYS pass --process_args even if there are no arguments to pass. -# In that case, pass "None". The prevents the shared library information from being sent -# to the console for the BB environments setup via the functions in this module... @DLH - -def sudo_ChangeMode(pEnv, pPathName, pMode): - l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s" % \ - (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "ChangeMode", pPathName, pMode) - return IssueCmd(l_Cmd) - -def sudo_ChangeOwner(pEnv, pPathName, pOwner, pGroup): - l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s,%s" % \ - (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "ChangeOwner", pPathName, pOwner, pGroup) - return IssueCmd(l_Cmd) - -# def sudo_CloseServer(pEnv, pName): -# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ -# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "CloseServer", pName) -# return IssueCmd(l_Cmd) - -def sudo_CreateDirectory(pEnv, pNewPathName): - l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ - (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "CreateDirectory", pNewPathName) - return IssueCmd(l_Cmd) - -def sudo_CreateLogicalVolume(pEnv, pMountpoint, pSize, pFlags=DEFAULT_BBCREATEFLAGS): - l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s,%s" % \ - (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "CreateLogicalVolume", pMountpoint, pSize, pFlags) - return IssueCmd(l_Cmd) - -# def sudo_GetServer(pEnv, pType): -# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ -# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "GetServer", pType) -# return IssueCmd(l_Cmd) - -# def sudo_OpenServer(pEnv, pName): -# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ -# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "OpenServer", pName) -# return IssueCmd(l_Cmd) - -def sudo_RemoveDirectory(pEnv, pPathName): - l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ - (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "RemoveDirectory", pPathName) - return IssueCmd(l_Cmd) - -def sudo_RemoveJobInfo(pEnv): - l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid %d --jobid_bump 0 --testcase RunProcedure --procedure %s --procedure_args %s" % \ - (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid"]+pEnv["jobid_bump"], "RemoveJobInfo", "None") - return IssueCmd(l_Cmd) - -def sudo_RemoveLogicalVolume(pEnv, pMountpoint): - l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ - (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "RemoveLogicalVolume", pMountpoint) - return IssueCmd(l_Cmd) - -def sudo_ResizeMountPoint(pEnv, pMountpoint, pSize, pFlags=DEFAULT_BBRESIZEFLAGS): - l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s,%s" % \ - (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "ResizeMountPoint", pMountpoint, pSize, pFlags) - return IssueCmd(l_Cmd) - -def sudo_RestartTransfers(pEnv, pHostName, pHandle, pTransferDefs, pTransferDefsSize): - l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s,%s,%s" % \ - (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "RestartTransfers", pEnv["procedure_args"], pHostName, pHandle, pTransferDefs, pTransferDefsSize) - return IssueCmd(l_Cmd) - -# def sudo_Resume(pEnv, pHostHame): -# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ -# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "Resume", pHostName) -# return IssueCmd(l_Cmd) - -# def sudo_RetrieveTransfers(pEnv, pHostHame, pHandle, pFlags=DEFAULT_sudo_RTV_TRANSFERDEFS_FLAGS): -# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s,%s" % \ -# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "RetrieveTransfers", pHostHame, pHandle, pFlags) -# return IssueCmd(l_Cmd)%s" % (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER - -# def sudo_SetServer(pEnv, pType, pName): -# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s" % \ -# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "SetServer", pType, pName) -# return IssueCmd(l_Cmd) - -def sudo_StageOutStart(pEnv, pMountpoint): - l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ - (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "StageOutStart", pMountpoint) - return IssueCmd(l_Cmd) - -# def sudo_StopTransfers(pEnv, pHostName, pHandle, pTransferDefs, pTransferDefsSize): -# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s,%s,%s" % \ -# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "StopServer", pHostName, pHandle, pTransferDefs, pTransferDefsSize) -# return IssueCmd(l_Cmd) - -# def sudo_Suspend(pEnv, pHostHame): -# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ -# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "Suspend", pHostHame) -# return IssueCmd(l_Cmd) +#!/usr/bin/python +#################################################### +# bbapiAdminProcs.py +# +# Copyright IBM Corporation 2017,2017. All Rights Reserved +# +# This program is licensed under the terms of the Eclipse Public License +# v1.0 as published by the Eclipse Foundation and available at +# http://www.eclipse.org/legal/epl-v10.html +# +# U.S. Government Users Restricted Rights: Use, duplication or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. +################################################### + +import os +import subprocess +import time + +from bbapi import * +import bberror + + +DEFAULT_PYTHON_INTERPRETER = '/u/dlherms/bin' + +# +# Helper routines +# + +def GetPythonPath(): + l_PythonLocation = None + + l_CompletedProcess = IssueCmd("which python", pCWD=DEFAULT_PYTHON_INTERPRETER, pReturnCompletedProcessInfo=True) + if (l_CompletedProcess and l_CompletedProcess.returncode == 0): + l_PythonLocation = l_CompletedProcess.stdout[:-1] + + return l_PythonLocation + +def IssueCmd(pCmd, pCWD='.', pReturnCompletedProcessInfo=False): + l_RC = -1 + l_CompletedProcess = None + + l_Attempts = 1 + while (l_RC in (-1, )): + l_RC = 0 + l_CompletedProcess = None + try: + l_CompletedProcess = subprocess.run(pCmd, universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + cwd=pCWD) + if (l_CompletedProcess and l_CompletedProcess.returncode): + l_RC = -1 + print("Failure returned from subprocess.run() with command of |", pCmd, + "|, returncode=", l_CompletedProcess.returncode, + ", stdout=", l_CompletedProcess.stdout, + ", stderr=", l_CompletedProcess.stderr) + + except Exception as error: +# except subprocess.CalledProcessError as error: + # NOTE: When invoking APIs via the subprocess module, + # we simply treat 0 and -2 as the only tolerated + # return codes. We do not tie into the extra + # capabilities of the return code processing + # of BBError for each of the APIs. + # Maybe someday, but not needed now... + # NOTE: If we receive a return code of -2, we still + # want to pass that back to our invoker. + # They may or may not tolerate the rc, but + # we will not signal the BBError exception. + l_RC = -1 + print("Failure when attempting to invoke subprocess.run() with command of |", pCmd, + "|, returncode=", l_CompletedProcess.returncode, + ", stdout=", l_CompletedProcess.stdout, + ", stderr=", l_CompletedProcess.stderr) + + if l_RC: + l_Attempts += 1 + if (l_Attempts > 25): + l_RC = l_CompletedProcess.returncode + print("IssueCmd(): Throwing BBError, l_RC=", l_RC) + raise bberror.BBError(rc=l_RC, text=pCmd) + else: + print("Attempting to re-submit the command, attempt number ", l_Attempts) + time.sleep(12) + + if pReturnCompletedProcessInfo: + return l_CompletedProcess + else: + return l_RC + + +# NOTE: Convention below is to ALWAYS pass --process_args even if there are no arguments to pass. +# In that case, pass "None". The prevents the shared library information from being sent +# to the console for the BB environments setup via the functions in this module... @DLH + +def sudo_ChangeMode(pEnv, pPathName, pMode): + l_Cmd = "sudo %s %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s" % \ + (GetPythonPath(), pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "ChangeMode", pPathName, pMode) + return IssueCmd(l_Cmd) + +def sudo_ChangeOwner(pEnv, pPathName, pOwner, pGroup): + l_Cmd = "sudo %s %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s,%s" % \ + (GetPythonPath(), pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "ChangeOwner", pPathName, pOwner, pGroup) + return IssueCmd(l_Cmd) + +# def sudo_CloseServer(pEnv, pName): +# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ +# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "CloseServer", pName) +# return IssueCmd(l_Cmd) + +def sudo_CreateDirectory(pEnv, pNewPathName): + l_Cmd = "sudo %s %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ + (GetPythonPath(), pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "CreateDirectory", pNewPathName) + return IssueCmd(l_Cmd) + +def sudo_CreateLogicalVolume(pEnv, pMountpoint, pSize, pFlags=DEFAULT_BBCREATEFLAGS): + l_Cmd = "sudo %s %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s,%s" % \ + (GetPythonPath(), pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "CreateLogicalVolume", pMountpoint, pSize, pFlags) + return IssueCmd(l_Cmd) + +# def sudo_GetServer(pEnv, pType): +# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ +# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "GetServer", pType) +# return IssueCmd(l_Cmd) + +# def sudo_OpenServer(pEnv, pName): +# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ +# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "OpenServer", pName) +# return IssueCmd(l_Cmd) + +def sudo_RemoveDirectory(pEnv, pPathName): + l_Cmd = "sudo %s %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ + (GetPythonPath(), pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "RemoveDirectory", pPathName) + return IssueCmd(l_Cmd) + +def sudo_RemoveJobInfo(pEnv): + l_Cmd = "sudo %s %s %s --libpath %s --testpath %s --iteration %d --jobid %d --jobid_bump 0 --testcase RunProcedure --procedure %s --procedure_args %s" % \ + (GetPythonPath(), pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid"]+pEnv["jobid_bump"], "RemoveJobInfo", "None") + return IssueCmd(l_Cmd) + +def sudo_RemoveLogicalVolume(pEnv, pMountpoint): + l_Cmd = "sudo %s %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ + (GetPythonPath(), pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "RemoveLogicalVolume", pMountpoint) + return IssueCmd(l_Cmd) + +def sudo_ResizeMountPoint(pEnv, pMountpoint, pSize, pFlags=DEFAULT_BBRESIZEFLAGS): + l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s,%s" % \ + (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "ResizeMountPoint", pMountpoint, pSize, pFlags) + return IssueCmd(l_Cmd) + +def sudo_RestartTransfers(pEnv, pHostName, pHandle, pTransferDefs, pTransferDefsSize): + l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s,%s,%s" % \ + (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "RestartTransfers", pEnv["procedure_args"], pHostName, pHandle, pTransferDefs, pTransferDefsSize) + return IssueCmd(l_Cmd) + +# def sudo_Resume(pEnv, pHostHame): +# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ +# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "Resume", pHostName) +# return IssueCmd(l_Cmd) + +# def sudo_RetrieveTransfers(pEnv, pHostHame, pHandle, pFlags=DEFAULT_sudo_RTV_TRANSFERDEFS_FLAGS): +# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s,%s" % \ +# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "RetrieveTransfers", pHostHame, pHandle, pFlags) +# return IssueCmd(l_Cmd)%s" % (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER + +# def sudo_SetServer(pEnv, pType, pName): +# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s" % \ +# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "SetServer", pType, pName) +# return IssueCmd(l_Cmd) + +def sudo_StageOutStart(pEnv, pMountpoint): + l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ + (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "StageOutStart", pMountpoint) + return IssueCmd(l_Cmd) + +# def sudo_StopTransfers(pEnv, pHostName, pHandle, pTransferDefs, pTransferDefsSize): +# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s,%s,%s,%s" % \ +# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "StopServer", pHostName, pHandle, pTransferDefs, pTransferDefsSize) +# return IssueCmd(l_Cmd) + +# def sudo_Suspend(pEnv, pHostHame): +# l_Cmd = "sudo python %s %s --libpath %s --testpath %s --iteration %d --jobid_bump %d --testcase RunProcedure --procedure %s --procedure_args %s" % \ +# (pEnv["COMMAND"], pEnv["COMMAND_LINE_ARGS"], pEnv["LIBPATH"], pEnv["WRAPPER_PATH"], pEnv["iteration"], pEnv["jobid_bump"], "Suspend", pHostHame) +# return IssueCmd(l_Cmd) diff --git a/bb/wrappers/bbapiTest.py b/bb/wrappers/bbapiTest.py index 63ac6dcea..b81e364ed 100644 --- a/bb/wrappers/bbapiTest.py +++ b/bb/wrappers/bbapiTest.py @@ -1,77 +1,77 @@ -#!/usr/bin/python -#################################################### -# bbapiTest.py -# -# Copyright IBM Corporation 2017,2017. All Rights Reserved -# -# This program is licensed under the terms of the Eclipse Public License -# v1.0 as published by the Eclipse Foundation and available at -# http://www.eclipse.org/legal/epl-v10.html -# -# U.S. Government Users Restricted Rights: Use, duplication or disclosure -# restricted by GSA ADP Schedule Contract with IBM Corp. -################################################### - -""" - This module provides wrappers to the following bbapi test functions. - -int Coral_GetVar(const char* pVariable); -int Coral_SetVar(const char* pVariable, const char* pValue); -int Coral_StageOutStart(const char* pMountpoint); -""" - -from ctypes import * -# import json - -import bb -from bberror import * - -PRINT_VALUE = False - - -# -# Wrappers for test bbapi calls -# - -# // NOTE: Coral_G/SetVar currently only supports writing ascii values to be read as positive integers -def Coral_GetVar(pVariable, pPrintValue=PRINT_VALUE): - l_Variable = bb.cvar("variable", pVariable) - l_Value = bb.api.Coral_GetVar(l_Variable) - if pPrintValue: - print "===> Variable %s = %d" % (pVariable, l_Value) - - return l_Value - -# // NOTE: BB_G/SetVar currently only supports writing ascii values to be read as positive integers -def Coral_SetVar(pVariable, pValue): - try: - l_Value = int(pValue) - if (l_Value < 0): - raise ValueError - - except ValueError: - l_Value = None - - if (l_Value != None): - l_Variable = bb.cvar("variable", pVariable) - l_Value = bb.cvar("value", pValue) - l_PreviousValue = Coral_GetVar(pVariable) -# print "===> The value for variable '%s' is being changed from %d to %s" % (pVariable, l_PreviousValue, pValue) - l_Value = bb.api.Coral_SetVar(l_Variable, l_Value) - l_CurrentValue = Coral_GetVar(pVariable, False) - if (int(pValue) != l_CurrentValue): - print "===> Coral_SetVar Error: Variable %s could not be set to a value of %s" % (pVariable, pValue) - else: - print "===> Coral_SetVar Error: Input value of %s not 0 or a positive value" % (pValue) - - return - -def Coral_StageOutStart(pMountpoint): - l_Mountpoint = bb.cvar("mountpoint", pMountpoint) - - print "%sCoral_StageOutStart issued for mountpoint %s" % (os.linesep, pMountpoint) - rc = bb.api.Coral_StageOutStart(l_Mountpoint) - if (rc): - raise Coral_StageOutStartError(rc) - - return +#!/usr/bin/python +#################################################### +# bbapiTest.py +# +# Copyright IBM Corporation 2017,2017. All Rights Reserved +# +# This program is licensed under the terms of the Eclipse Public License +# v1.0 as published by the Eclipse Foundation and available at +# http://www.eclipse.org/legal/epl-v10.html +# +# U.S. Government Users Restricted Rights: Use, duplication or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. +################################################### + +""" + This module provides wrappers to the following bbapi test functions. + +int Coral_GetVar(const char* pVariable); +int Coral_SetVar(const char* pVariable, const char* pValue); +int Coral_StageOutStart(const char* pMountpoint); +""" + +from ctypes import * +# import json + +import bb +from bberror import * + +PRINT_VALUE = False + + +# +# Wrappers for test bbapi calls +# + +# // NOTE: Coral_G/SetVar currently only supports writing ascii values to be read as positive integers +def Coral_GetVar(pVariable, pPrintValue=PRINT_VALUE): + l_Variable = bb.cvar("variable", pVariable) + l_Value = bb.api.Coral_GetVar(l_Variable) + if pPrintValue: + print("===> Variable %s = %d" % (pVariable, l_Value)) + + return l_Value + +# // NOTE: BB_G/SetVar currently only supports writing ascii values to be read as positive integers +def Coral_SetVar(pVariable, pValue): + try: + l_Value = int(pValue) + if (l_Value < 0): + raise ValueError + + except ValueError: + l_Value = None + + if (l_Value != None): + l_Variable = bb.cvar("variable", pVariable) + l_Value = bb.cvar("value", pValue) + l_PreviousValue = Coral_GetVar(pVariable) +# print "===> The value for variable '%s' is being changed from %d to %s" % (pVariable, l_PreviousValue, pValue) + l_Value = bb.api.Coral_SetVar(l_Variable, l_Value) + l_CurrentValue = Coral_GetVar(pVariable, False) + if (int(pValue) != l_CurrentValue): + print("===> Coral_SetVar Error: Variable %s could not be set to a value of %s" % (pVariable, pValue)) + else: + print("===> Coral_SetVar Error: Input value of %s not 0 or a positive value" % (pValue)) + + return + +def Coral_StageOutStart(pMountpoint): + l_Mountpoint = bb.cvar("mountpoint", pMountpoint) + + print("%sCoral_StageOutStart issued for mountpoint %s" % (os.linesep, pMountpoint)) + rc = bb.api.Coral_StageOutStart(l_Mountpoint) + if (rc): + raise Coral_StageOutStartError(rc) + + return diff --git a/bb/wrappers/bbapi_main.py b/bb/wrappers/bbapi_main.py index c69081d99..2e8bb2427 100644 --- a/bb/wrappers/bbapi_main.py +++ b/bb/wrappers/bbapi_main.py @@ -1,431 +1,444 @@ -#!/usr/bin/python -#################################################### -# bbapi_main.py -# -# Copyright IBM Corporation 2017,2017. All Rights Reserved -# -# This program is licensed under the terms of the Eclipse Public License -# v1.0 as published by the Eclipse Foundation and available at -# http://www.eclipse.org/legal/epl-v10.html -# -# U.S. Government Users Restricted Rights: Use, duplication or disclosure -# restricted by GSA ADP Schedule Contract with IBM Corp. -################################################### - - -""" - This module is the main driver of the bbapi functions using Python wrappers. - The following operations are performed: - 1) Invokes BB_InitLibrary() - 2) Invokes the identified Python testcase - 3) Invokes BB_TerminateLibrary() - - Options: - -h --help Display usage information - -a --cn_failover CN failover value. Default is 0, which is False. Any other integer value is True. - -b --io_failover ESS failover value. Default is 0, which is False. Any other integer value is True. - -c --contrib Contrib value to use. Default is (0,). - -d --name Name for Set/Open/CloseServer. No default. - -e --floor Floor path. Default is /opt/ibm - NOTE: This value is ignored if libpath is provided. - -f --flags Flags. No default. - -g --group Group name to use. Default is "users". - -h --handle Handle value. Default is 0. - -i --hostname Hostname to use. Default is "". - -j --jobid Jobid value to use. Default is 1. - -k --jobstepid Jobstepid value to use. Default is 1. - -l --libpath Library path to libbbAPI.so. Default is "$FLOOR/bb/lib". - -m --mount Path for mountpoint(s). Default is "/tmp/$USER/mnt". - -n --mode Mode value to use. No default. If not specified and needed, - the testcase must provide it. - -o --owner Owner value to use. Default is $USER. - -p --testpath Path to testcase to run. Default is ".", the current working directory. - -q --orgsrc Path to original source files. Default is "/gpfs/gpfs0/$USER/source". - -r --contribid Contribid value to use. Default is 0. - -s --size Size of the logical volume to create. Default is "1G". - NOTE: If specifying a value like one-half gigabyte, the leading - zero MUST be specified. (e.g., 0.5G and not .5G) - -t --tag Tag value to use. Default is 1. - -u --unixpath Unixpath to use. No default. If not specified, the value is - obtained from the configuration file. - -v --type Type for GetServer. No default. - -w --procedure Procedure to invoke. - -x --target Path prefix for target(s) Default is "/gpfs/gpfs0/$USER/target". - -y --testcase Testcase to run. Default is "bbtests". - NOTE: This must be a python script file, but do not include the .py - -z --config Configuration file to use. Default is "/etc/ibm/bb.cfg". - -1 --process_args Process arguments (bbapiAdminProcs interface only) - -2 --iteration Iteration number (NOTE: Test convention is to add 10 times this value - to the jobid value in initEnv() to prevent jobid - collision. Used to generate unique jobid values - by iteration.) - -3 --jobid_bump JobId bump value (NOTE: Test convention is to add this value - to the jobid value in initEnv() to prevent jobid - collision. Used to generate unique jobid values - within a testcase.) - -4 --cancelscope Cancel scope. Default is BBSCOPETAG. - - All paths can be given as either absolute or relative to the current working directory. - - This main program establishes a burst buffer environment for an already running - instance of bbServer/bbProxy. It basically will: - - Invoke BB_InitLibrary() - - Invoke the specified testcase. - This testcase may run one or more 'variations'. - This testcase will typically create any necessary - libraries/files and any necessary logical volume(s). - - Invoke BB_TerminateLibrary() - - The options available on this main program allow tests to: - - Stage-in files from an original source location (--orgsrc) to - the mountpoint location (--mount), which is intended to be mounted - on a logical volume that will allocate space from the bb volume group. - - Stage-out files from the mountpoint location (--mount) to - the target location (--target). - - Typical example using a personal sandbox: - ~/git/bluecoral/bb/wrappers -> python bbapi_main.py --libpath ~/git/bluecoral/work/bb/lib - - The current working directory is the source side bb/wrappers directory. This is so you can make changes directly - to the python source code and have it immediately take effect for the next run. Otherwise, you have to install the - python changes to the /work side before they would take effect. - - The configuration file that is specified is the same as was used to start the bbServer/bbProxy that is to be - tested. - - The libpath specified is the library where the libbbAPI.so can be found. It must be the same libbbAPI.so that is - associated with the bbServer/bbProxy that is to be tested. - - If the configuration file to be used is the normal configuration file for a sandbox and libpath is specified, - the default configuration file used is one directory up from the libpath specification appended with "scripts/bb.cfg". - Therefore, if libpath is specified and the normal sandbox configuration file is to be used, the config specification - can be omitted. -""" - -import ctypes -import getopt -import importlib -import os -import pprint -import sys - -import bb -from bbapi import BBCANCELSCOPE, DEFAULT_BBCANCELSCOPE, INVALID_BBCANCELSCOPE - -# -# Default values -# - -DEFAULT_TESTPATH = "." -DEFAULT_TESTCASE = "bbtests" - -DEFAULT_CONTRIB = (0,) -DEFAULT_CONTRIBID = 0 -DEFAULT_JOBID = 1 -DEFAULT_JOBSTEPID = 1 -DEFAULT_TAG = 1 - -DEFAULT_HANDLE = 0 -DEFAULT_HOSTNAME = "" - -l_Temp = os.environ['USER'] -DEFAULT_CONFIG = (os.path.join(*("%s,etc,ibm,bb.cfg" % (os.path.sep)).split(","))) -DEFAULT_ORGSRC = (os.path.join(*("%s,gpfs,gpfs0,%s,source" % (os.path.sep, l_Temp)).split(","))) -DEFAULT_MOUNT = (os.path.join(*("%s,tmp,%s,mnt" % (os.path.sep, l_Temp)).split(","))) -DEFAULT_TARGET = (os.path.join(*("%s,gpfs,gpfs0,%s,target" % (os.path.sep, l_Temp)).split(","))) -DEFAULT_OWNER = l_Temp -DEFAULT_GROUP = "users" -DEFAULT_SIZE = "1.5G" - -DEFAULT_CN_FAILOVER = False -DEFAULT_IO_FAILOVER = False - -# NOTE: When --procedure_args is a zero length, the shared library infomation is sent to the console. -# When invoked via the command line, it will be the DEFAULT and the length will be zero. -# When invoked via bbapiAdminProcs, it is always passed as a value, even if a particular -# API doesn't have any arguments to pass. -DEFAULT_PROCEDURE_ARGS = "" -DEFAULT_ITERATION = 0 -DEFAULT_JOBID_BUMP = 0 - -NOCONFIG = "" -NOUNIXPATH = "" -ALL = "*" - -# -# Wrappers for bbapi calls needed for testcase setup -# -def BB_GetVersion(pSize, pAPI_Version): - return bb.api.BB_GetVersion(pSize.value, pAPI_Version) - -def BB_InitLibrary(pContribId, pClientVersion): - return bb.api.BB_InitLibrary(pContribId.value, pClientVersion) - -def BB_TerminateLibrary(): - return bb.api.BB_TerminateLibrary() - -def Coral_InitLibrary(pContribId, pClientVersion, pConfig, pUnixPath): - return bb.api.Coral_InitLibrary(pContribId.value, pClientVersion, pConfig, pUnixPath) - - -# -# Helper routines -# - -def usage(code, msg=''): - print >> sys.stderr, __doc__ - if msg: - print >> sys.stderr, msg - sys.exit(code) - - -def setDefaults(pEnv): - setDefaultFloor(pEnv) - setLibPath(pEnv) - pEnv["cancelscope"] = DEFAULT_BBCANCELSCOPE - pEnv["contrib"] = DEFAULT_CONTRIB - pEnv["contribid"] = DEFAULT_CONTRIBID - pEnv["handle"] = DEFAULT_HANDLE - pEnv["hostname"] = DEFAULT_HOSTNAME - pEnv["jobid"] = DEFAULT_JOBID - pEnv["jobstepid"] = DEFAULT_JOBSTEPID - pEnv["tag"] = DEFAULT_TAG - - pEnv['CONFIG'] = DEFAULT_CONFIG - - pEnv["ORGSRC"] = DEFAULT_ORGSRC - pEnv["MOUNT"] = DEFAULT_MOUNT - pEnv["TARGET"] = DEFAULT_TARGET - pEnv["SIZE"] = DEFAULT_SIZE - - pEnv["OWNER"] = DEFAULT_OWNER - pEnv["GROUP"] = DEFAULT_GROUP - - pEnv["CN_FAILOVER"] = DEFAULT_CN_FAILOVER - pEnv["IO_FAILOVER"] = DEFAULT_IO_FAILOVER - - setDefaultTestPath(pEnv) - pEnv["TESTCASE"] = DEFAULT_TESTCASE - pEnv["procedure_args"] = DEFAULT_PROCEDURE_ARGS - # NOTE: The iteration value can be passed into the main() - # routine of this module - if (not pEnv.has_key("iteration")): - pEnv["iteration"] = DEFAULT_ITERATION - # NOTE: The jobid_bump value can be passed into the main() - # routine of this module - if (not pEnv.has_key("jobid_bump")): - pEnv["jobid_bump"] = DEFAULT_JOBID_BUMP - - return - -def setDefaultFloor(pEnv): - pEnv["FLOOR"] = os.path.join(*(",%s,opt,ibm" % (os.path.sep)).split(",")) - - return - -def setDefaultTestCase(pEnv): - pEnv["TESTCASE"] = DEFAULT_TESTCASE - - return - -def setDefaultTestPath(pEnv): - pEnv["TESTPATH"] = os.path.abspath(DEFAULT_TESTPATH) - - return - -def setLibPath(pEnv): - pEnv["LIBPATH"] = pEnv["FLOOR"] + os.path.join(*("%s,bb,lib" % (os.path.sep)).split(",")) - - return - -def setLib(pEnv): - pEnv["LIB"] = pEnv["LIBPATH"] + os.path.join(*("%s,libbbAPI.so" % (os.path.sep)).split(",")) - - return - -def setSysPath(pEnv): - if pEnv["TESTPATH"] not in sys.path: - sys.path.append(pEnv["TESTPATH"]) -# print "sys.path=" -# pprint.pprint(sys.path) - - return - -def setWrapperPath(pEnv): - l_LibPath = pEnv["LIBPATH"].split(os.path.sep) - pEnv["WRAPPER_PATH"] = os.path.sep.join(l_LibPath[:-1]) + os.path.sep + 'wrappers' - - return - -def processArgs(pEnv, pArgs): - setDefaults(pEnv) - - try: - l_Opts, l_Args = getopt.getopt(pArgs,"ha:b:c:d:e:f:g:h:i:j:k:l:m:n:o:p:q:r:s:t:u:v:w:x:y:z:1:2:3:4:",["cn_failover=","io_failover=","contrib=","name=","floor=","flags=","group=","handle=","hostname=","jobid=","jobstepid=","libpath=","mount=","mode=","owner=","testpath=","orgsrc=","contribid=","size=","tag=","unixpath=","type=","procedure=","target=","testcase=","config=","procedure_args=","iteration=","jobid_bump=","cancelscope="]) - except getopt.GetoptError as e: - print >> sys.stderr, e - usage(1, "Invalid arguments passed") - - l_ConfigSpecified = False - l_LibPathSpecified = False - for l_Opt, l_Arg in l_Opts: -# print l_Opt, l_Arg - if l_Opt == '-h': - print __doc__ - sys.exit() - elif l_Opt in ("-a", "--cn_failover"): - if (l_Arg == "0"): - pEnv["CN_FAILOVER"] = False - else: - pEnv["CN_FAILOVER"] = True - elif l_Opt in ("-b", "--io_failover"): - if (l_Arg == "0"): - pEnv["IO_FAILOVER"] = False - else: - pEnv["IO_FAILOVER"] = True - elif l_Opt in ("-c", "--contrib"): - l_Temp = l_Arg.split(",") - pEnv["contrib"] = tuple(map(int, l_Temp)) - elif l_Opt in ("-d", "--name"): - pEnv["name"] = l_Arg - elif l_Opt in ("-e", "--floor"): - pEnv["FLOOR"] = os.path.abspath(l_Arg) - setLibPath(pEnv["FLOOR"]) - elif l_Opt in ("-f", "--flags"): - pEnv["FLAGS"] = l_Arg - elif l_Opt in ("-g", "--group"): - pEnv["GROUP"] = l_Arg - elif l_Opt in ("-h", "--handle"): - if (str(l_Arg).upper() != ALL): - pEnv["handle"] = int(l_Arg) - else: - pEnv["handle"] = bb.NO_HANDLE - elif l_Opt in ("-i", "--hostname"): - if (str(l_Arg).upper() != ALL): - pEnv["hostname"] = l_Arg - else: - pEnv["hostname"] = bb.NO_HOSTNAME - elif l_Opt in ("-j", "--jobid"): - if (str(l_Arg).upper() != ALL): - pEnv["jobid"] = int(l_Arg) - else: - pEnv["jobid"] = bb.NO_JOBID - elif l_Opt in ("-k", "--jobstepid"): - if (str(l_Arg).upper() != ALL): - pEnv["jobstepid"] = int(l_Arg) - else: - pEnv["jobstepid"] = bb.NO_JOBSTEPID - elif l_Opt in ("-l", "--libpath"): - pEnv["LIBPATH"] = os.path.abspath(l_Arg) - l_LibPathSpecified = True - elif l_Opt in ("-m", "--mount"): - pEnv["MOUNT"] = os.path.abspath(l_Arg) - elif l_Opt in ("-n", "--mode"): - pEnv["MODE"] = int(l_Arg) - elif l_Opt in ("-o", "--owner"): - pEnv["OWNER"] = l_Arg - elif l_Opt in ("-p", "--testpath"): - pEnv["TESTPATH"] = os.path.abspath(l_Arg) - elif l_Opt in ("-q", "--orgsrc"): - pEnv["ORGSRC"] = os.path.abspath(l_Arg) - elif l_Opt in ("-r", "--contribid"): - if (str(l_Arg).upper() != ALL): - pEnv["contribid"] = int(l_Arg) - else: - pEnv["contribid"] = bb.NO_CONTRIBID - elif l_Opt in ("-s", "--size"): - pEnv["SIZE"] = l_Arg - elif l_Opt in ("-t", "--tag"): - pEnv["tag"] = int(l_Arg) - elif l_Opt in ("-u", "--unixpath"): - pEnv["UNIXPATH"] = os.path.abspath(l_Arg) - elif l_Opt in ("-v", "--type"): - pEnv["type"] = l_Arg - elif l_Opt in ("-w", "--procedure"): - pEnv["procedure"] = l_Arg - elif l_Opt in ("-x", "--target"): - pEnv["TARGET"] = os.path.abspath(l_Arg) - elif l_Opt in ("-y", "--testcase"): - pEnv["TESTCASE"] = l_Arg - elif l_Opt in ("-z", "--config"): - pEnv["CONFIG"] = os.path.abspath(l_Arg) - l_ConfigSpecified = True - elif l_Opt in ("-1", "--procedure_args"): - pEnv["procedure_args"] = os.path.abspath(l_Arg) - elif l_Opt in ("-2", "--iteration"): - pEnv["iteration"] = int(l_Arg) - elif l_Opt in ("-3", "--jobid_bump"): - pEnv["jobid_bump"] = int(l_Arg) - elif l_Opt in ("-4", "--cancelscope"): - pEnv["cancelscope"] = BBCANCELSCOPE[l_Arg] - - pEnv["COMMAND"] = os.path.abspath(__file__) - pEnv["COMMAND_LINE_ARGS"] = " ".join(pArgs) - setWrapperPath(pEnv) - - setLib(pEnv) -# pprint.pprint(pEnv) - - return - - -def buildLibraryWrapper(pEnv): - - return ctypes.CDLL(pEnv["LIB"]) - - -def main(pArgs): - l_Env = {} - processArgs(l_Env, pArgs) - - # Build the ctypes interfaces to the BB APIs... - bb.api = buildLibraryWrapper(l_Env) - - # We only print out the shared library information for the main routine... - if (len(l_Env["procedure_args"]) == 0): - print "Shared library: %s" % (l_Env["LIB"]) - - l_Contribid = bb.cvar("contribid", l_Env) - - l_Size = bb.cvar("size", 256) - l_API_Version = ctypes.create_string_buffer(256) - rc = BB_GetVersion(l_Size, l_API_Version) - - if (rc == 0): - if (l_Env["TESTCASE"] != "None"): - if l_Env.has_key("CONFIG") or l_Env.has_key("UNIXPATH"): - l_Config = ctypes.c_char_p(l_Env.get("CONFIG", NOCONFIG)) - l_UnixPath = ctypes.c_char_p(l_Env.get("UNIXPATH", NOUNIXPATH)) - rc = Coral_InitLibrary(l_Contribid, l_API_Version, l_Config, l_UnixPath) - else: - rc = BB_InitLibrary(l_Contribid, l_API_Version) - - if (rc == 0): - setSysPath(l_Env) - l_TestCase = importlib.import_module(l_Env["TESTCASE"]) - rc = l_TestCase.main(l_Env) - print "bbapi_main.main(): rc = %d" % (rc) - else: - print "BB_InitLibrary(): rc = %d" % (rc) - - BB_TerminateLibrary() - else: - print "BB_GetVersion(): rc = %d" % (rc) - - return rc - - -# -# Invoke main routine -# - -if __name__ == '__main__': - rc = main(sys.argv[1:]) - - if (rc in (0, -2)): - rc = 0 - else: - rc = -1 - print "bbapi_main: rc = %d" % (rc) - - sys.exit(rc) +#!/usr/bin/python +#################################################### +# bbapi_main.py +# +# Copyright IBM Corporation 2017,2017. All Rights Reserved +# +# This program is licensed under the terms of the Eclipse Public License +# v1.0 as published by the Eclipse Foundation and available at +# http://www.eclipse.org/legal/epl-v10.html +# +# U.S. Government Users Restricted Rights: Use, duplication or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. +################################################### + + +""" + This module is the main driver of the bbapi functions using Python wrappers. + The following operations are performed: + 1) Invokes BB_InitLibrary() + 2) Invokes the identified Python testcase + 3) Invokes BB_TerminateLibrary() + + Options: + -h --help Display usage information + -a --cn_failover CN failover value. Default is 0, which is False. Any other integer value is True. + -b --io_failover ESS failover value. Default is 0, which is False. Any other integer value is True. + -c --contrib Contrib value to use. Default is (0,). + -d --name Name for Set/Open/CloseServer. No default. + -e --floor Floor path. Default is /opt/ibm + NOTE: This value is ignored if libpath is provided. + -f --flags Flags. No default. + -g --group Group name to use. Default is "users". + -h --handle Handle value. Default is 0. + -i --hostname Hostname to use. Default is "". + -j --jobid Jobid value to use. Default is 1. + -k --jobstepid Jobstepid value to use. Default is 1. + -l --libpath Library path to libbbAPI.so. Default is "$FLOOR/bb/lib". + -m --mount Path for mountpoint(s). Default is "/tmp/$USER/mnt". + -n --mode Mode value to use. No default. If not specified and needed, + the testcase must provide it. + -o --owner Owner value to use. Default is $USER. + -p --testpath Path to testcase to run. Default is ".", the current working directory. + -q --orgsrc Path to original source files. Default is "/gpfs/gpfs0/$USER/source". + -r --contribid Contribid value to use. Default is 0. + -s --size Size of the logical volume to create. Default is "1G". + NOTE: If specifying a value like one-half gigabyte, the leading + zero MUST be specified. (e.g., 0.5G and not .5G) + -t --tag Tag value to use. Default is 1. + -u --unixpath Unixpath to use. No default. If not specified, the value is + obtained from the configuration file. + -v --type Type for GetServer. No default. + -w --procedure Procedure to invoke. + -x --target Path prefix for target(s) Default is "/gpfs/gpfs0/$USER/target". + -y --testcase Testcase to run. Default is "bbtests". + NOTE: This must be a python script file, but do not include the .py + -z --config Configuration file to use. Default is "/etc/ibm/bb.cfg". + -1 --process_args Process arguments (bbapiAdminProcs interface only) + -2 --iteration Iteration number (NOTE: Test convention is to add 10 times this value + to the jobid value in initEnv() to prevent jobid + collision. Used to generate unique jobid values + by iteration.) + -3 --jobid_bump JobId bump value (NOTE: Test convention is to add this value + to the jobid value in initEnv() to prevent jobid + collision. Used to generate unique jobid values + within a testcase.) + -4 --cancelscope Cancel scope. Default is BBSCOPETAG. + + All paths can be given as either absolute or relative to the current working directory. + + This main program establishes a burst buffer environment for an already running + instance of bbServer/bbProxy. It basically will: + - Invoke BB_InitLibrary() + - Invoke the specified testcase. + This testcase may run one or more 'variations'. + This testcase will typically create any necessary + libraries/files and any necessary logical volume(s). + - Invoke BB_TerminateLibrary() + + The options available on this main program allow tests to: + - Stage-in files from an original source location (--orgsrc) to + the mountpoint location (--mount), which is intended to be mounted + on a logical volume that will allocate space from the bb volume group. + - Stage-out files from the mountpoint location (--mount) to + the target location (--target). + + Typical example using a personal sandbox: + ~/git/bluecoral/bb/wrappers -> python bbapi_main.py --libpath ~/git/bluecoral/work/bb/lib + + The current working directory is the source side bb/wrappers directory. This is so you can make changes directly + to the python source code and have it immediately take effect for the next run. Otherwise, you have to install the + python changes to the /work side before they would take effect. + + The configuration file that is specified is the same as was used to start the bbServer/bbProxy that is to be + tested. + + The libpath specified is the library where the libbbAPI.so can be found. It must be the same libbbAPI.so that is + associated with the bbServer/bbProxy that is to be tested. + + If the configuration file to be used is the normal configuration file for a sandbox and libpath is specified, + the default configuration file used is one directory up from the libpath specification appended with "scripts/bb.cfg". + Therefore, if libpath is specified and the normal sandbox configuration file is to be used, the config specification + can be omitted. +""" + +import ctypes +import getopt +import importlib +import os +import pprint +import sys + +import bb +from bbapi import BBCANCELSCOPE, DEFAULT_BBCANCELSCOPE, INVALID_BBCANCELSCOPE + +# +# Default values +# + +DEFAULT_TESTPATH = "." +DEFAULT_TESTCASE = "bbtests" + +DEFAULT_CONTRIB = (0,) +DEFAULT_CONTRIBID = 0 +DEFAULT_JOBID = 1 +DEFAULT_JOBSTEPID = 1 +DEFAULT_TAG = 1 + +DEFAULT_HANDLE = 0 +DEFAULT_HOSTNAME = "" + +l_Temp = os.environ['USER'] +#DEFAULT_CONFIG = (os.path.join(*("%s,etc,ibm,bb.cfg" % (os.path.sep)).split(","))) +DEFAULT_CONFIG = (os.path.join(*("%s,u,dlherms,CAST,work,bb,scripts,bb.cfg" % (os.path.sep)).split(","))) +DEFAULT_ORGSRC = (os.path.join(*("%s,gpfs,gpfs0,%s,source" % (os.path.sep, l_Temp)).split(","))) +DEFAULT_MOUNT = (os.path.join(*("%s,tmp,%s,mnt" % (os.path.sep, l_Temp)).split(","))) +DEFAULT_TARGET = (os.path.join(*("%s,gpfs,gpfs0,%s,target" % (os.path.sep, l_Temp)).split(","))) +DEFAULT_OWNER = l_Temp +DEFAULT_GROUP = "users" +DEFAULT_SIZE = "1.5G" + +DEFAULT_CN_FAILOVER = False +DEFAULT_IO_FAILOVER = False + +# NOTE: When --procedure_args is a zero length, the shared library infomation is sent to the console. +# When invoked via the command line, it will be the DEFAULT and the length will be zero. +# When invoked via bbapiAdminProcs, it is always passed as a value, even if a particular +# API doesn't have any arguments to pass. +DEFAULT_PROCEDURE_ARGS = "" +DEFAULT_ITERATION = 0 +DEFAULT_JOBID_BUMP = 0 + +NOCONFIG = "" +NOUNIXPATH = "" +ALL = "*" + +# +# Wrappers for bbapi calls needed for testcase setup +# +def BB_GetVersion(pSize, pAPI_Version): + return bb.api.BB_GetVersion(pSize.value, pAPI_Version) + +def BB_InitLibrary(pContribId, pClientVersion): + return bb.api.BB_InitLibrary(pContribId.value, pClientVersion) + +def BB_TerminateLibrary(): + return bb.api.BB_TerminateLibrary() + +def Coral_InitLibrary(pContribId, pClientVersion, pConfig, pUnixPath): + return bb.api.Coral_InitLibrary(pContribId.value, pClientVersion, pConfig, pUnixPath) + + +# +# Helper routines +# + +def usage(code, msg=''): + print(__doc__, file=sys.stderr) + if msg: + print(msg, file=sys.stderr) + sys.exit(code) + + +def setDefaults(pEnv): + setDefaultFloor(pEnv) + setLibPath(pEnv) + pEnv["cancelscope"] = DEFAULT_BBCANCELSCOPE + pEnv["contrib"] = DEFAULT_CONTRIB + pEnv["contribid"] = DEFAULT_CONTRIBID + pEnv["handle"] = DEFAULT_HANDLE + pEnv["hostname"] = DEFAULT_HOSTNAME + pEnv["jobid"] = DEFAULT_JOBID + pEnv["jobstepid"] = DEFAULT_JOBSTEPID + pEnv["tag"] = DEFAULT_TAG + + pEnv['CONFIG'] = DEFAULT_CONFIG + + pEnv["ORGSRC"] = DEFAULT_ORGSRC + pEnv["MOUNT"] = DEFAULT_MOUNT + pEnv["TARGET"] = DEFAULT_TARGET + pEnv["SIZE"] = DEFAULT_SIZE + + pEnv["OWNER"] = DEFAULT_OWNER + pEnv["GROUP"] = DEFAULT_GROUP + + pEnv["CN_FAILOVER"] = DEFAULT_CN_FAILOVER + pEnv["IO_FAILOVER"] = DEFAULT_IO_FAILOVER + + setDefaultTestPath(pEnv) + pEnv["TESTCASE"] = DEFAULT_TESTCASE + pEnv["procedure_args"] = DEFAULT_PROCEDURE_ARGS + # NOTE: The iteration value can be passed into the main() + # routine of this module + if ("iteration" not in pEnv): + pEnv["iteration"] = DEFAULT_ITERATION + # NOTE: The jobid_bump value can be passed into the main() + # routine of this module + if ("jobid_bump" not in pEnv): + pEnv["jobid_bump"] = DEFAULT_JOBID_BUMP + + return + +def setDefaultFloor(pEnv): + pEnv["FLOOR"] = os.path.join(*(",%s,opt,ibm" % (os.path.sep)).split(",")) + + return + +def setDefaultTestCase(pEnv): + pEnv["TESTCASE"] = DEFAULT_TESTCASE + + return + +def setDefaultTestPath(pEnv): + pEnv["TESTPATH"] = os.path.abspath(DEFAULT_TESTPATH) + + return + +def setLibPath(pEnv): + pEnv["LIBPATH"] = pEnv["FLOOR"] + os.path.join(*("%s,bb,lib" % (os.path.sep)).split(",")) + + return + +def setLib(pEnv): + pEnv["LIB"] = pEnv["LIBPATH"] + os.path.join(*("%s,libbbAPI.so" % (os.path.sep)).split(",")) + + return + +def setSysPath(pEnv): + if pEnv["TESTPATH"] not in sys.path: + sys.path.append(pEnv["TESTPATH"]) +# print "sys.path=" +# pprint.pprint(sys.path) + + return + +def setWrapperPath(pEnv): + l_LibPath = pEnv["LIBPATH"].split(os.path.sep) + pEnv["WRAPPER_PATH"] = os.path.sep.join(l_LibPath[:-1]) + os.path.sep + 'wrappers' + + return + +def processArgs(pEnv, pArgs): + setDefaults(pEnv) + + try: + l_Opts, l_Args = getopt.getopt(pArgs,"ha:b:c:d:e:f:g:h:i:j:k:l:m:n:o:p:q:r:s:t:u:v:w:x:y:z:1:2:3:4:",["cn_failover=","io_failover=","contrib=","name=","floor=","flags=","group=","handle=","hostname=","jobid=","jobstepid=","libpath=","mount=","mode=","owner=","testpath=","orgsrc=","contribid=","size=","tag=","unixpath=","type=","procedure=","target=","testcase=","config=","procedure_args=","iteration=","jobid_bump=","cancelscope="]) + except getopt.GetoptError as e: + print(e, file=sys.stderr) + usage(1, "Invalid arguments passed") + + l_ConfigSpecified = False + l_LibPathSpecified = False + for l_Opt, l_Arg in l_Opts: +# print l_Opt, l_Arg + if l_Opt == '-h': + print(__doc__) + sys.exit() + elif l_Opt in ("-a", "--cn_failover"): + if (l_Arg == "0"): + pEnv["CN_FAILOVER"] = False + else: + pEnv["CN_FAILOVER"] = True + elif l_Opt in ("-b", "--io_failover"): + if (l_Arg == "0"): + pEnv["IO_FAILOVER"] = False + else: + pEnv["IO_FAILOVER"] = True + elif l_Opt in ("-c", "--contrib"): + l_Temp = l_Arg.split(",") + pEnv["contrib"] = tuple(map(int, l_Temp)) + elif l_Opt in ("-d", "--name"): + pEnv["name"] = l_Arg + elif l_Opt in ("-e", "--floor"): + pEnv["FLOOR"] = os.path.abspath(l_Arg) + setLibPath(pEnv["FLOOR"]) + elif l_Opt in ("-f", "--flags"): + pEnv["FLAGS"] = l_Arg + elif l_Opt in ("-g", "--group"): + pEnv["GROUP"] = l_Arg + elif l_Opt in ("-h", "--handle"): + if (str(l_Arg).upper() != ALL): + pEnv["handle"] = int(l_Arg) + else: + pEnv["handle"] = bb.NO_HANDLE + elif l_Opt in ("-i", "--hostname"): + if (str(l_Arg).upper() != ALL): + pEnv["hostname"] = l_Arg + else: + pEnv["hostname"] = bb.NO_HOSTNAME + elif l_Opt in ("-j", "--jobid"): + if (str(l_Arg).upper() != ALL): + pEnv["jobid"] = int(l_Arg) + else: + pEnv["jobid"] = bb.NO_JOBID + elif l_Opt in ("-k", "--jobstepid"): + if (str(l_Arg).upper() != ALL): + pEnv["jobstepid"] = int(l_Arg) + else: + pEnv["jobstepid"] = bb.NO_JOBSTEPID + elif l_Opt in ("-l", "--libpath"): + pEnv["LIBPATH"] = os.path.abspath(l_Arg) + l_LibPathSpecified = True + elif l_Opt in ("-m", "--mount"): + pEnv["MOUNT"] = os.path.abspath(l_Arg) + elif l_Opt in ("-n", "--mode"): + pEnv["MODE"] = int(l_Arg) + elif l_Opt in ("-o", "--owner"): + pEnv["OWNER"] = l_Arg + elif l_Opt in ("-p", "--testpath"): + pEnv["TESTPATH"] = os.path.abspath(l_Arg) + elif l_Opt in ("-q", "--orgsrc"): + pEnv["ORGSRC"] = os.path.abspath(l_Arg) + elif l_Opt in ("-r", "--contribid"): + if (str(l_Arg).upper() != ALL): + pEnv["contribid"] = int(l_Arg) + else: + pEnv["contribid"] = bb.NO_CONTRIBID + elif l_Opt in ("-s", "--size"): + pEnv["SIZE"] = l_Arg + elif l_Opt in ("-t", "--tag"): + pEnv["tag"] = int(l_Arg) + elif l_Opt in ("-u", "--unixpath"): + pEnv["UNIXPATH"] = os.path.abspath(l_Arg) + elif l_Opt in ("-v", "--type"): + pEnv["type"] = l_Arg + elif l_Opt in ("-w", "--procedure"): + pEnv["procedure"] = l_Arg + elif l_Opt in ("-x", "--target"): + pEnv["TARGET"] = os.path.abspath(l_Arg) + elif l_Opt in ("-y", "--testcase"): + pEnv["TESTCASE"] = l_Arg + elif l_Opt in ("-z", "--config"): + pEnv["CONFIG"] = os.path.abspath(l_Arg) + l_ConfigSpecified = True + elif l_Opt in ("-1", "--procedure_args"): + pEnv["procedure_args"] = os.path.abspath(l_Arg) + elif l_Opt in ("-2", "--iteration"): + pEnv["iteration"] = int(l_Arg) + elif l_Opt in ("-3", "--jobid_bump"): + pEnv["jobid_bump"] = int(l_Arg) + elif l_Opt in ("-4", "--cancelscope"): + pEnv["cancelscope"] = BBCANCELSCOPE[l_Arg] + + pEnv["COMMAND"] = os.path.abspath(__file__) + pEnv["COMMAND_LINE_ARGS"] = " ".join(pArgs) + setWrapperPath(pEnv) + + setLib(pEnv) +# pprint.pprint(pEnv) + + return + + +def buildLibraryWrapper(pEnv): + + return ctypes.CDLL(pEnv["LIB"]) + + +def main(pArgs): + l_SavedPath = os.environ['PATH'] + l_NewPath = '.:' + l_SavedPath + os.environ['PATH'] = l_NewPath +# print("Environment variable $PATH set to %s" % (l_NewPath)) + + l_Env = {} + try: + processArgs(l_Env, pArgs) + + # Build the ctypes interfaces to the BB APIs... + bb.api = buildLibraryWrapper(l_Env) + + # We only print out the shared library information for the main routine... + if (len(l_Env["procedure_args"]) == 0): + print("Shared library: %s" % (l_Env["LIB"])) + + l_Contribid = bb.cvar("contribid", l_Env) + + l_Size = bb.cvar("size", 256) + l_API_Version = ctypes.create_string_buffer(256) + rc = BB_GetVersion(l_Size, l_API_Version) + + if (rc == 0): + if (l_Env["TESTCASE"] != "None"): + if "CONFIG" in l_Env or "UNIXPATH" in l_Env: + l_Config = ctypes.c_char_p(l_Env.get("CONFIG", NOCONFIG).encode()) + l_UnixPath = ctypes.c_char_p(l_Env.get("UNIXPATH", NOUNIXPATH).encode()) + rc = Coral_InitLibrary(l_Contribid, l_API_Version, l_Config, l_UnixPath) + else: + rc = BB_InitLibrary(l_Contribid, l_API_Version) + + if (rc == 0): + setSysPath(l_Env) + l_TestCase = importlib.import_module(l_Env["TESTCASE"]) + rc = l_TestCase.main(l_Env) + print("bbapi_main.main(): rc = %d" % (rc)) + else: + print("BB_InitLibrary(): rc = %d" % (rc)) + + BB_TerminateLibrary() + else: + print("BB_GetVersion(): rc = %d" % (rc)) + except Exception as e: + rc = -1 + print('Exception raised from bbapi_main::main(), %s' % (e)) + finally: + os.environ['PATH'] = l_SavedPath +# print("Environment variable $PATH set to %s" % (l_SavedPath)) + + return rc + + +# +# Invoke main routine +# + +if __name__ == '__main__': + rc = main(sys.argv[1:]) + + if (rc in (0, -2)): + rc = 0 + else: + rc = -1 + print("bbapi_main: rc = %d" % (rc)) + + sys.exit(rc) diff --git a/bb/wrappers/bberror.py b/bb/wrappers/bberror.py index 810010539..bdcec47e3 100644 --- a/bb/wrappers/bberror.py +++ b/bb/wrappers/bberror.py @@ -1,281 +1,290 @@ -#!/usr/bin/python -#################################################### -# bberror.py -# -# Copyright IBM Corporation 2017,2017. All Rights Reserved -# -# This program is licensed under the terms of the Eclipse Public License -# v1.0 as published by the Eclipse Foundation and available at -# http://www.eclipse.org/legal/epl-v10.html -# -# U.S. Government Users Restricted Rights: Use, duplication or disclosure -# restricted by GSA ADP Schedule Contract with IBM Corp. -################################################### -from ctypes import byref -from datetime import datetime - -import inspect -import json -import os -import pprint - -import bb - - -# -# BB_GetLastErrorDetails BBERRORFORMAT -# -BBERRORFORMAT = {"BBERRORJSON" : 1, #### Output string will be in JSON format - "BBERRORXML" : 2, #### Output string will be in XML format - "BBERRORFLAT" : 3, #### Output string will be in a non-hierarchical name value format - 1 : "BBERRORJSON", #### Output string will be in JSON format - 2 : "BBERRORXML" , #### Output string will be in XML format - 3 : "BBERRORFLAT", #### Output string will be in a non-hierarchical name value format - } - -def BB_GetLastErrorDetails(pBBError=None, pFormat=BBERRORFORMAT["BBERRORJSON"]): - l_Format = bb.cvar("errformat", pFormat) - l_NumBytesAvailable = bb.cvar("numbytesavailable", 4095) - l_BufferSize = bb.cvar("size", 0) - while (l_NumBytesAvailable.value > l_BufferSize.value): - l_BufferSize = bb.cvar("size", l_NumBytesAvailable.value + 1) - l_Buffer = bb.cvar("buffer", l_BufferSize.value) - rc = bb.api.BB_GetLastErrorDetails(l_Format, byref(l_NumBytesAvailable), l_BufferSize, byref(l_Buffer)); - if (rc): - raise BB_GetLastErrorDetailsError(rc) - - l_Temp = [] - for i in xrange(l_BufferSize.value): - if l_Buffer[i] != '\x00': - l_Temp.append(l_Buffer[i]) - else: - break - l_Output = "".join(l_Temp) - - if (pBBError): - lastErrorDetails = l_Output - else: - print "BB_GetLastErrorDetails: %s" % (l_Output) - - return l_Output - - -""" - User exceptions to bbapi functions. -""" - -class BBError(Exception): - DEFAULT_NORMAL_RCs = [0,] - DEFAULT_TOLERATED_ERROR_RCs = [-2,] - - def __init__(self, rc=-1, text="", func=""): - self.rc = rc - if (func == ""): - self.func = inspect.stack()[2][3] - if (len(text) > 0): - self.text = text - self.lastErrorDetails = "" - else: - self.text = "Error raised" - self.lastErrorDetails = self.getLastErrorDetails() - self.normalRCs = BBError.DEFAULT_NORMAL_RCs - self.toleratedErrorRCs = BBError.DEFAULT_TOLERATED_ERROR_RCs - - def __repr__(self): - return "BBError(rc=%d, text=%s, func=%s, normalRCs=%s toleratedErrorRCs=%s, lastErrorDetails=%s)" % (self.rc, self.text, self.func, self.normalRCs, self.toleratedErrorRCs, self.lastErrorDetails) - - def __str__(self): - return os.linesep + self.func + ": " + self.text + ", rc=" + `self.rc` + os.linesep + self.getLastErrorDetailsSummary() - - def getLastErrorDetails(self): - return json.loads(BB_GetLastErrorDetails(self)) - - def getLastErrorDetailsSummary(self): - KEYS_IN_SUMMARY = ("rc","env","in","dft","out","error",) - - l_SummaryKeys = KEYS_IN_SUMMARY - - # For tolerated exceptions, return a true summary. - # Otherwise, return everything in the error details. - if int(self.lastErrorDetails.get("rc", "0")) not in self.getAllToleratedRCs(): - l_SummaryKeys = self.lastErrorDetails.keys() - - # Return only those keys with data... - l_Output = [] - for l_Key in l_SummaryKeys: - if l_Key in self.lastErrorDetails: - l_Output.append("[%s]=%s" % (l_Key, bb.bbpprint().pformat(self.lastErrorDetails[l_Key]))) - - return (os.linesep.join(l_Output)) - - def getAllToleratedRCs(self): - return self.normalRCs + self.toleratedErrorRCs - - def getNormalRCs(self): - return self.normalRCs - - def getToleratedErrorRCs(self): - return self.toleratedErrorRCs - - def handleError(self, moreToleratedRCs=None): - l_Continue = True - l_Prefix = "Tolerated" - l_AllToleratedRCs = self.getAllToleratedRCs() - - if moreToleratedRCs: - if type(moreToleratedRCs) in (tuple, list): - for l_RC in moreToleratedRCs: - l_AllToleratedRCs.append(l_RC) - else: - l_AllToleratedRCs.append(moreToleratedRCs) - - if self.rc not in l_AllToleratedRCs: - l_Prefix = "Non-tolerated" - l_Continue = False - - print datetime.now().strftime("Current date/time: %Y-%m-%d %H:%M:%S") - print "%s exception from %s, rc=%d" % (l_Prefix, self.func, self.rc) - if (not l_Continue): - print "%s" % (`self`) - - return l_Continue - -class BB_AddFilesError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_AddKeysError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_CancelTransferError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_ChangeModeError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_ChangeOwnerError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_CloseServerError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - self.toleratedErrorRCs.append(22) - -class BB_CreateDirectoryError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_CreateLogicalVolumeError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_CreateTransferDefError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_FreeTransferDefError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_GetDeviceUsageError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_GetServerError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_GetServerByNameError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_GetThrottleRateError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_GetTransferHandleError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_GetTransferKeysError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_GetTransferListError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_GetTransferInfoError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_GetUsageError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - # Add directory not found as a tolerated exception - self.toleratedErrorRCs.append(2) - -class BB_OpenServerError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - self.toleratedErrorRCs.append(114) - -class BB_RemoveDirectoryError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_RemoveJobInfoError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_RemoveLogicalVolumeError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_ResizeMountPointError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_RestartTransfersError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_ResumeError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_RetrieveTransfersError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_SetServerError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_SetThrottleRateError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_SetUsageLimitError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_StartTransferError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_StopTransfersError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class BB_SuspendError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) - -class Coral_StageOutStartError(BBError): - def __init__(self, rc): - BBError.__init__(self, rc) +#!/usr/bin/python +#################################################### +# bberror.py +# +# Copyright IBM Corporation 2017,2017. All Rights Reserved +# +# This program is licensed under the terms of the Eclipse Public License +# v1.0 as published by the Eclipse Foundation and available at +# http://www.eclipse.org/legal/epl-v10.html +# +# U.S. Government Users Restricted Rights: Use, duplication or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. +################################################### +from ctypes import byref +from datetime import datetime + +import inspect +import json +import os +import pprint +import traceback + +import bb + + +# +# BB_GetLastErrorDetails BBERRORFORMAT +# +BBERRORFORMAT = {"BBERRORJSON" : 1, #### Output string will be in JSON format + "BBERRORXML" : 2, #### Output string will be in XML format + "BBERRORFLAT" : 3, #### Output string will be in a non-hierarchical name value format + 1 : "BBERRORJSON", #### Output string will be in JSON format + 2 : "BBERRORXML" , #### Output string will be in XML format + 3 : "BBERRORFLAT", #### Output string will be in a non-hierarchical name value format + } + +def BB_GetLastErrorDetails(pBBError=None, pFormat=BBERRORFORMAT["BBERRORJSON"]): + l_Format = bb.cvar("errformat", pFormat) + l_NumBytesAvailable = bb.cvar("numbytesavailable", 4095) + l_BufferSize = bb.cvar("size", 0) + while (l_NumBytesAvailable.value > l_BufferSize.value): + l_BufferSize = bb.cvar("size", l_NumBytesAvailable.value + 1) + l_Buffer = bb.cvar("buffer", l_BufferSize.value) + rc = bb.api.BB_GetLastErrorDetails(l_Format, byref(l_NumBytesAvailable), l_BufferSize, byref(l_Buffer)); + if (rc): + raise BB_GetLastErrorDetailsError(rc) + + l_Temp = [] + for i in range(l_BufferSize.value): + if l_Buffer[i] != b'\0': + l_Temp.append(l_Buffer[i]) + else: + break + l_Output = (b"".join(l_Temp)).decode() + + if (pBBError): + lastErrorDetails = l_Output + else: + print("BB_GetLastErrorDetails: %s" % (l_Output)) + + return l_Output + + +""" + User exceptions to bbapi functions. +""" + +class BBError(Exception): + DEFAULT_NORMAL_RCs = [0,] + DEFAULT_TOLERATED_ERROR_RCs = [-2,] + + def __init__(self, rc=-1, text="", func=""): + self.rc = rc + if (func == ""): + self.func = inspect.stack()[2].function + if (len(text) > 0): + self.text = text + self.lastErrorDetails = "" + else: + self.text = "Error raised" + self.lastErrorDetails = self.getLastErrorDetails() + self.normalRCs = BBError.DEFAULT_NORMAL_RCs + self.toleratedErrorRCs = BBError.DEFAULT_TOLERATED_ERROR_RCs + + def __repr__(self): + return "BBError(rc=%d, text=%s, func=%s, normalRCs=%s toleratedErrorRCs=%s, lastErrorDetails=%s)" % (self.rc, self.text, self.func, self.normalRCs, self.toleratedErrorRCs, self.lastErrorDetails) + + def __str__(self): + return os.linesep + self.func + ": " + self.text + ", rc=" + repr(self.rc) + os.linesep + self.getLastErrorDetailsSummary() + + def getLastErrorDetails(self): + l_Temp = None + + try: + l_Temp = BB_GetLastErrorDetails(self) + l_Temp = json.loads(l_Temp) + except Exception as error: + print(error) + + return l_Temp + + def getLastErrorDetailsSummary(self): + KEYS_IN_SUMMARY = ("rc","env","in","dft","out","error",) + + l_SummaryKeys = KEYS_IN_SUMMARY + + # For tolerated exceptions, return a true summary. + # Otherwise, return everything in the error details. + if int(self.lastErrorDetails.get("rc", "0")) not in self.getAllToleratedRCs(): + l_SummaryKeys = list(self.lastErrorDetails.keys()) + + # Return only those keys with data... + l_Output = [] + for l_Key in l_SummaryKeys: + if l_Key in self.lastErrorDetails: + l_Output.append("[%s]=%s" % (l_Key, bb.bbpprint().pformat(self.lastErrorDetails[l_Key]))) + + return (os.linesep.join(l_Output)) + + def getAllToleratedRCs(self): + return self.normalRCs + self.toleratedErrorRCs + + def getNormalRCs(self): + return self.normalRCs + + def getToleratedErrorRCs(self): + return self.toleratedErrorRCs + + def handleError(self, moreToleratedRCs=None): + l_Continue = True + l_Prefix = "Tolerated" + l_AllToleratedRCs = self.getAllToleratedRCs() + + if moreToleratedRCs: + if type(moreToleratedRCs) in (tuple, list): + for l_RC in moreToleratedRCs: + l_AllToleratedRCs.append(l_RC) + else: + l_AllToleratedRCs.append(moreToleratedRCs) + + if self.rc not in l_AllToleratedRCs: + l_Prefix = "Non-tolerated" + l_Continue = False + + print(datetime.now().strftime("Current date/time: %Y-%m-%d %H:%M:%S")) + print("%s exception from %s, rc=%d" % (l_Prefix, self.func, self.rc)) + if (not l_Continue): + print("%s" % (repr(self))) + + return l_Continue + +class BB_AddFilesError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_AddKeysError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_CancelTransferError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_ChangeModeError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_ChangeOwnerError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_CloseServerError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + self.toleratedErrorRCs.append(22) + +class BB_CreateDirectoryError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_CreateLogicalVolumeError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_CreateTransferDefError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_FreeTransferDefError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_GetDeviceUsageError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_GetServerError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_GetServerByNameError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_GetThrottleRateError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_GetTransferHandleError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_GetTransferKeysError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_GetTransferListError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_GetTransferInfoError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_GetUsageError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + # Add directory not found as a tolerated exception + self.toleratedErrorRCs.append(2) + +class BB_OpenServerError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + self.toleratedErrorRCs.append(114) + +class BB_RemoveDirectoryError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_RemoveJobInfoError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_RemoveLogicalVolumeError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_ResizeMountPointError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_RestartTransfersError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_ResumeError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_RetrieveTransfersError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_SetServerError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_SetThrottleRateError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_SetUsageLimitError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_StartTransferError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_StopTransfersError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class BB_SuspendError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) + +class Coral_StageOutStartError(BBError): + def __init__(self, rc): + BBError.__init__(self, rc) diff --git a/bb/wrappers/bbtests.py b/bb/wrappers/bbtests.py index 66daa7691..b1118a925 100644 --- a/bb/wrappers/bbtests.py +++ b/bb/wrappers/bbtests.py @@ -1,448 +1,448 @@ -#!/usr/bin/python -#################################################### -# bbtests.py -# -# Copyright IBM Corporation 2017,2017. All Rights Reserved -# -# This program is licensed under the terms of the Eclipse Public License -# v1.0 as published by the Eclipse Foundation and available at -# http://www.eclipse.org/legal/epl-v10.html -# -# U.S. Government Users Restricted Rights: Use, duplication or disclosure -# restricted by GSA ADP Schedule Contract with IBM Corp. -################################################### -import time - -""" - Testcases for to bbapi functions. -""" - -import collections -import ctypes -import os -import pprint -import sys -import time - -import bb -from bberror import * -from bbapi import * -from bbapiAdminProcs import * -from bbapiTest import * - - -def BasicTransfers(pEnv): - rc = 0 - l_FuncName = sys._getframe().f_code.co_name - - print " >>>>> Start: %s.%s..." % (__name__, l_FuncName) - - l_Owner = pEnv.get("OWNER", None) - l_Group = pEnv.get("GROUP", None) - l_Mode = int(pEnv.get("MODE", 0755)) - l_LVSize = pEnv.get("SIZE", "1G") - - l_OrgSrc = pEnv["ORGSRC"] - l_Mountpoint = pEnv["MOUNT"] - l_Target = pEnv["TARGET"] - - l_FileAttrs = (("file1", 256*1024), - ("file2", 0), - ("file3", 1024*1024), - ("file4", 4*1024*1024), - ("file5", 16*1024*1024), - ) - l_SourceFiles = ((), - (os.path.join(l_OrgSrc, "file1"),), - (os.path.join(l_Mountpoint, "file1"),), - (os.path.join(l_Mountpoint, "file1b"),), - (os.path.join(l_OrgSrc, "file2"), os.path.join(l_OrgSrc, "file3"), os.path.join(l_OrgSrc, "file4"), os.path.join(l_OrgSrc, "file5"),), - (os.path.join(l_Mountpoint, "file1"), os.path.join(l_Mountpoint, "file2"), os.path.join(l_Mountpoint, "file3"), os.path.join(l_Mountpoint, "file4"), os.path.join(l_Mountpoint, "file5"),), - ) - l_TargetFiles = ((), - (os.path.join(l_Mountpoint, "file1"),), - (os.path.join(l_Mountpoint, "file1b"),), - (os.path.join(l_Target, "file1b"),), - (os.path.join(l_Mountpoint, "file2"), os.path.join(l_Mountpoint, "file3"), os.path.join(l_Mountpoint, "file4"), os.path.join(l_Mountpoint, "file5"),), - (os.path.join(l_Target, "file1"), os.path.join(l_Target, "file2"), os.path.join(l_Target, "file3"), os.path.join(l_Target, "file4"), os.path.join(l_Target, "file5"),), - ) - l_FileFlags = (("None",), - ("None",), - ("None",), - ("None",), - ("BBTestStageIn","BBTestStageIn","BBTestStageIn","BBTestStageIn",), - ("BBTestStageOut","BBTestStageOut","BBTestStageOut","BBTestStageOut","BBTestStageOut",), - ) - l_Tag = 0 - l_Contrib = pEnv["contrib"] - l_NumHandles = 16 - - # NOTE: Any non-zero return code that is returned from any of the BB_* api calls - # will cause an exception to be thrown and caught in the handler below. - # The return code is available as exception data... - try: - # Print out the environment to setup... - bb.printEnv(pEnv) - - # Cleanup any prior running of this variation and setup to run this time... - rc = bb.initEnv(pEnv, l_Mountpoint, l_Target) - - if (rc == 0): - # Create the files to be transferred - print "%sGenerating files with randfile..." % (os.linesep) - for l_File, l_FileSize in (l_FileAttrs): - bb.createRandomFile(pEnv, (os.path.join(l_OrgSrc, l_File)), l_FileSize) - - # Create the necessary directories... - sudo_CreateDirectory(pEnv, l_Target) - sudo_ChangeOwner(pEnv, l_Target, l_Owner, l_Group) - sudo_ChangeMode(pEnv, l_Target, l_Mode) - sudo_CreateDirectory(pEnv, l_Mountpoint) - sudo_ChangeOwner(pEnv, l_Mountpoint, l_Owner, l_Group) - sudo_ChangeMode(pEnv, l_Mountpoint, l_Mode) - - ''' - # This should fail because the logical volume has not yet been created. - # - # This error should be handled, as handleError() will tolerate all - # exceptions where the rc is -2. If rc is not -2, then the exception - # is raised again to the outermost exception handler. - # - # The error is always printed out, with pertinent information taken - # from BB_GetLastErrorDetails(). - # - # NOTE: This is included here to show how to easily handle/tolerate - # bb exceptions... - try: - print "%sThe following BB_GetTransferHandle() is expected to fail with a rc=-2" % (os.linesep) - l_Handle = BB_GetTransferHandle(l_Tag, len(l_Contrib), l_Contrib) - except BBError as error: - if not error.handleError(): - raise - ''' - - # Create the logical volume for the mountpoint... - sudo_CreateLogicalVolume(pEnv, l_Mountpoint, l_LVSize) - - # Run the testcase... - for l_VarNum in xrange(0, len(l_SourceFiles)): - print " >>>>> Start: Variation %d..." % (l_VarNum) - - l_TransferDef = BB_CreateTransferDef() - for l_FileNum in xrange(len(l_SourceFiles[l_VarNum])): - BB_AddFiles(l_TransferDef, l_SourceFiles[l_VarNum][l_FileNum], l_TargetFiles[l_VarNum][l_FileNum], BBFILEFLAGS[l_FileFlags[l_VarNum][l_FileNum]]) - - l_Tag += 1 - l_Handle = BB_GetTransferHandle(l_Tag, len(l_Contrib), l_Contrib) - BB_StartTransfer(l_TransferDef, l_Handle) - - if l_VarNum+1 == len(l_SourceFiles): - # Last variation, so we are doing the stageout processing. - # NOTE: This isn't required, but included for test purposes. - sudo_StageOutStart(pEnv, l_Mountpoint) - - l_Handles = bb.getHandles(BBSTATUS["BBALL"]) - l_AllFullSuccess = bb.waitForCompletion(pEnv, l_Handles) - - BB_FreeTransferDef(l_TransferDef) - - print " >>>>> End: Variation %d..." % (l_VarNum) - - if not l_AllFullSuccess: - raise BBError(rc=-1, text="Not all transfers had a status of BBFULLSUCCESS") - - bb.checkFiles(l_SourceFiles, l_TargetFiles) - - # Cleanup... - - sudo_RemoveLogicalVolume(pEnv, l_Mountpoint) - sudo_RemoveDirectory(pEnv, l_Mountpoint) - if (len(l_Contrib) == 1): - sudo_RemoveJobInfo(pEnv) - elif (pEnv["contribid"] == 0): - time.sleep(5) - sudo_RemoveJobInfo(pEnv) - - except BBError as error: - rc = error.rc - # NOTE: BB_GetLastErrorDetails() output is contained within the error object - # and pertinant information is printed out from that data... - print `error` - - print " >>>>> End: %s.%s..." % (__name__, l_FuncName) - - return rc - - -def BasicTransfers2(pEnv): - rc = 0 - l_FuncName = sys._getframe().f_code.co_name - - print " >>>>> Start: %s.%s..." % (__name__, l_FuncName) - - l_Owner = pEnv.get("OWNER", None) - l_Group = pEnv.get("GROUP", None) - l_Mode = int(pEnv.get("MODE", 0755)) - l_LVSize = pEnv.get("SIZE", "1G") - - l_OrgSrc = pEnv["ORGSRC"] - l_Mountpoint = pEnv["MOUNT"] - l_Target = pEnv["TARGET"] - - l_FileAttrs = (("file1", 256*1024), - ("file2", 0), - ("file3", 1024*1024), - ("file4", 4*1024*1024), - ("file5", 16*1024*1024), - ) - l_SourceFiles = ((os.path.join(l_OrgSrc, "file1"),), - (os.path.join(l_Mountpoint, "file1"),), - ) - l_TargetFiles = ((os.path.join(l_Mountpoint, "file1"),), - (os.path.join(l_Target, "file1"),), - ) - l_FileFlags = (("None",), - ("None",), - ) - - l_Tag = 0 # The first tag to use will be 1... - l_Contrib = pEnv["contrib"] - l_NumHandles = 16 - - # NOTE: Any non-zero return code that is returned from any of the BB_* api calls - # will cause an exception to be thrown and caught in the handler below. - # The return code is available as exception data... - try: - # Print out the environment to setup... - bb.printEnv(pEnv) - - # Cleanup any prior running of this variation and setup to run this time... - rc = bb.initEnv(pEnv, None, None) - - if (rc == 0): - # Create the files to be transferred - print "%sGenerating files with randfile..." % (os.linesep) - for l_File, l_FileSize in (l_FileAttrs): - bb.createRandomFile(pEnv, (os.path.join(l_OrgSrc, l_File)), l_FileSize) - - # Remove/Create the necessary directories... - sudo_RemoveDirectory(pEnv, l_Target) - sudo_CreateDirectory(pEnv, l_Target) - sudo_ChangeOwner(pEnv, l_Target, l_Owner, l_Group) - sudo_ChangeMode(pEnv, l_Target, l_Mode) - - # Run the testcase... - for l_VarNum in xrange(0, len(l_SourceFiles)): - print " >>>>> Start: Variation %d..." % (l_VarNum) - - l_TransferDef = BB_CreateTransferDef() - for l_FileNum in xrange(len(l_SourceFiles[l_VarNum])): - BB_AddFiles(l_TransferDef, l_SourceFiles[l_VarNum][l_FileNum], l_TargetFiles[l_VarNum][l_FileNum], BBFILEFLAGS[l_FileFlags[l_VarNum][l_FileNum]]) - - l_Tag += 1 - l_Handle = BB_GetTransferHandle(l_Tag, len(l_Contrib), l_Contrib) - BB_StartTransfer(l_TransferDef, l_Handle) - - if l_VarNum+1 == len(l_SourceFiles): - # Last variation, so we are doing the stageout processing. - # NOTE: This isn't required, but included for test purposes. - sudo_StageOutStart(pEnv, l_Mountpoint) - - l_Handles = bb.getHandles(BBSTATUS["BBALL"]) - l_AllFullSuccess = bb.waitForCompletion(pEnv, l_Handles) - - BB_FreeTransferDef(l_TransferDef) - - print " >>>>> End: Variation %d..." % (l_VarNum) - - if not l_AllFullSuccess: - raise BBError(rc=-1, text="Not all transfers had a status of BBFULLSUCCESS") - - bb.checkFiles(l_SourceFiles, l_TargetFiles) - - except BBError as error: - rc = error.rc - # NOTE: BB_GetLastErrorDetails() output is contained within the error object - # and pertinant information is printed out from that data... - print `error` - - print " >>>>> End: %s.%s..." % (__name__, l_FuncName) - - return rc - - -def BasicTransfers3(pEnv): - rc = 0 - l_FuncName = sys._getframe().f_code.co_name - - print " >>>>> Start: %s.%s..." % (__name__, l_FuncName) - - l_Owner = pEnv.get("OWNER", None) - l_Group = pEnv.get("GROUP", None) - l_Mode = int(pEnv.get("MODE", 0755)) - l_LVSize = "8G" - - l_OrgSrc = pEnv["ORGSRC"] - l_Mountpoint = pEnv["MOUNT"] - l_Target = pEnv["TARGET"] - - l_FileAttrs = (("file1", 256*1024), - ("file2", 0), - ("file3", 512*1024*1024), - ("file4", 768*1024*1024), - ("file5", 1024*1024*1024), - ) - l_SourceFiles = ((), - (os.path.join(l_OrgSrc, "file1"),), - (os.path.join(l_Mountpoint, "file1"),), - (os.path.join(l_Mountpoint, "file1b"),), - (os.path.join(l_OrgSrc, "file2"), os.path.join(l_OrgSrc, "file3"), os.path.join(l_OrgSrc, "file4"), os.path.join(l_OrgSrc, "file5"),), - (os.path.join(l_Mountpoint, "file1"), os.path.join(l_Mountpoint, "file2"), os.path.join(l_Mountpoint, "file3"), os.path.join(l_Mountpoint, "file4"), os.path.join(l_Mountpoint, "file5"),), - ) - l_TargetFiles = ((), - (os.path.join(l_Mountpoint, "file1"),), - (os.path.join(l_Mountpoint, "file1b"),), - (os.path.join(l_Target, "file1b"),), - (os.path.join(l_Mountpoint, "file2"), os.path.join(l_Mountpoint, "file3"), os.path.join(l_Mountpoint, "file4"), os.path.join(l_Mountpoint, "file5"),), - (os.path.join(l_Target, "file1"), os.path.join(l_Target, "file2"), os.path.join(l_Target, "file3"), os.path.join(l_Target, "file4"), os.path.join(l_Target, "file5"),), - ) - l_FileFlags = (("None",), - ("None",), - ("None",), - ("None",), - ("BBTestStageIn","BBTestStageIn","BBTestStageIn","BBTestStageIn",), - ("BBTestStageOut","BBTestStageOut","BBTestStageOut","BBTestStageOut","BBTestStageOut",), - ) - l_Tag = 0 - l_Contrib = pEnv["contrib"] - l_NumHandles = 16 - - # NOTE: Any non-zero return code that is returned from any of the BB_* api calls - # will cause an exception to be thrown and caught in the handler below. - # The return code is available as exception data... - try: - # Print out the environment to setup... - bb.printEnv(pEnv) - - # Cleanup any prior running of this variation and setup to run this time... - rc = bb.initEnv(pEnv, l_Mountpoint, l_Target) - - if (rc == 0): - # Create the files to be transferred - print "%sGenerating files with randfile..." % (os.linesep) - for l_File, l_FileSize in (l_FileAttrs): - bb.createRandomFile(pEnv, (os.path.join(l_OrgSrc, l_File)), l_FileSize) - - # Create the necessary directories... - sudo_CreateDirectory(pEnv, l_Target) - sudo_ChangeOwner(pEnv, l_Target, l_Owner, l_Group) - sudo_ChangeMode(pEnv, l_Target, l_Mode) - sudo_CreateDirectory(pEnv, l_Mountpoint) - sudo_ChangeOwner(pEnv, l_Mountpoint, l_Owner, l_Group) - sudo_ChangeMode(pEnv, l_Mountpoint, l_Mode) - - # This should fail because the logical volume has not yet been created. - # - # This error should be handled, as handleError() will tolerate all - # exceptions where the rc is -2. If rc is not -2, then the exception - # is raised again to the outermost exception handler. - # - # The error is always printed out, with pertinent information taken - # from BB_GetLastErrorDetails(). - # - # NOTE: This is included here to show how to easily handle/tolerate - # bb exceptions... - """ - try: - print "%sThe following BB_GetTransferHandle() is expected to fail with a rc=-2" % (os.linesep) - l_Handle = BB_GetTransferHandle(l_Tag, len(l_Contrib), l_Contrib) - except BBError as error: - if not error.handleError(): - raise - """ - - for i in xrange(1,100): - - # Create the logical volume for the mountpoint... - sudo_CreateLogicalVolume(pEnv, l_Mountpoint, l_LVSize) - - # Run the testcase... - for l_VarNum in xrange(0, len(l_SourceFiles)): - print " >>>>> Start: Variation %d..." % (l_VarNum) - - l_TransferDef = BB_CreateTransferDef() - for l_FileNum in xrange(len(l_SourceFiles[l_VarNum])): - BB_AddFiles(l_TransferDef, l_SourceFiles[l_VarNum][l_FileNum], l_TargetFiles[l_VarNum][l_FileNum], BBFILEFLAGS[l_FileFlags[l_VarNum][l_FileNum]]) - - l_Tag += 1 - l_Handle = BB_GetTransferHandle(l_Tag, len(l_Contrib), l_Contrib) - BB_StartTransfer(l_TransferDef, l_Handle) - - """ - if l_VarNum+1 == len(l_SourceFiles): - # Last variation, so we are doing the stageout processing. - # NOTE: This isn't required, but included for test purposes. - sudo_StageOutStart(pEnv, l_Mountpoint) - """ - - l_Handles = bb.getHandles(BBSTATUS["BBALL"]) - l_AllFullSuccess = bb.waitForCompletion(pEnv, l_Handles) - - BB_FreeTransferDef(l_TransferDef) - - print " >>>>> End: Variation %d..." % (l_VarNum) - - if not l_AllFullSuccess: - raise BBError(rc=-1, text="Not all transfers had a status of BBFULLSUCCESS") - - bb.checkFiles(l_SourceFiles, l_TargetFiles) - - # Cleanup... - - sudo_RemoveLogicalVolume(pEnv, l_Mountpoint) - if (len(l_Contrib) == 1): - sudo_RemoveJobInfo(pEnv) - elif (pEnv["contribid"] == 0): - time.sleep(5) - sudo_RemoveJobInfo(pEnv) - - sudo_RemoveDirectory(pEnv, l_Mountpoint) - - except BBError as error: - rc = error.rc - # NOTE: BB_GetLastErrorDetails() output is contained within the error object - # and pertinant information is printed out from that data... - print `error` - - print " >>>>> End: %s.%s..." % (__name__, l_FuncName) - - return rc - - -def main(pEnv): - rc = 0; - - l_TestCases = ( - BasicTransfers, -# BasicTransfers2, -# BasicTransfers3, - ) - - for i in xrange(1): - for l_TestCase in l_TestCases: - rc = l_TestCase(pEnv) - print "Testcase -> %s, rc = %d" % (os.path.splitext(os.path.basename(l_TestCase.__name__))[0], rc) - if (rc): - break - if (rc): - break - - return rc - - -# -# Invoke main routine -# - -if __name__ == '__main__': - main(sys.argv[1]) +#!/usr/bin/python +#################################################### +# bbtests.py +# +# Copyright IBM Corporation 2017,2017. All Rights Reserved +# +# This program is licensed under the terms of the Eclipse Public License +# v1.0 as published by the Eclipse Foundation and available at +# http://www.eclipse.org/legal/epl-v10.html +# +# U.S. Government Users Restricted Rights: Use, duplication or disclosure +# restricted by GSA ADP Schedule Contract with IBM Corp. +################################################### +import time + +""" + Testcases for to bbapi functions. +""" + +import collections +import ctypes +import os +import pprint +import sys +import time + +import bb +from bberror import * +from bbapi import * +from bbapiAdminProcs import * +from bbapiTest import * + + +def BasicTransfers(pEnv): + rc = 0 + l_FuncName = sys._getframe().f_code.co_name + + print(" >>>>> Start: %s.%s..." % (__name__, l_FuncName)) + + l_Owner = pEnv.get("OWNER", None) + l_Group = pEnv.get("GROUP", None) + l_Mode = int(pEnv.get("MODE", 0o755)) + l_LVSize = pEnv.get("SIZE", "1G") + + l_OrgSrc = pEnv["ORGSRC"] + l_Mountpoint = pEnv["MOUNT"] + l_Target = pEnv["TARGET"] + + l_FileAttrs = (("file1", 256*1024), + ("file2", 0), + ("file3", 1024*1024), + ("file4", 4*1024*1024), + ("file5", 16*1024*1024), + ) + l_SourceFiles = ((), + (os.path.join(l_OrgSrc, "file1"),), + (os.path.join(l_Mountpoint, "file1"),), + (os.path.join(l_Mountpoint, "file1b"),), + (os.path.join(l_OrgSrc, "file2"), os.path.join(l_OrgSrc, "file3"), os.path.join(l_OrgSrc, "file4"), os.path.join(l_OrgSrc, "file5"),), + (os.path.join(l_Mountpoint, "file1"), os.path.join(l_Mountpoint, "file2"), os.path.join(l_Mountpoint, "file3"), os.path.join(l_Mountpoint, "file4"), os.path.join(l_Mountpoint, "file5"),), + ) + l_TargetFiles = ((), + (os.path.join(l_Mountpoint, "file1"),), + (os.path.join(l_Mountpoint, "file1b"),), + (os.path.join(l_Target, "file1b"),), + (os.path.join(l_Mountpoint, "file2"), os.path.join(l_Mountpoint, "file3"), os.path.join(l_Mountpoint, "file4"), os.path.join(l_Mountpoint, "file5"),), + (os.path.join(l_Target, "file1"), os.path.join(l_Target, "file2"), os.path.join(l_Target, "file3"), os.path.join(l_Target, "file4"), os.path.join(l_Target, "file5"),), + ) + l_FileFlags = (("None",), + ("None",), + ("None",), + ("None",), + ("BBTestStageIn","BBTestStageIn","BBTestStageIn","BBTestStageIn",), + ("BBTestStageOut","BBTestStageOut","BBTestStageOut","BBTestStageOut","BBTestStageOut",), + ) + l_Tag = 0 + l_Contrib = pEnv["contrib"] + l_NumHandles = 16 + + # NOTE: Any non-zero return code that is returned from any of the BB_* api calls + # will cause an exception to be thrown and caught in the handler below. + # The return code is available as exception data... + try: + # Print out the environment to setup... + bb.printEnv(pEnv) + + # Cleanup any prior running of this variation and setup to run this time... + rc = bb.initEnv(pEnv, l_Mountpoint, l_Target) + + if (rc == 0): + # Create the files to be transferred + print("%sGenerating files with randfile..." % (os.linesep)) + for l_File, l_FileSize in (l_FileAttrs): + bb.createRandomFile(pEnv, (os.path.join(l_OrgSrc, l_File)), l_FileSize) + + # Create the necessary directories... + sudo_CreateDirectory(pEnv, l_Target) + sudo_ChangeOwner(pEnv, l_Target, l_Owner, l_Group) + sudo_ChangeMode(pEnv, l_Target, l_Mode) + sudo_CreateDirectory(pEnv, l_Mountpoint) + sudo_ChangeOwner(pEnv, l_Mountpoint, l_Owner, l_Group) + sudo_ChangeMode(pEnv, l_Mountpoint, l_Mode) + + ''' + # This should fail because the logical volume has not yet been created. + # + # This error should be handled, as handleError() will tolerate all + # exceptions where the rc is -2. If rc is not -2, then the exception + # is raised again to the outermost exception handler. + # + # The error is always printed out, with pertinent information taken + # from BB_GetLastErrorDetails(). + # + # NOTE: This is included here to show how to easily handle/tolerate + # bb exceptions... + try: + print "%sThe following BB_GetTransferHandle() is expected to fail with a rc=-2" % (os.linesep) + l_Handle = BB_GetTransferHandle(l_Tag, len(l_Contrib), l_Contrib) + except BBError as error: + if not error.handleError(): + raise + ''' + + # Create the logical volume for the mountpoint... + sudo_CreateLogicalVolume(pEnv, l_Mountpoint, l_LVSize) + + # Run the testcase... + for l_VarNum in range(0, len(l_SourceFiles)): + print(" >>>>> Start: Variation %d..." % (l_VarNum)) + + l_TransferDef = BB_CreateTransferDef() + for l_FileNum in range(len(l_SourceFiles[l_VarNum])): + BB_AddFiles(l_TransferDef, l_SourceFiles[l_VarNum][l_FileNum], l_TargetFiles[l_VarNum][l_FileNum], BBFILEFLAGS[l_FileFlags[l_VarNum][l_FileNum]]) + + l_Tag += 1 + l_Handle = BB_GetTransferHandle(l_Tag, len(l_Contrib), l_Contrib) + BB_StartTransfer(l_TransferDef, l_Handle) + + if l_VarNum+1 == len(l_SourceFiles): + # Last variation, so we are doing the stageout processing. + # NOTE: This isn't required, but included for test purposes. + sudo_StageOutStart(pEnv, l_Mountpoint) + + l_Handles = bb.getHandles(BBSTATUS["BBALL"]) + l_AllFullSuccess = bb.waitForCompletion(pEnv, l_Handles) + + BB_FreeTransferDef(l_TransferDef) + + print(" >>>>> End: Variation %d..." % (l_VarNum)) + + if not l_AllFullSuccess: + raise BBError(rc=-1, text="Not all transfers had a status of BBFULLSUCCESS") + + bb.checkFiles(l_SourceFiles, l_TargetFiles) + + # Cleanup... + + sudo_RemoveLogicalVolume(pEnv, l_Mountpoint) + sudo_RemoveDirectory(pEnv, l_Mountpoint) + if (len(l_Contrib) == 1): + sudo_RemoveJobInfo(pEnv) + elif (pEnv["contribid"] == 0): + time.sleep(5) + sudo_RemoveJobInfo(pEnv) + + except BBError as error: + rc = error.rc + # NOTE: BB_GetLastErrorDetails() output is contained within the error object + # and pertinant information is printed out from that data... + print(repr(error)) + + print(" >>>>> End: %s.%s..." % (__name__, l_FuncName)) + + return rc + + +def BasicTransfers2(pEnv): + rc = 0 + l_FuncName = sys._getframe().f_code.co_name + + print(" >>>>> Start: %s.%s..." % (__name__, l_FuncName)) + + l_Owner = pEnv.get("OWNER", None) + l_Group = pEnv.get("GROUP", None) + l_Mode = int(pEnv.get("MODE", 0o755)) + l_LVSize = pEnv.get("SIZE", "1G") + + l_OrgSrc = pEnv["ORGSRC"] + l_Mountpoint = pEnv["MOUNT"] + l_Target = pEnv["TARGET"] + + l_FileAttrs = (("file1", 256*1024), + ("file2", 0), + ("file3", 1024*1024), + ("file4", 4*1024*1024), + ("file5", 16*1024*1024), + ) + l_SourceFiles = ((os.path.join(l_OrgSrc, "file1"),), + (os.path.join(l_Mountpoint, "file1"),), + ) + l_TargetFiles = ((os.path.join(l_Mountpoint, "file1"),), + (os.path.join(l_Target, "file1"),), + ) + l_FileFlags = (("None",), + ("None",), + ) + + l_Tag = 0 # The first tag to use will be 1... + l_Contrib = pEnv["contrib"] + l_NumHandles = 16 + + # NOTE: Any non-zero return code that is returned from any of the BB_* api calls + # will cause an exception to be thrown and caught in the handler below. + # The return code is available as exception data... + try: + # Print out the environment to setup... + bb.printEnv(pEnv) + + # Cleanup any prior running of this variation and setup to run this time... + rc = bb.initEnv(pEnv, None, None) + + if (rc == 0): + # Create the files to be transferred + print("%sGenerating files with randfile..." % (os.linesep)) + for l_File, l_FileSize in (l_FileAttrs): + bb.createRandomFile(pEnv, (os.path.join(l_OrgSrc, l_File)), l_FileSize) + + # Remove/Create the necessary directories... + sudo_RemoveDirectory(pEnv, l_Target) + sudo_CreateDirectory(pEnv, l_Target) + sudo_ChangeOwner(pEnv, l_Target, l_Owner, l_Group) + sudo_ChangeMode(pEnv, l_Target, l_Mode) + + # Run the testcase... + for l_VarNum in range(0, len(l_SourceFiles)): + print(" >>>>> Start: Variation %d..." % (l_VarNum)) + + l_TransferDef = BB_CreateTransferDef() + for l_FileNum in range(len(l_SourceFiles[l_VarNum])): + BB_AddFiles(l_TransferDef, l_SourceFiles[l_VarNum][l_FileNum], l_TargetFiles[l_VarNum][l_FileNum], BBFILEFLAGS[l_FileFlags[l_VarNum][l_FileNum]]) + + l_Tag += 1 + l_Handle = BB_GetTransferHandle(l_Tag, len(l_Contrib), l_Contrib) + BB_StartTransfer(l_TransferDef, l_Handle) + + if l_VarNum+1 == len(l_SourceFiles): + # Last variation, so we are doing the stageout processing. + # NOTE: This isn't required, but included for test purposes. + sudo_StageOutStart(pEnv, l_Mountpoint) + + l_Handles = bb.getHandles(BBSTATUS["BBALL"]) + l_AllFullSuccess = bb.waitForCompletion(pEnv, l_Handles) + + BB_FreeTransferDef(l_TransferDef) + + print(" >>>>> End: Variation %d..." % (l_VarNum)) + + if not l_AllFullSuccess: + raise BBError(rc=-1, text="Not all transfers had a status of BBFULLSUCCESS") + + bb.checkFiles(l_SourceFiles, l_TargetFiles) + + except BBError as error: + rc = error.rc + # NOTE: BB_GetLastErrorDetails() output is contained within the error object + # and pertinant information is printed out from that data... + print(repr(error)) + + print(" >>>>> End: %s.%s..." % (__name__, l_FuncName)) + + return rc + + +def BasicTransfers3(pEnv): + rc = 0 + l_FuncName = sys._getframe().f_code.co_name + + print(" >>>>> Start: %s.%s..." % (__name__, l_FuncName)) + + l_Owner = pEnv.get("OWNER", None) + l_Group = pEnv.get("GROUP", None) + l_Mode = int(pEnv.get("MODE", 0o755)) + l_LVSize = "8G" + + l_OrgSrc = pEnv["ORGSRC"] + l_Mountpoint = pEnv["MOUNT"] + l_Target = pEnv["TARGET"] + + l_FileAttrs = (("file1", 256*1024), + ("file2", 0), + ("file3", 512*1024*1024), + ("file4", 768*1024*1024), + ("file5", 1024*1024*1024), + ) + l_SourceFiles = ((), + (os.path.join(l_OrgSrc, "file1"),), + (os.path.join(l_Mountpoint, "file1"),), + (os.path.join(l_Mountpoint, "file1b"),), + (os.path.join(l_OrgSrc, "file2"), os.path.join(l_OrgSrc, "file3"), os.path.join(l_OrgSrc, "file4"), os.path.join(l_OrgSrc, "file5"),), + (os.path.join(l_Mountpoint, "file1"), os.path.join(l_Mountpoint, "file2"), os.path.join(l_Mountpoint, "file3"), os.path.join(l_Mountpoint, "file4"), os.path.join(l_Mountpoint, "file5"),), + ) + l_TargetFiles = ((), + (os.path.join(l_Mountpoint, "file1"),), + (os.path.join(l_Mountpoint, "file1b"),), + (os.path.join(l_Target, "file1b"),), + (os.path.join(l_Mountpoint, "file2"), os.path.join(l_Mountpoint, "file3"), os.path.join(l_Mountpoint, "file4"), os.path.join(l_Mountpoint, "file5"),), + (os.path.join(l_Target, "file1"), os.path.join(l_Target, "file2"), os.path.join(l_Target, "file3"), os.path.join(l_Target, "file4"), os.path.join(l_Target, "file5"),), + ) + l_FileFlags = (("None",), + ("None",), + ("None",), + ("None",), + ("BBTestStageIn","BBTestStageIn","BBTestStageIn","BBTestStageIn",), + ("BBTestStageOut","BBTestStageOut","BBTestStageOut","BBTestStageOut","BBTestStageOut",), + ) + l_Tag = 0 + l_Contrib = pEnv["contrib"] + l_NumHandles = 16 + + # NOTE: Any non-zero return code that is returned from any of the BB_* api calls + # will cause an exception to be thrown and caught in the handler below. + # The return code is available as exception data... + try: + # Print out the environment to setup... + bb.printEnv(pEnv) + + # Cleanup any prior running of this variation and setup to run this time... + rc = bb.initEnv(pEnv, l_Mountpoint, l_Target) + + if (rc == 0): + # Create the files to be transferred + print("%sGenerating files with randfile..." % (os.linesep)) + for l_File, l_FileSize in (l_FileAttrs): + bb.createRandomFile(pEnv, (os.path.join(l_OrgSrc, l_File)), l_FileSize) + + # Create the necessary directories... + sudo_CreateDirectory(pEnv, l_Target) + sudo_ChangeOwner(pEnv, l_Target, l_Owner, l_Group) + sudo_ChangeMode(pEnv, l_Target, l_Mode) + sudo_CreateDirectory(pEnv, l_Mountpoint) + sudo_ChangeOwner(pEnv, l_Mountpoint, l_Owner, l_Group) + sudo_ChangeMode(pEnv, l_Mountpoint, l_Mode) + + # This should fail because the logical volume has not yet been created. + # + # This error should be handled, as handleError() will tolerate all + # exceptions where the rc is -2. If rc is not -2, then the exception + # is raised again to the outermost exception handler. + # + # The error is always printed out, with pertinent information taken + # from BB_GetLastErrorDetails(). + # + # NOTE: This is included here to show how to easily handle/tolerate + # bb exceptions... + """ + try: + print "%sThe following BB_GetTransferHandle() is expected to fail with a rc=-2" % (os.linesep) + l_Handle = BB_GetTransferHandle(l_Tag, len(l_Contrib), l_Contrib) + except BBError as error: + if not error.handleError(): + raise + """ + + for i in range(1,100): + + # Create the logical volume for the mountpoint... + sudo_CreateLogicalVolume(pEnv, l_Mountpoint, l_LVSize) + + # Run the testcase... + for l_VarNum in range(0, len(l_SourceFiles)): + print(" >>>>> Start: Variation %d..." % (l_VarNum)) + + l_TransferDef = BB_CreateTransferDef() + for l_FileNum in range(len(l_SourceFiles[l_VarNum])): + BB_AddFiles(l_TransferDef, l_SourceFiles[l_VarNum][l_FileNum], l_TargetFiles[l_VarNum][l_FileNum], BBFILEFLAGS[l_FileFlags[l_VarNum][l_FileNum]]) + + l_Tag += 1 + l_Handle = BB_GetTransferHandle(l_Tag, len(l_Contrib), l_Contrib) + BB_StartTransfer(l_TransferDef, l_Handle) + + """ + if l_VarNum+1 == len(l_SourceFiles): + # Last variation, so we are doing the stageout processing. + # NOTE: This isn't required, but included for test purposes. + sudo_StageOutStart(pEnv, l_Mountpoint) + """ + + l_Handles = bb.getHandles(BBSTATUS["BBALL"]) + l_AllFullSuccess = bb.waitForCompletion(pEnv, l_Handles) + + BB_FreeTransferDef(l_TransferDef) + + print(" >>>>> End: Variation %d..." % (l_VarNum)) + + if not l_AllFullSuccess: + raise BBError(rc=-1, text="Not all transfers had a status of BBFULLSUCCESS") + + bb.checkFiles(l_SourceFiles, l_TargetFiles) + + # Cleanup... + + sudo_RemoveLogicalVolume(pEnv, l_Mountpoint) + if (len(l_Contrib) == 1): + sudo_RemoveJobInfo(pEnv) + elif (pEnv["contribid"] == 0): + time.sleep(5) + sudo_RemoveJobInfo(pEnv) + + sudo_RemoveDirectory(pEnv, l_Mountpoint) + + except BBError as error: + rc = error.rc + # NOTE: BB_GetLastErrorDetails() output is contained within the error object + # and pertinant information is printed out from that data... + print(repr(error)) + + print(" >>>>> End: %s.%s..." % (__name__, l_FuncName)) + + return rc + + +def main(pEnv): + rc = 0; + + l_TestCases = ( + BasicTransfers, +# BasicTransfers2, +# BasicTransfers3, + ) + + for i in range(1): + for l_TestCase in l_TestCases: + rc = l_TestCase(pEnv) + print("Testcase -> %s, rc = %d" % (os.path.splitext(os.path.basename(l_TestCase.__name__))[0], rc)) + if (rc): + break + if (rc): + break + + return rc + + +# +# Invoke main routine +# + +if __name__ == '__main__': + main(sys.argv[1])