From e5e99c06c44f6ccae563bfdda416a3de69c801ac Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Wed, 9 Mar 2016 16:14:37 +0100 Subject: [PATCH 01/22] Initial commit of instruemntview running in loop. Plotter class has been removed, do not push without putting it back in and rearranging datalistener --- run_instrumentvisualizer.py | 28 ++++++++++++++++++ visualizer/__init__.py | 2 +- visualizer/visualizer.py | 58 +++++++++++++++++++++++-------------- 3 files changed, 66 insertions(+), 22 deletions(-) create mode 100644 run_instrumentvisualizer.py diff --git a/run_instrumentvisualizer.py b/run_instrumentvisualizer.py new file mode 100644 index 0000000..0fcbc5e --- /dev/null +++ b/run_instrumentvisualizer.py @@ -0,0 +1,28 @@ +from threading import Thread +import PyQt4 +import sys +from logger import setup_global_logger +import ports +import command_line_parser +from visualizer import DataListener +from visualizer import InstrumentView + +app = PyQt4.QtGui.QApplication(sys.argv) + +setup_global_logger(level=command_line_parser.get_log_level()) + +data_listener = DataListener(host=command_line_parser.get_host(), port=ports.result_stream) +data_listener_thread = Thread(target=data_listener.run) +data_listener_thread.daemon = True +instrumentview = InstrumentView(data_listener) +print "til datalistener" +data_listener.clear.connect(instrumentview.clear) +data_listener.new_data.connect(instrumentview.updateInstrumentView) +data_listener_thread.start() +print "post thread start" + +## Start Qt event loop unless running in interactive mode or using pyside. +if __name__ == '__main__': + import sys + if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'): + app.instance().exec_() diff --git a/visualizer/__init__.py b/visualizer/__init__.py index 0aaffd0..a2aab47 100644 --- a/visualizer/__init__.py +++ b/visualizer/__init__.py @@ -1,2 +1,2 @@ -from visualizer import Plotter +from visualizer import InstrumentView from visualizer import DataListener diff --git a/visualizer/visualizer.py b/visualizer/visualizer.py index a89c2b0..6e2a95b 100644 --- a/visualizer/visualizer.py +++ b/visualizer/visualizer.py @@ -1,41 +1,57 @@ from collections import deque import zmq - -import pyqtgraph as pg -from pyqtgraph.Qt import QtCore +import sys import numpy +import PyQt4 +import mantid.simpleapi as simpleapi +import mantidqtpython as mpy from logger import log +ConfigService = simpleapi.ConfigService +#InstrumentWidget = mpy.MantidQt.MantidWidgets.InstrumentWidget +#app = PyQt4.QtGui.QApplication(sys.argv) -class Plotter(object): +class InstrumentView(object): def __init__(self, dataListener): self.dataListener = dataListener - self.win = pg.GraphicsWindow() - self.win.resize(800,350) - self.win.setWindowTitle('pyqtgraph example: Histogram') - self.plt1 = self.win.addPlot() - self.curves = {} + print "reaches iv clas" + ws = simpleapi.CreateSimulationWorkspace(Instrument='data/POWDIFF_Definition.xml', BinParams='1,0.5,2') + self._number_of_spectra = ws.getNumberHistograms() + ws = simpleapi.WorkspaceFactory.Instance().create("EventWorkspace", self._number_of_spectra, 1, 1) + simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) + ws = simpleapi.AnalysisDataService['POWDIFF_test'] + simpleapi.LoadInstrument(Workspace=ws, Filename='data/POWDIFF_Definition.xml', RewriteSpectraMap=True) + ws.getAxis(0).setUnit('tof') + print "after getAxis" + + InstrumentWidget = mpy.MantidQt.MantidWidgets.InstrumentWidget + self.iw = InstrumentWidget("POWDIFF_test") + self.iw.show() + #app.exec_() def clear(self): self.curves = {} - self.plt1.clear() - def update(self): + def updateInstrumentView(self): + print "reaches update" while self.dataListener.data: - index,x,y = self.dataListener.data.popleft() - if not index in self.curves: - self.curves[index] = self.plt1.plot(stepMode=True, pen=(index), name=str(index)) - self.curves[index].setData(x, y) - self.plt1.enableAutoRange('xy', False) - + index, x, y = self.dataListener.data.popleft() + + # ws = WorkspaceFactory.Instance().create("EventWorkspace", self._number_of_spectra, 1, 1); + # AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) + # ws = AnalysisDataService['POWDIFF_test'] + # simpleapi.LoadInstrument(Workspace=ws, Filename='data/POWDIFF_Definition.xml', RewriteSpectraMap=True) + # ws.getAxis(0).setUnit('tof') + # iw = InstrumentWidget("ws") + # iw.show() -class DataListener(QtCore.QObject): - clear = QtCore.pyqtSignal() - new_data = QtCore.pyqtSignal() +class DataListener(PyQt4.QtCore.QObject): + clear = PyQt4.QtCore.pyqtSignal() + new_data = PyQt4.QtCore.pyqtSignal() def __init__(self, host, port): - QtCore.QObject.__init__(self) + PyQt4.QtCore.QObject.__init__(self) self.data = deque() self._host = host self._port = port From af9991914d31ed736942f5370ce22e58292e372e Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Thu, 10 Mar 2016 11:09:57 +0100 Subject: [PATCH 02/22] added publisher for instrumentview --- backend/__init__.py | 1 + backend/backend_mantid_reducer.py | 2 + backend/instrumentview_publisher.py | 74 +++++++++++++++++++++++++++++ ports.py | 1 + run_backend_reducer.py | 14 ++++-- 5 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 backend/instrumentview_publisher.py diff --git a/backend/__init__.py b/backend/__init__.py index 9675c3a..2a7a7bb 100644 --- a/backend/__init__.py +++ b/backend/__init__.py @@ -1,5 +1,6 @@ from backend_event_listener import BackendEventListener from backend_mantid_reducer import BackendMantidReducer from result_publisher import ResultPublisher +from instrumentview_publisher import InstrumentViewPublisher from zmq_queue import ZMQQueueServer from zmq_queue import ZMQQueueClient diff --git a/backend/backend_mantid_reducer.py b/backend/backend_mantid_reducer.py index 92c25da..f7ffdac 100644 --- a/backend/backend_mantid_reducer.py +++ b/backend/backend_mantid_reducer.py @@ -30,6 +30,8 @@ def __init__(self, data_queue_in): self._rebin_transition = MantidRebinTransition(self._filter_transition) self._gather_histogram_transition = GatherHistogramTransition(self._rebin_transition) self._gather_histogram_transition.accumulate_data = True + self._rebin_for_instrumentview_transition = MantidRebinTransition(self._filter_transition) + self._rebin_for_instrumentview_transition.set_bin_parameters('0,1,1') #set to single bin def _process_command(self, command): setattr(self, command[0], command[1]) diff --git a/backend/instrumentview_publisher.py b/backend/instrumentview_publisher.py new file mode 100644 index 0000000..364f7b5 --- /dev/null +++ b/backend/instrumentview_publisher.py @@ -0,0 +1,74 @@ +import time +import numpy +import zmq + +from logger import log +import ports +from controllable import Controllable + + +class InstrumentViewPublisher(Controllable): + def __init__(self, eventListener): + super(InstrumentViewPublisher, self).__init__(type(self).__name__) + self.eventListener = eventListener + self._update_rate = 1.0 + self.socket = None + self._last_count = 0 + + def run(self): + log.info("Starting InstrumentViewPublisher") + self.connect() + + self._publish_clear() + while True: + count = len(self.eventListener._rebin_for_instrumentview_transition.get_checkpoint()) + if count != self._last_count: + self._publish_clear() + self._last_count = count + for i in range(count): + if self.eventListener._rebin_for_instrumentview_transition.get_checkpoint()[i]: + self._publish(i) + time.sleep(self.update_rate) + + def connect(self): + context = zmq.Context() + self.socket = context.socket(zmq.PUB) + uri = 'tcp://*:{0:d}'.format(ports.instrumentview_result_stream) + self.socket.bind(uri) + log.info('Bound to ' + uri) + + def _create_header(self, command, index): + return { 'command':command, 'index':index } + + def _publish_clear(self): + header = self._create_header('clear', None) + self.socket.send_json(header) + + def _publish(self, index): + #boundaries, values = self.eventListener._rebin_for_instrumentview_transition.get_checkpoint()[index].data + #packet = numpy.concatenate((boundaries, values)) + #should want readX, readY, readE and spectrum num via getNumberofHistograms() + currentws = self.eventListener._rebin_for_instrumentview_transition.get_checkpoint()[index].data + for i in range(currentws.getNumberHistograms()): + seriesX = currentws.readX(i) + print seriesX + + seriesY = currentws.readY(i) + print seriesY + seriesE = currentws.readE(i) + print seriesE + packet = numpy.concatenate((seriesX, seriesY, seriesE)) + header = self._create_header('data', index) + self.socket.send_json(header, flags=zmq.SNDMORE) + self.socket.send(packet) + + def get_parameter_dict(self): + return {'update_rate':'float'} + + @property + def update_rate(self): + return self._update_rate + + @update_rate.setter + def update_rate(self, update_rate): + self._update_rate = update_rate diff --git a/ports.py b/ports.py index 5e99fbb..a603355 100644 --- a/ports.py +++ b/ports.py @@ -3,3 +3,4 @@ result_stream = 10003 result_publisher_control = 10004 rebin_control = 10005 +instrumentview_result_stream = 10006 diff --git a/run_backend_reducer.py b/run_backend_reducer.py index 46dc1c9..30b6578 100644 --- a/run_backend_reducer.py +++ b/run_backend_reducer.py @@ -1,6 +1,7 @@ from logger import setup_global_logger from backend import BackendMantidReducer from backend import ResultPublisher +from backend import InstrumentViewPublisher from backend import ZMQQueueServer from backend import ZMQQueueClient from parameter_control_server import ParameterControlServer @@ -36,11 +37,14 @@ reducer_controller_thread = threading.Thread(target=reducer_controller.run) reducer_controller_thread.start() - resultPublisher = ResultPublisher(reducer) - resultPublisher_thread = threading.Thread(target=resultPublisher.run) - resultPublisher_thread.start() - - parameterController = ParameterControlServer(controllees=[resultPublisher], port=ports.result_publisher_control) + #resultPublisher = ResultPublisher(reducer) + #resultPublisher_thread = threading.Thread(target=resultPublisher.run) + #resultPublisher_thread.start() + instrumentViewPublisher = InstrumentViewPublisher(reducer) + instrumentViewPublisher_thread = threading.Thread(target=instrumentViewPublisher.run) + instrumentViewPublisher_thread.start() + parameterController = ParameterControlServer(controllees=[instrumentViewPublisher], port=ports.result_publisher_control) + #parameterController = ParameterControlServer(controllees=[resultPublisher], port=ports.result_publisher_control) parameterController_thread = threading.Thread(target=parameterController.run) parameterController_thread.start() From 3e55f7fa294f33e648d642b71a490acc760a6818 Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Thu, 10 Mar 2016 15:24:09 +0100 Subject: [PATCH 03/22] instrumentviewer displays --- backend/instrumentview_publisher.py | 4 ---- run_instrumentvisualizer.py | 4 +--- visualizer/visualizer.py | 34 ++++++++++++++--------------- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/backend/instrumentview_publisher.py b/backend/instrumentview_publisher.py index 364f7b5..3e895a9 100644 --- a/backend/instrumentview_publisher.py +++ b/backend/instrumentview_publisher.py @@ -51,12 +51,8 @@ def _publish(self, index): currentws = self.eventListener._rebin_for_instrumentview_transition.get_checkpoint()[index].data for i in range(currentws.getNumberHistograms()): seriesX = currentws.readX(i) - print seriesX - seriesY = currentws.readY(i) - print seriesY seriesE = currentws.readE(i) - print seriesE packet = numpy.concatenate((seriesX, seriesY, seriesE)) header = self._create_header('data', index) self.socket.send_json(header, flags=zmq.SNDMORE) diff --git a/run_instrumentvisualizer.py b/run_instrumentvisualizer.py index 0fcbc5e..4731802 100644 --- a/run_instrumentvisualizer.py +++ b/run_instrumentvisualizer.py @@ -11,15 +11,13 @@ setup_global_logger(level=command_line_parser.get_log_level()) -data_listener = DataListener(host=command_line_parser.get_host(), port=ports.result_stream) +data_listener = DataListener(host=command_line_parser.get_host(), port=ports.instrumentview_result_stream) data_listener_thread = Thread(target=data_listener.run) data_listener_thread.daemon = True instrumentview = InstrumentView(data_listener) -print "til datalistener" data_listener.clear.connect(instrumentview.clear) data_listener.new_data.connect(instrumentview.updateInstrumentView) data_listener_thread.start() -print "post thread start" ## Start Qt event loop unless running in interactive mode or using pyside. if __name__ == '__main__': diff --git a/visualizer/visualizer.py b/visualizer/visualizer.py index 6e2a95b..8c115ba 100644 --- a/visualizer/visualizer.py +++ b/visualizer/visualizer.py @@ -15,7 +15,6 @@ class InstrumentView(object): def __init__(self, dataListener): self.dataListener = dataListener - print "reaches iv clas" ws = simpleapi.CreateSimulationWorkspace(Instrument='data/POWDIFF_Definition.xml', BinParams='1,0.5,2') self._number_of_spectra = ws.getNumberHistograms() ws = simpleapi.WorkspaceFactory.Instance().create("EventWorkspace", self._number_of_spectra, 1, 1) @@ -23,28 +22,27 @@ def __init__(self, dataListener): ws = simpleapi.AnalysisDataService['POWDIFF_test'] simpleapi.LoadInstrument(Workspace=ws, Filename='data/POWDIFF_Definition.xml', RewriteSpectraMap=True) ws.getAxis(0).setUnit('tof') - print "after getAxis" InstrumentWidget = mpy.MantidQt.MantidWidgets.InstrumentWidget self.iw = InstrumentWidget("POWDIFF_test") self.iw.show() - #app.exec_() def clear(self): self.curves = {} + #TODO def updateInstrumentView(self): - print "reaches update" + ws = simpleapi.AnalysisDataService['POWDIFF_test'] + ws = simpleapi.CreateWorkspace(dataX, dataY) + simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) + simpleapi.LoadInstrument(Workspace=ws, Filename='data/POWDIFF_Definition.xml', RewriteSpectraMap=True) while self.dataListener.data: - index, x, y = self.dataListener.data.popleft() - - # ws = WorkspaceFactory.Instance().create("EventWorkspace", self._number_of_spectra, 1, 1); - # AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) - # ws = AnalysisDataService['POWDIFF_test'] - # simpleapi.LoadInstrument(Workspace=ws, Filename='data/POWDIFF_Definition.xml', RewriteSpectraMap=True) - # ws.getAxis(0).setUnit('tof') - # iw = InstrumentWidget("ws") - # iw.show() + index, x, y, e = self.dataListener.data.popleft() + #update 'POWDIFF_test' with these + #ws.setX(index, x) + #ws.setY(index, y) + #ws.setE(index, e) + class DataListener(PyQt4.QtCore.QObject): clear = PyQt4.QtCore.pyqtSignal() @@ -62,8 +60,8 @@ def run(self): while True: command, index = self._receive_header() if command == 'data': - x,y = self.get_histogram() - self.data.append((index,x,y)) + x,y,e = self.get_histogram() + self.data.append((index,x,y,e)) self.new_data.emit() elif command == 'clear': self.clear.emit() @@ -74,12 +72,12 @@ def connect(self): uri = 'tcp://{0}:{1:d}'.format(self._host, self._port) self.socket.connect(uri) self.socket.setsockopt(zmq.SUBSCRIBE, '') - log.info('Substribed to result publisher at ' + uri) + log.info('Subscribed to result publisher at ' + uri) def get_histogram(self): data = numpy.frombuffer(self.socket.recv(), numpy.float64) - x,y = numpy.array_split(data, 2) - return x,y + x, y, e = numpy.array_split(data, 3) + return x, y, e def _receive_header(self): header = self.socket.recv_json() From a215731918f5e4430a6aafdd96e52a866f44546c Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Mon, 21 Mar 2016 15:02:52 +0100 Subject: [PATCH 04/22] updated to send 1000 spectra --- backend/backend_mantid_reducer.py | 2 +- backend/instrumentview_publisher.py | 22 ++++++++++++---------- visualizer/visualizer.py | 17 ++++++++--------- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/backend/backend_mantid_reducer.py b/backend/backend_mantid_reducer.py index f7ffdac..91ab788 100644 --- a/backend/backend_mantid_reducer.py +++ b/backend/backend_mantid_reducer.py @@ -30,7 +30,7 @@ def __init__(self, data_queue_in): self._rebin_transition = MantidRebinTransition(self._filter_transition) self._gather_histogram_transition = GatherHistogramTransition(self._rebin_transition) self._gather_histogram_transition.accumulate_data = True - self._rebin_for_instrumentview_transition = MantidRebinTransition(self._filter_transition) + self._rebin_for_instrumentview_transition = MantidRebinTransition(self._create_workspace_from_events_transition) self._rebin_for_instrumentview_transition.set_bin_parameters('0,1,1') #set to single bin def _process_command(self, command): diff --git a/backend/instrumentview_publisher.py b/backend/instrumentview_publisher.py index 3e895a9..d80f78e 100644 --- a/backend/instrumentview_publisher.py +++ b/backend/instrumentview_publisher.py @@ -21,13 +21,8 @@ def run(self): self._publish_clear() while True: - count = len(self.eventListener._rebin_for_instrumentview_transition.get_checkpoint()) - if count != self._last_count: - self._publish_clear() - self._last_count = count - for i in range(count): - if self.eventListener._rebin_for_instrumentview_transition.get_checkpoint()[i]: - self._publish(i) + if self.eventListener._rebin_for_instrumentview_transition.get_checkpoint(): + self._publish() time.sleep(self.update_rate) def connect(self): @@ -44,17 +39,24 @@ def _publish_clear(self): header = self._create_header('clear', None) self.socket.send_json(header) - def _publish(self, index): + def _publish(self): #boundaries, values = self.eventListener._rebin_for_instrumentview_transition.get_checkpoint()[index].data #packet = numpy.concatenate((boundaries, values)) #should want readX, readY, readE and spectrum num via getNumberofHistograms() - currentws = self.eventListener._rebin_for_instrumentview_transition.get_checkpoint()[index].data + currentws = self.eventListener._rebin_for_instrumentview_transition.get_checkpoint().data + #print currentws.getNumberHistograms() + #current workspace is what + print type(currentws) for i in range(currentws.getNumberHistograms()): + #print i seriesX = currentws.readX(i) + #print seriesX seriesY = currentws.readY(i) + #print seriesY seriesE = currentws.readE(i) + #print seriesE packet = numpy.concatenate((seriesX, seriesY, seriesE)) - header = self._create_header('data', index) + header = self._create_header('data', i) self.socket.send_json(header, flags=zmq.SNDMORE) self.socket.send(packet) diff --git a/visualizer/visualizer.py b/visualizer/visualizer.py index 8c115ba..7b103c6 100644 --- a/visualizer/visualizer.py +++ b/visualizer/visualizer.py @@ -8,8 +8,6 @@ from logger import log ConfigService = simpleapi.ConfigService -#InstrumentWidget = mpy.MantidQt.MantidWidgets.InstrumentWidget -#app = PyQt4.QtGui.QApplication(sys.argv) class InstrumentView(object): @@ -17,7 +15,8 @@ def __init__(self, dataListener): self.dataListener = dataListener ws = simpleapi.CreateSimulationWorkspace(Instrument='data/POWDIFF_Definition.xml', BinParams='1,0.5,2') self._number_of_spectra = ws.getNumberHistograms() - ws = simpleapi.WorkspaceFactory.Instance().create("EventWorkspace", self._number_of_spectra, 1, 1) + print "this is num spectra " + str(self._number_of_spectra) + ws = simpleapi.WorkspaceFactory.Instance().create("Workspace2D", self._number_of_spectra, 2, 1) simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) ws = simpleapi.AnalysisDataService['POWDIFF_test'] simpleapi.LoadInstrument(Workspace=ws, Filename='data/POWDIFF_Definition.xml', RewriteSpectraMap=True) @@ -33,15 +32,15 @@ def clear(self): def updateInstrumentView(self): ws = simpleapi.AnalysisDataService['POWDIFF_test'] - ws = simpleapi.CreateWorkspace(dataX, dataY) - simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) - simpleapi.LoadInstrument(Workspace=ws, Filename='data/POWDIFF_Definition.xml', RewriteSpectraMap=True) while self.dataListener.data: index, x, y, e = self.dataListener.data.popleft() #update 'POWDIFF_test' with these - #ws.setX(index, x) - #ws.setY(index, y) - #ws.setE(index, e) + ws.setX(index, x) + #print "this index: " + str(index) + " and this x: " + str(x) + ws.setY(index, y) + #print "this y: " + str(y) + ws.setE(index, e) + #print "this e: " + str(e) class DataListener(PyQt4.QtCore.QObject): From 4153b6f04c164e463c4bc49a13f40fbc0fe7ccbc Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Tue, 29 Mar 2016 13:44:28 +0200 Subject: [PATCH 05/22] data in instrumentview updates --- visualizer/visualizer.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/visualizer/visualizer.py b/visualizer/visualizer.py index 7b103c6..b006aae 100644 --- a/visualizer/visualizer.py +++ b/visualizer/visualizer.py @@ -5,7 +5,7 @@ import PyQt4 import mantid.simpleapi as simpleapi import mantidqtpython as mpy - +import random from logger import log ConfigService = simpleapi.ConfigService @@ -15,15 +15,23 @@ def __init__(self, dataListener): self.dataListener = dataListener ws = simpleapi.CreateSimulationWorkspace(Instrument='data/POWDIFF_Definition.xml', BinParams='1,0.5,2') self._number_of_spectra = ws.getNumberHistograms() - print "this is num spectra " + str(self._number_of_spectra) ws = simpleapi.WorkspaceFactory.Instance().create("Workspace2D", self._number_of_spectra, 2, 1) simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) - ws = simpleapi.AnalysisDataService['POWDIFF_test'] - simpleapi.LoadInstrument(Workspace=ws, Filename='data/POWDIFF_Definition.xml', RewriteSpectraMap=True) - ws.getAxis(0).setUnit('tof') + simpleapi.LoadInstrument(Workspace=ws, Filename='data/POWDIFF_Definition.xml', RewriteSpectraMap=True) + ws = simpleapi.Rebin(ws, "0,1,2") + simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) + #simpleapi.AnalysisDataService['POWDIFF_test'] + #simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) + + #ws = simpleapi.AnalysisDataService['POWDIFF_test'] + #simpleapi.LoadInstrument(Workspace=ws, Filename='data/POWDIFF_Definition.xml', RewriteSpectraMap=True) + #ws.getAxis(0).setUnit('tof') + #ws.setX(1,numpy.array([0,1])) + #ws.setY(1,numpy.array([1])) + #ws.setE(1,numpy.array([1])) InstrumentWidget = mpy.MantidQt.MantidWidgets.InstrumentWidget - self.iw = InstrumentWidget("POWDIFF_test") + self.iw = InstrumentWidget('POWDIFF_test') self.iw.show() def clear(self): @@ -31,18 +39,14 @@ def clear(self): #TODO def updateInstrumentView(self): + #pass ws = simpleapi.AnalysisDataService['POWDIFF_test'] + while self.dataListener.data: index, x, y, e = self.dataListener.data.popleft() - #update 'POWDIFF_test' with these - ws.setX(index, x) - #print "this index: " + str(index) + " and this x: " + str(x) - ws.setY(index, y) - #print "this y: " + str(y) - ws.setE(index, e) - #print "this e: " + str(e) - - + #ws.dataY(index)[1] = y + ws.dataY(index)[1] = random.randint(1,5) + class DataListener(PyQt4.QtCore.QObject): clear = PyQt4.QtCore.pyqtSignal() new_data = PyQt4.QtCore.pyqtSignal() From c0fbe87dd95cb638b0e022084057c490fd472ce8 Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Tue, 29 Mar 2016 15:22:30 +0200 Subject: [PATCH 06/22] checkpoint --- backend/instrumentview_publisher.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/backend/instrumentview_publisher.py b/backend/instrumentview_publisher.py index d80f78e..ed3fcaa 100644 --- a/backend/instrumentview_publisher.py +++ b/backend/instrumentview_publisher.py @@ -40,22 +40,28 @@ def _publish_clear(self): self.socket.send_json(header) def _publish(self): - #boundaries, values = self.eventListener._rebin_for_instrumentview_transition.get_checkpoint()[index].data - #packet = numpy.concatenate((boundaries, values)) #should want readX, readY, readE and spectrum num via getNumberofHistograms() currentws = self.eventListener._rebin_for_instrumentview_transition.get_checkpoint().data #print currentws.getNumberHistograms() #current workspace is what - print type(currentws) + #clone workspace here pass copy for i in range(currentws.getNumberHistograms()): #print i - seriesX = currentws.readX(i) - #print seriesX - seriesY = currentws.readY(i) - #print seriesY - seriesE = currentws.readE(i) - #print seriesE + #seriesX = currentws.readX(i) + seriesX = numpy.array([0,1]) + #seriesX = [0,1] + #seriesY = currentws.readY(i) + seriesY = numpy.array([1]) + #seriesY = [1] + #seriesE = currentws.readE(i) + seriesE = numpy.array([1]) + #seriesE = [1] packet = numpy.concatenate((seriesX, seriesY, seriesE)) + print "here packet" + print packet + print "here split" + x, y, e = numpy.array_split(packet, 3) + print x, y, e header = self._create_header('data', i) self.socket.send_json(header, flags=zmq.SNDMORE) self.socket.send(packet) From 49f70ebb769aa91c6ea0e29f89355ee5fdf55210 Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Wed, 30 Mar 2016 15:08:57 +0200 Subject: [PATCH 07/22] Working instrumentvisualiser to data stream --- backend/instrumentview_publisher.py | 26 +++++--------------------- visualizer/visualizer.py | 14 +++----------- 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/backend/instrumentview_publisher.py b/backend/instrumentview_publisher.py index ed3fcaa..3e9ee4b 100644 --- a/backend/instrumentview_publisher.py +++ b/backend/instrumentview_publisher.py @@ -1,7 +1,7 @@ import time import numpy import zmq - +import mantid.simpleapi as simpleapi from logger import log import ports from controllable import Controllable @@ -40,28 +40,12 @@ def _publish_clear(self): self.socket.send_json(header) def _publish(self): - #should want readX, readY, readE and spectrum num via getNumberofHistograms() - currentws = self.eventListener._rebin_for_instrumentview_transition.get_checkpoint().data - #print currentws.getNumberHistograms() - #current workspace is what - #clone workspace here pass copy + currentws = simpleapi.CloneWorkspace(self.eventListener._rebin_for_instrumentview_transition.get_checkpoint().data) for i in range(currentws.getNumberHistograms()): - #print i - #seriesX = currentws.readX(i) - seriesX = numpy.array([0,1]) - #seriesX = [0,1] - #seriesY = currentws.readY(i) - seriesY = numpy.array([1]) - #seriesY = [1] - #seriesE = currentws.readE(i) - seriesE = numpy.array([1]) - #seriesE = [1] + seriesX = currentws.readX(i) + seriesY = currentws.readY(i) + seriesE = currentws.readE(i) packet = numpy.concatenate((seriesX, seriesY, seriesE)) - print "here packet" - print packet - print "here split" - x, y, e = numpy.array_split(packet, 3) - print x, y, e header = self._create_header('data', i) self.socket.send_json(header, flags=zmq.SNDMORE) self.socket.send(packet) diff --git a/visualizer/visualizer.py b/visualizer/visualizer.py index b006aae..1d53394 100644 --- a/visualizer/visualizer.py +++ b/visualizer/visualizer.py @@ -20,16 +20,8 @@ def __init__(self, dataListener): simpleapi.LoadInstrument(Workspace=ws, Filename='data/POWDIFF_Definition.xml', RewriteSpectraMap=True) ws = simpleapi.Rebin(ws, "0,1,2") simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) - #simpleapi.AnalysisDataService['POWDIFF_test'] - #simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) + ws.getAxis(0).setUnit('tof') - #ws = simpleapi.AnalysisDataService['POWDIFF_test'] - #simpleapi.LoadInstrument(Workspace=ws, Filename='data/POWDIFF_Definition.xml', RewriteSpectraMap=True) - #ws.getAxis(0).setUnit('tof') - - #ws.setX(1,numpy.array([0,1])) - #ws.setY(1,numpy.array([1])) - #ws.setE(1,numpy.array([1])) InstrumentWidget = mpy.MantidQt.MantidWidgets.InstrumentWidget self.iw = InstrumentWidget('POWDIFF_test') self.iw.show() @@ -44,8 +36,8 @@ def updateInstrumentView(self): while self.dataListener.data: index, x, y, e = self.dataListener.data.popleft() - #ws.dataY(index)[1] = y - ws.dataY(index)[1] = random.randint(1,5) + ws.dataY(index)[1] = y + class DataListener(PyQt4.QtCore.QObject): clear = PyQt4.QtCore.pyqtSignal() From 25c168264e973162c8ddbebc5085ad3133226edd Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Thu, 31 Mar 2016 10:21:38 +0200 Subject: [PATCH 08/22] checkpointing --- backend/instrumentview_publisher.py | 1 + visualizer/visualizer.py | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/backend/instrumentview_publisher.py b/backend/instrumentview_publisher.py index 3e9ee4b..ac83ade 100644 --- a/backend/instrumentview_publisher.py +++ b/backend/instrumentview_publisher.py @@ -40,6 +40,7 @@ def _publish_clear(self): self.socket.send_json(header) def _publish(self): + #TODO send all data for currentws in one go cos latency etc currentws = simpleapi.CloneWorkspace(self.eventListener._rebin_for_instrumentview_transition.get_checkpoint().data) for i in range(currentws.getNumberHistograms()): seriesX = currentws.readX(i) diff --git a/visualizer/visualizer.py b/visualizer/visualizer.py index 1d53394..b157693 100644 --- a/visualizer/visualizer.py +++ b/visualizer/visualizer.py @@ -18,7 +18,7 @@ def __init__(self, dataListener): ws = simpleapi.WorkspaceFactory.Instance().create("Workspace2D", self._number_of_spectra, 2, 1) simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) simpleapi.LoadInstrument(Workspace=ws, Filename='data/POWDIFF_Definition.xml', RewriteSpectraMap=True) - ws = simpleapi.Rebin(ws, "0,1,2") + ws = simpleapi.Rebin(ws, "0,0.5,1.5") simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) ws.getAxis(0).setUnit('tof') @@ -27,16 +27,18 @@ def __init__(self, dataListener): self.iw.show() def clear(self): - self.curves = {} - #TODO + ws = simpleapi.AnalysisDataService['POWDIFF_test'] + def updateInstrumentView(self): #pass ws = simpleapi.AnalysisDataService['POWDIFF_test'] - while self.dataListener.data: index, x, y, e = self.dataListener.data.popleft() - ws.dataY(index)[1] = y + ws.dataY(index)[1] = y #is 2 + ws.dataE(index)[1] = e + ws.dataX(index)[0] = x[0] + ws.dataX(index)[2] = x[1] class DataListener(PyQt4.QtCore.QObject): From 56e4bbe4e1cd3fb9272a45dae84e19cb00e33225 Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Thu, 31 Mar 2016 16:22:53 +0200 Subject: [PATCH 09/22] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8ab8aaa..fdaf918 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,9 @@ mpirun -n python run_backend_reducer.py ``` where `` is the desired number of MPI ranks. It *must* match the number of ranks used for the backend-live-listener. -**Visualizer:** +**Instrument View Visualizer:** ```sh -python run_visualizer.py +python run_instrumentvisualizer.py ``` In all four cases the run script supports the option `-h` to print help, and further options, as described in the respective help. In particular, in several cases you can set the log level with `-l` and the host with `-H` (which defaults to localhost). From 13b162236265b909f84433531b5a1ab9855ca855 Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Mon, 4 Apr 2016 10:35:35 +0200 Subject: [PATCH 10/22] updated to rebins so include all data --- backend/backend_mantid_reducer.py | 2 +- backend/instrumentview_publisher.py | 6 ++++++ visualizer/visualizer.py | 16 +++++++++------- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/backend/backend_mantid_reducer.py b/backend/backend_mantid_reducer.py index 91ab788..9f3e86d 100644 --- a/backend/backend_mantid_reducer.py +++ b/backend/backend_mantid_reducer.py @@ -31,7 +31,7 @@ def __init__(self, data_queue_in): self._gather_histogram_transition = GatherHistogramTransition(self._rebin_transition) self._gather_histogram_transition.accumulate_data = True self._rebin_for_instrumentview_transition = MantidRebinTransition(self._create_workspace_from_events_transition) - self._rebin_for_instrumentview_transition.set_bin_parameters('0,1,1') #set to single bin + self._rebin_for_instrumentview_transition.set_bin_parameters('0,5,10') #set to single bin def _process_command(self, command): setattr(self, command[0], command[1]) diff --git a/backend/instrumentview_publisher.py b/backend/instrumentview_publisher.py index ac83ade..24cac87 100644 --- a/backend/instrumentview_publisher.py +++ b/backend/instrumentview_publisher.py @@ -44,8 +44,14 @@ def _publish(self): currentws = simpleapi.CloneWorkspace(self.eventListener._rebin_for_instrumentview_transition.get_checkpoint().data) for i in range(currentws.getNumberHistograms()): seriesX = currentws.readX(i) + print "this is seriesX" + print seriesX seriesY = currentws.readY(i) + print "this is seriesY" + print seriesY seriesE = currentws.readE(i) + print "this is seriesE" + print seriesE packet = numpy.concatenate((seriesX, seriesY, seriesE)) header = self._create_header('data', i) self.socket.send_json(header, flags=zmq.SNDMORE) diff --git a/visualizer/visualizer.py b/visualizer/visualizer.py index b157693..27fc970 100644 --- a/visualizer/visualizer.py +++ b/visualizer/visualizer.py @@ -18,7 +18,7 @@ def __init__(self, dataListener): ws = simpleapi.WorkspaceFactory.Instance().create("Workspace2D", self._number_of_spectra, 2, 1) simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) simpleapi.LoadInstrument(Workspace=ws, Filename='data/POWDIFF_Definition.xml', RewriteSpectraMap=True) - ws = simpleapi.Rebin(ws, "0,0.5,1.5") + ws = simpleapi.Rebin(ws, "0,5,10") simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) ws.getAxis(0).setUnit('tof') @@ -28,19 +28,21 @@ def __init__(self, dataListener): def clear(self): ws = simpleapi.AnalysisDataService['POWDIFF_test'] - + #could either delete workspace and go through the __init__ again or clear the data individually? def updateInstrumentView(self): - #pass ws = simpleapi.AnalysisDataService['POWDIFF_test'] + #simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) while self.dataListener.data: index, x, y, e = self.dataListener.data.popleft() - ws.dataY(index)[1] = y #is 2 - ws.dataE(index)[1] = e + ws.dataY(index)[0] = y[0] #is 2 + ws.dataE(index)[0] = e[0] ws.dataX(index)[0] = x[0] - ws.dataX(index)[2] = x[1] + ws.dataX(index)[1] = x[1] + ws.dataX(index)[2] = x[2] + # simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) - + class DataListener(PyQt4.QtCore.QObject): clear = PyQt4.QtCore.pyqtSignal() new_data = PyQt4.QtCore.pyqtSignal() From 87f0a788fea6bc0264d05a7dc0f0100263bb72f3 Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Mon, 4 Apr 2016 15:05:19 +0200 Subject: [PATCH 11/22] updated to send all workspace pizel updates in one go --- backend/instrumentview_publisher.py | 20 ++++++++++---------- visualizer/visualizer.py | 26 +++++++++++++++----------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/backend/instrumentview_publisher.py b/backend/instrumentview_publisher.py index 24cac87..e1895d5 100644 --- a/backend/instrumentview_publisher.py +++ b/backend/instrumentview_publisher.py @@ -42,20 +42,20 @@ def _publish_clear(self): def _publish(self): #TODO send all data for currentws in one go cos latency etc currentws = simpleapi.CloneWorkspace(self.eventListener._rebin_for_instrumentview_transition.get_checkpoint().data) + #concatPacket = numpy.empty([1000,3]) #it IS 1000 length, make this improvement later + concatPacket = [] + concatIndex = [] for i in range(currentws.getNumberHistograms()): seriesX = currentws.readX(i) - print "this is seriesX" - print seriesX seriesY = currentws.readY(i) - print "this is seriesY" - print seriesY seriesE = currentws.readE(i) - print "this is seriesE" - print seriesE - packet = numpy.concatenate((seriesX, seriesY, seriesE)) - header = self._create_header('data', i) - self.socket.send_json(header, flags=zmq.SNDMORE) - self.socket.send(packet) + packet = numpy.concatenate((seriesX, seriesY, seriesE)) #and index? + concatPacket= numpy.concatenate((concatPacket, packet)) + #concatIndex.append(i) + #concatPacket.append(packet) + header = self._create_header('data', 1) + self.socket.send_json(header, flags=zmq.SNDMORE) + self.socket.send(concatPacket) def get_parameter_dict(self): return {'update_rate':'float'} diff --git a/visualizer/visualizer.py b/visualizer/visualizer.py index 27fc970..3c2bb9e 100644 --- a/visualizer/visualizer.py +++ b/visualizer/visualizer.py @@ -32,14 +32,18 @@ def clear(self): def updateInstrumentView(self): ws = simpleapi.AnalysisDataService['POWDIFF_test'] - #simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) + simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) while self.dataListener.data: - index, x, y, e = self.dataListener.data.popleft() - ws.dataY(index)[0] = y[0] #is 2 - ws.dataE(index)[0] = e[0] - ws.dataX(index)[0] = x[0] - ws.dataX(index)[1] = x[1] - ws.dataX(index)[2] = x[2] + data = self.dataListener.data.popleft() + index = 0 + for packet in numpy.array_split(data, 1000): + x, y, e = numpy.array_split(packet, 3) + ws.dataY(index)[0] = y[0] #is 2 + ws.dataE(index)[0] = e[0] + ws.dataX(index)[0] = x[0] + ws.dataX(index)[1] = x[1] + ws.dataX(index)[2] = x[2] + index+=1 # simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) @@ -59,8 +63,9 @@ def run(self): while True: command, index = self._receive_header() if command == 'data': - x,y,e = self.get_histogram() - self.data.append((index,x,y,e)) + #index, x,y,e = self.get_histogram() + data = self.get_histogram() + self.data.append((data)) self.new_data.emit() elif command == 'clear': self.clear.emit() @@ -75,8 +80,7 @@ def connect(self): def get_histogram(self): data = numpy.frombuffer(self.socket.recv(), numpy.float64) - x, y, e = numpy.array_split(data, 3) - return x, y, e + return data def _receive_header(self): header = self.socket.recv_json() From b480a06ccb329346545f79774ed6e1043cc1bf5f Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Tue, 5 Apr 2016 10:37:31 +0200 Subject: [PATCH 12/22] readded in plotter, works alongside instrumentview --- backend/instrumentview_publisher.py | 6 +---- backend/result_publisher.py | 2 +- run_backend_reducer.py | 8 +++--- visualizer/__init__.py | 1 + visualizer/visualizer.py | 39 +++++++++++++++++++++++------ 5 files changed, 39 insertions(+), 17 deletions(-) diff --git a/backend/instrumentview_publisher.py b/backend/instrumentview_publisher.py index e1895d5..b5900a1 100644 --- a/backend/instrumentview_publisher.py +++ b/backend/instrumentview_publisher.py @@ -40,9 +40,7 @@ def _publish_clear(self): self.socket.send_json(header) def _publish(self): - #TODO send all data for currentws in one go cos latency etc currentws = simpleapi.CloneWorkspace(self.eventListener._rebin_for_instrumentview_transition.get_checkpoint().data) - #concatPacket = numpy.empty([1000,3]) #it IS 1000 length, make this improvement later concatPacket = [] concatIndex = [] for i in range(currentws.getNumberHistograms()): @@ -51,9 +49,7 @@ def _publish(self): seriesE = currentws.readE(i) packet = numpy.concatenate((seriesX, seriesY, seriesE)) #and index? concatPacket= numpy.concatenate((concatPacket, packet)) - #concatIndex.append(i) - #concatPacket.append(packet) - header = self._create_header('data', 1) + header = self._create_header('instrumentData', 1) self.socket.send_json(header, flags=zmq.SNDMORE) self.socket.send(concatPacket) diff --git a/backend/result_publisher.py b/backend/result_publisher.py index 33231e5..1f63347 100644 --- a/backend/result_publisher.py +++ b/backend/result_publisher.py @@ -47,7 +47,7 @@ def _publish_clear(self): def _publish(self, index): boundaries, values = self.eventListener._gather_histogram_transition.get_checkpoint()[index].data packet = numpy.concatenate((boundaries, values)) - header = self._create_header('data', index) + header = self._create_header('graphData', index) self.socket.send_json(header, flags=zmq.SNDMORE) self.socket.send(packet) diff --git a/run_backend_reducer.py b/run_backend_reducer.py index 30b6578..294e25e 100644 --- a/run_backend_reducer.py +++ b/run_backend_reducer.py @@ -37,14 +37,14 @@ reducer_controller_thread = threading.Thread(target=reducer_controller.run) reducer_controller_thread.start() - #resultPublisher = ResultPublisher(reducer) - #resultPublisher_thread = threading.Thread(target=resultPublisher.run) - #resultPublisher_thread.start() + resultPublisher = ResultPublisher(reducer) + resultPublisher_thread = threading.Thread(target=resultPublisher.run) + resultPublisher_thread.start() instrumentViewPublisher = InstrumentViewPublisher(reducer) instrumentViewPublisher_thread = threading.Thread(target=instrumentViewPublisher.run) instrumentViewPublisher_thread.start() parameterController = ParameterControlServer(controllees=[instrumentViewPublisher], port=ports.result_publisher_control) - #parameterController = ParameterControlServer(controllees=[resultPublisher], port=ports.result_publisher_control) + parameterController = ParameterControlServer(controllees=[resultPublisher], port=ports.result_publisher_control) parameterController_thread = threading.Thread(target=parameterController.run) parameterController_thread.start() diff --git a/visualizer/__init__.py b/visualizer/__init__.py index a2aab47..c73bfd8 100644 --- a/visualizer/__init__.py +++ b/visualizer/__init__.py @@ -1,2 +1,3 @@ from visualizer import InstrumentView from visualizer import DataListener +from visualizer import Plotter diff --git a/visualizer/visualizer.py b/visualizer/visualizer.py index 3c2bb9e..82e95af 100644 --- a/visualizer/visualizer.py +++ b/visualizer/visualizer.py @@ -7,6 +7,8 @@ import mantidqtpython as mpy import random from logger import log +import pyqtgraph as pg +from pyqtgraph.Qt import QtCore ConfigService = simpleapi.ConfigService @@ -38,14 +40,33 @@ def updateInstrumentView(self): index = 0 for packet in numpy.array_split(data, 1000): x, y, e = numpy.array_split(packet, 3) - ws.dataY(index)[0] = y[0] #is 2 - ws.dataE(index)[0] = e[0] + ws.dataY(index)[1] = y[0] #is 2 + ws.dataE(index)[1] = e[0] ws.dataX(index)[0] = x[0] ws.dataX(index)[1] = x[1] ws.dataX(index)[2] = x[2] index+=1 - # simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) +class Plotter(object): + def __init__(self, dataListener): + self.dataListener = dataListener + self.win = pg.GraphicsWindow() + self.win.resize(800,350) + self.win.setWindowTitle('pyqtgraph example: Histogram') + self.plt1 = self.win.addPlot() + self.curves = {} + + def clear(self): + self.curves = {} + self.plt1.clear() + + def update(self): + while self.dataListener.data: + index,x,y = self.dataListener.data.popleft() + if not index in self.curves: + self.curves[index] = self.plt1.plot(stepMode=True, pen=(index), name=str(index)) + self.curves[index].setData(x, y) + self.plt1.enableAutoRange('xy', False) class DataListener(PyQt4.QtCore.QObject): clear = PyQt4.QtCore.pyqtSignal() @@ -62,11 +83,14 @@ def run(self): self.connect() while True: command, index = self._receive_header() - if command == 'data': - #index, x,y,e = self.get_histogram() - data = self.get_histogram() + if command == 'instrumentData': + data = numpy.frombuffer(self.socket.recv(), numpy.float64) self.data.append((data)) self.new_data.emit() + if command == 'graphData': + x,y = self.get_histogram() + self.data.append((index,x,y)) + self.new_data.emit() elif command == 'clear': self.clear.emit() @@ -80,7 +104,8 @@ def connect(self): def get_histogram(self): data = numpy.frombuffer(self.socket.recv(), numpy.float64) - return data + x,y = numpy.array_split(data, 2) + return x,y def _receive_header(self): header = self.socket.recv_json() From 018fa1d57a2a8a982e32a58f6c2bc876bcd30663 Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Tue, 5 Apr 2016 14:27:25 +0200 Subject: [PATCH 13/22] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fdaf918..d768b3e 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,10 @@ mpirun -n python run_backend_reducer.py ``` where `` is the desired number of MPI ranks. It *must* match the number of ranks used for the backend-live-listener. -**Instrument View Visualizer:** +**Visualizers for instrumentview and histogram:** ```sh python run_instrumentvisualizer.py +python run_visualizer.py ``` In all four cases the run script supports the option `-h` to print help, and further options, as described in the respective help. In particular, in several cases you can set the log level with `-l` and the host with `-H` (which defaults to localhost). From 257b9398290316def83ba36b586c294e2481f77f Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Wed, 6 Apr 2016 15:42:55 +0200 Subject: [PATCH 14/22] added single spectra visualiser --- backend/__init__.py | 1 + backend/backend_mantid_reducer.py | 3 ++ backend/gather_spectra_transition.py | 25 ++++++++++++ backend/result_publisher.py | 1 + backend/spectraview_publisher.py | 60 ++++++++++++++++++++++++++++ ports.py | 1 + run_backend_reducer.py | 4 ++ run_spectraVisualizer.py | 27 +++++++++++++ visualizer/__init__.py | 1 + visualizer/visualizer.py | 31 ++++++++++++++ 10 files changed, 154 insertions(+) create mode 100644 backend/gather_spectra_transition.py create mode 100644 backend/spectraview_publisher.py create mode 100644 run_spectraVisualizer.py diff --git a/backend/__init__.py b/backend/__init__.py index 2a7a7bb..b753697 100644 --- a/backend/__init__.py +++ b/backend/__init__.py @@ -2,5 +2,6 @@ from backend_mantid_reducer import BackendMantidReducer from result_publisher import ResultPublisher from instrumentview_publisher import InstrumentViewPublisher +from spectraview_publisher import SpectraViewPublisher from zmq_queue import ZMQQueueServer from zmq_queue import ZMQQueueClient diff --git a/backend/backend_mantid_reducer.py b/backend/backend_mantid_reducer.py index 9f3e86d..8024fbf 100644 --- a/backend/backend_mantid_reducer.py +++ b/backend/backend_mantid_reducer.py @@ -13,6 +13,7 @@ from mantid_filter_transition import MantidFilterTransition from mantid_rebin_transition import MantidRebinTransition from gather_histogram_transition import GatherHistogramTransition +from gather_spectra_transition import GatherSpectraTransition class BackendMantidReducer(BackendWorker): @@ -32,6 +33,8 @@ def __init__(self, data_queue_in): self._gather_histogram_transition.accumulate_data = True self._rebin_for_instrumentview_transition = MantidRebinTransition(self._create_workspace_from_events_transition) self._rebin_for_instrumentview_transition.set_bin_parameters('0,5,10') #set to single bin + self._gather_spectra_transition = GatherSpectraTransition(self._create_workspace_from_events_transition) + #at this point set up what spectra index you want etc def _process_command(self, command): setattr(self, command[0], command[1]) diff --git a/backend/gather_spectra_transition.py b/backend/gather_spectra_transition.py new file mode 100644 index 0000000..ffc1f79 --- /dev/null +++ b/backend/gather_spectra_transition.py @@ -0,0 +1,25 @@ +import numpy +from mpi4py import MPI +import random +from transition import Transition +from histogram_checkpoint import HistogramCheckpoint + + +class GatherSpectraTransition(Transition): + def __init__(self, parent): + self._comm = MPI.COMM_WORLD + self._spectra_id = '11' + super(GatherSpectraTransition, self).__init__(parents=[parent]) + + def _do_transition(self, data): + index = int(self._spectra_id) + data = data[0].data + x = data.readX(index) + y = data.readY(index) + #gathered = self._comm.gather(bin_values, root=0) + #if self._comm.Get_rank() == 0: + # bin_values = sum(gathered) + return x, y, index + + def _create_checkpoint(self): + return HistogramCheckpoint() diff --git a/backend/result_publisher.py b/backend/result_publisher.py index 1f63347..a347499 100644 --- a/backend/result_publisher.py +++ b/backend/result_publisher.py @@ -47,6 +47,7 @@ def _publish_clear(self): def _publish(self, index): boundaries, values = self.eventListener._gather_histogram_transition.get_checkpoint()[index].data packet = numpy.concatenate((boundaries, values)) +# print "this boundaries " +str(boundaries) header = self._create_header('graphData', index) self.socket.send_json(header, flags=zmq.SNDMORE) self.socket.send(packet) diff --git a/backend/spectraview_publisher.py b/backend/spectraview_publisher.py new file mode 100644 index 0000000..e820438 --- /dev/null +++ b/backend/spectraview_publisher.py @@ -0,0 +1,60 @@ +import time +import numpy +import zmq +import mantid.simpleapi as simpleapi +from logger import log +import ports +from controllable import Controllable + + +class SpectraViewPublisher(Controllable): + def __init__(self, eventListener): + super(SpectraViewPublisher, self).__init__(type(self).__name__) + self.eventListener = eventListener + self._update_rate = 1.0 + self.socket = None + self._last_count = 0 + + def run(self): + log.info("Starting SpectraViewPublisher") + self.connect() + + self._publish_clear() + while True: + if self.eventListener._create_workspace_from_events_transition.get_checkpoint(): + self._publish() + time.sleep(self.update_rate) + + def connect(self): + context = zmq.Context() + self.socket = context.socket(zmq.PUB) + uri = 'tcp://*:{0:d}'.format(ports.spectra_result_stream) + self.socket.bind(uri) + log.info('Bound to ' + uri) + + def _create_header(self, command, index): + return { 'command':command, 'index':index } + + def _publish_clear(self): + header = self._create_header('clear', None) + self.socket.send_json(header) + + def _publish(self): + x, y = self.eventListener._gather_spectra_transition.get_checkpoint().data + index = self.eventListener._gather_spectra_transition._spectra_id + packet = numpy.concatenate((x, y)) + print y + header = self._create_header('spectraData', index) + self.socket.send_json(header, flags=zmq.SNDMORE) + self.socket.send(packet) + + def get_parameter_dict(self): + return {'update_rate':'float'} + + @property + def update_rate(self): + return self._update_rate + + @update_rate.setter + def update_rate(self, update_rate): + self._update_rate = update_rate diff --git a/ports.py b/ports.py index a603355..d218f36 100644 --- a/ports.py +++ b/ports.py @@ -4,3 +4,4 @@ result_publisher_control = 10004 rebin_control = 10005 instrumentview_result_stream = 10006 +spectra_result_stream = 10007 diff --git a/run_backend_reducer.py b/run_backend_reducer.py index 294e25e..ea6f6d4 100644 --- a/run_backend_reducer.py +++ b/run_backend_reducer.py @@ -2,6 +2,7 @@ from backend import BackendMantidReducer from backend import ResultPublisher from backend import InstrumentViewPublisher +from backend import SpectraViewPublisher from backend import ZMQQueueServer from backend import ZMQQueueClient from parameter_control_server import ParameterControlServer @@ -43,6 +44,9 @@ instrumentViewPublisher = InstrumentViewPublisher(reducer) instrumentViewPublisher_thread = threading.Thread(target=instrumentViewPublisher.run) instrumentViewPublisher_thread.start() + spectraViewPublisher = SpectraViewPublisher(reducer) + spectraViewPublisher_thread = threading.Thread(target=spectraViewPublisher.run) + spectraViewPublisher_thread.start() parameterController = ParameterControlServer(controllees=[instrumentViewPublisher], port=ports.result_publisher_control) parameterController = ParameterControlServer(controllees=[resultPublisher], port=ports.result_publisher_control) parameterController_thread = threading.Thread(target=parameterController.run) diff --git a/run_spectraVisualizer.py b/run_spectraVisualizer.py new file mode 100644 index 0000000..7296046 --- /dev/null +++ b/run_spectraVisualizer.py @@ -0,0 +1,27 @@ +from threading import Thread + +from pyqtgraph.Qt import QtGui + +from logger import setup_global_logger +import ports +import command_line_parser +from visualizer import DataListener +from visualizer import SpectraPlotter + + +setup_global_logger(level=command_line_parser.get_log_level()) + +data_listener = DataListener(host=command_line_parser.get_host(), port=ports.spectra_result_stream) +data_listener_thread = Thread(target=data_listener.run) +data_listener_thread.daemon = True +spectraPlotter = SpectraPlotter(data_listener) + +data_listener.clear.connect(spectraPlotter.clear) +data_listener.new_data.connect(spectraPlotter.update) +data_listener_thread.start() + +## Start Qt event loop unless running in interactive mode or using pyside. +if __name__ == '__main__': + import sys + if (sys.flags.interactive != 1) or not hasattr(QtCore, 'PYQT_VERSION'): + QtGui.QApplication.instance().exec_() diff --git a/visualizer/__init__.py b/visualizer/__init__.py index c73bfd8..4f5a01a 100644 --- a/visualizer/__init__.py +++ b/visualizer/__init__.py @@ -1,3 +1,4 @@ from visualizer import InstrumentView from visualizer import DataListener from visualizer import Plotter +from visualizer import SpectraPlotter diff --git a/visualizer/visualizer.py b/visualizer/visualizer.py index 82e95af..3e8c873 100644 --- a/visualizer/visualizer.py +++ b/visualizer/visualizer.py @@ -68,6 +68,33 @@ def update(self): self.curves[index].setData(x, y) self.plt1.enableAutoRange('xy', False) +class SpectraPlotter(object): + def __init__(self, dataListener): + self.dataListener = dataListener + self.win = pg.GraphicsWindow() + self.win.resize(800,350) + self.win.setWindowTitle('Spectra for dectector') + self.plt1 = self.win.addPlot() + self.curves = {} + self.old_index = None + self.firstData = True + + def clear(self): + self.curves = {} + self.plt1.clear() + + def update(self): + while self.dataListener.data: + index,x,y = self.dataListener.data.popleft() + if not self.firstData: #feels weird fast updating at the moment... maybe it can not update as much + self.plt1.removeItem(self.curves[1]) + self.firstData = False + self.win.setWindowTitle('Spectra for detector ' + str(index)) + y2 = numpy.append(y, [0]) #check best way to make y match x len + self.curves[1] = self.plt1.plot(x, y2)#stepMode=True, pen=(index), name=str(index)) + self.plt1.enableAutoRange('xy', True) + + class DataListener(PyQt4.QtCore.QObject): clear = PyQt4.QtCore.pyqtSignal() new_data = PyQt4.QtCore.pyqtSignal() @@ -91,6 +118,10 @@ def run(self): x,y = self.get_histogram() self.data.append((index,x,y)) self.new_data.emit() + if command == 'spectraData': + x, y = self.get_histogram() + self.data.append((index,x,y)) + self.new_data.emit() elif command == 'clear': self.clear.emit() From 8a6124deb79f44f8b4d207aed50853eadc6e2d96 Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Wed, 6 Apr 2016 18:27:32 +0200 Subject: [PATCH 15/22] Can change spectra to be viewed using control.py --- backend/backend_mantid_reducer.py | 17 +++++++++++++++-- backend/gather_spectra_transition.py | 4 ++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/backend/backend_mantid_reducer.py b/backend/backend_mantid_reducer.py index 8024fbf..eff69b0 100644 --- a/backend/backend_mantid_reducer.py +++ b/backend/backend_mantid_reducer.py @@ -34,7 +34,7 @@ def __init__(self, data_queue_in): self._rebin_for_instrumentview_transition = MantidRebinTransition(self._create_workspace_from_events_transition) self._rebin_for_instrumentview_transition.set_bin_parameters('0,5,10') #set to single bin self._gather_spectra_transition = GatherSpectraTransition(self._create_workspace_from_events_transition) - #at this point set up what spectra index you want etc + self._gather_spectra_transition.set_spectra_id('1') def _process_command(self, command): setattr(self, command[0], command[1]) @@ -81,8 +81,11 @@ def get_bin_boundaries(self): def get_bin_values(self): return self._rebin_transition.get_checkpoint()[-1].data.readY(0) + def get_spectra_id(self): + return self._gather_spectra_transition._spectra_id + def get_parameter_dict(self): - return {'bin_parameters':'str', 'filter_interval_parameters':'str', 'filter_pulses':'bool'} + return {'bin_parameters':'str', 'filter_interval_parameters':'str', 'filter_pulses':'bool', 'spectra_id':'str'} @property def bin_parameters(self): @@ -94,6 +97,16 @@ def bin_parameters(self, parameters): self._rebin_transition.set_bin_parameters(parameters) self._lock.release() + @property + def spectra_id(self): + return self._gather_spectra_transition._spectra_id + + @spectra_id.setter + def spectra_id(self, spectra): + self._lock.acquire() + self._gather_spectra_transition.set_spectra_id(spectra) + self._lock.release() + @property def filter_pulses(self): return self._filter_pulses diff --git a/backend/gather_spectra_transition.py b/backend/gather_spectra_transition.py index ffc1f79..f1f44b8 100644 --- a/backend/gather_spectra_transition.py +++ b/backend/gather_spectra_transition.py @@ -21,5 +21,9 @@ def _do_transition(self, data): # bin_values = sum(gathered) return x, y, index + def set_spectra_id(self, spectra_id): + self._spectra_id = spectra_id + self.trigger_rerun() + def _create_checkpoint(self): return HistogramCheckpoint() From 9d6965c1beb56eefe959f24467e5861d07a71826 Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Mon, 25 Apr 2016 14:31:01 +0200 Subject: [PATCH 16/22] current status checkpoint --- backend/backend_event_listener.py | 19 ++++++++++-- backend/gather_spectra_transition.py | 7 +++-- backend/result_publisherM.py | 46 ++++++++++++++++++++++++++++ visualizer/visualizer.py | 9 ++++++ 4 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 backend/result_publisherM.py diff --git a/backend/backend_event_listener.py b/backend/backend_event_listener.py index b78d40a..e6dcb6f 100644 --- a/backend/backend_event_listener.py +++ b/backend/backend_event_listener.py @@ -57,12 +57,17 @@ def _distribute_stream(self, what, data): if what == 'meta_data': split = [data] * self._comm.size else: + #split = distribute_stream_split(self._comm.size, data) split = [] for i in range(self._comm.size): split.append([]) - for i in data: - detector_id = int(i[0]) + for i in data: #make this into a separate function to be used by here and spectra_transition + detector_id = int(i[0]) + #print "this is detector ID" + #print detector_id target = detector_id % self._comm.size + #print "whatever i is:" + #print i split[target].append(i) else: split = None @@ -72,3 +77,13 @@ def _distribute_stream(self, what, data): return what, data else: return what, numpy.array(data) + +#distribute_stream_split(comm_size, data): +# split = [] +# for i in range(comm.size): +# split.append([]) +# for i in data: +# detector_id = int(i[0]) +# target = detector_id % comm.size +# split[target].append(i) + diff --git a/backend/gather_spectra_transition.py b/backend/gather_spectra_transition.py index f1f44b8..a84dde9 100644 --- a/backend/gather_spectra_transition.py +++ b/backend/gather_spectra_transition.py @@ -16,9 +16,10 @@ def _do_transition(self, data): data = data[0].data x = data.readX(index) y = data.readY(index) - #gathered = self._comm.gather(bin_values, root=0) - #if self._comm.Get_rank() == 0: - # bin_values = sum(gathered) + #if self._comm.Get_rank() == 0: #assuming rootrank + # self._comm.recv(index, source=MPI.ANY_SOURCE) #wants to receive the x,y data (and index?) + #else: + # self._comm.send(index, dest=0) #if it is the rank with the readX/Y data, it wants to send it to the root rank return x, y, index def set_spectra_id(self, spectra_id): diff --git a/backend/result_publisherM.py b/backend/result_publisherM.py new file mode 100644 index 0000000..31a91f6 --- /dev/null +++ b/backend/result_publisherM.py @@ -0,0 +1,46 @@ +import numpy +import zmq + +from logger import log +import ports +from controllable import Controllable + +class ResultPublisher(Controllable): + def __init__(self, eventListener): + super(ResultPublisher, self).__init__(type(self).__name__) + self.eventListener = eventListener + self._update_rate = 1.0 + self.socket = None + self._last_count = 0 + self.ports = None + + def run(self): + pass + + def connect(self): + context = zmq.Context() + self.socket = context.socket(zmq.PUB) + uri = 'tcp://*:{0:d}'.format(self.ports) + self.socket.bind(uri) + log.info('Bound to ' + uri) + + def _create_header(self,command,index): + return {'command':command, 'index':index} + + def _publish_clear(self): + header = self._create_header('clear', None) + self.socket.send_json(header) + + def _publish(self, index): + pass + + def get_parameter_dict(self): + return {'update_rate':'float'} + + @property + def update_rate(self): + return self._update_rate + + @update_rate.setter + def update_rate(self, update_rate): + self._update_rate = update_rate diff --git a/visualizer/visualizer.py b/visualizer/visualizer.py index 3e8c873..b3a012b 100644 --- a/visualizer/visualizer.py +++ b/visualizer/visualizer.py @@ -14,6 +14,7 @@ class InstrumentView(object): def __init__(self, dataListener): + self.dataListener = dataListener ws = simpleapi.CreateSimulationWorkspace(Instrument='data/POWDIFF_Definition.xml', BinParams='1,0.5,2') self._number_of_spectra = ws.getNumberHistograms() @@ -27,12 +28,19 @@ def __init__(self, dataListener): InstrumentWidget = mpy.MantidQt.MantidWidgets.InstrumentWidget self.iw = InstrumentWidget('POWDIFF_test') self.iw.show() +# pickTab = self.iw.getTab(InstrumentWindow.PICK) + #self.connect(pickTab, SIGNAL("pickTab.wsIndexChanged(int)"),self.getNewIndex()) + + def getNewIndex(): + print "got new index" + def clear(self): ws = simpleapi.AnalysisDataService['POWDIFF_test'] #could either delete workspace and go through the __init__ again or clear the data individually? def updateInstrumentView(self): + #receive signal that it has updated here, then print out new number? ws = simpleapi.AnalysisDataService['POWDIFF_test'] simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) while self.dataListener.data: @@ -98,6 +106,7 @@ def update(self): class DataListener(PyQt4.QtCore.QObject): clear = PyQt4.QtCore.pyqtSignal() new_data = PyQt4.QtCore.pyqtSignal() + def __init__(self, host, port): PyQt4.QtCore.QObject.__init__(self) From 9756ca33174b3cf9c169fee63a11814717519d83 Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Tue, 26 Apr 2016 15:40:45 +0200 Subject: [PATCH 17/22] spectra graph now updates depending on detID hovered over. Requires live-data-prototype branch of mantid --- visualizer/visualizer.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/visualizer/visualizer.py b/visualizer/visualizer.py index b3a012b..ec57a05 100644 --- a/visualizer/visualizer.py +++ b/visualizer/visualizer.py @@ -1,4 +1,6 @@ from collections import deque +from parameter_control_client import ParameterControlClient +from general_parameter_controller import GeneralParameterController import zmq import sys import numpy @@ -24,16 +26,20 @@ def __init__(self, dataListener): ws = simpleapi.Rebin(ws, "0,5,10") simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) ws.getAxis(0).setUnit('tof') + control_client = ParameterControlClient("localhost", 10005) + self.controller = GeneralParameterController(control_client) + InstrumentWidget = mpy.MantidQt.MantidWidgets.InstrumentWidget self.iw = InstrumentWidget('POWDIFF_test') self.iw.show() -# pickTab = self.iw.getTab(InstrumentWindow.PICK) - #self.connect(pickTab, SIGNAL("pickTab.wsIndexChanged(int)"),self.getNewIndex()) - - def getNewIndex(): - print "got new index" + self.pickTab = self.iw.getTab("Pick") + self.pickTab.__class__ = mpy.MantidQt.MantidWidgets.InstrumentWidgetPickTab + dir(self.pickTab) + def updateDetectorID(self): + self.controller.set_parameter_value('BackendMantidReducer', 'spectra_id', self.pickTab.get_currentPickID()) + #update this to be -1 and therefore null when mouse off spectragraph? def clear(self): ws = simpleapi.AnalysisDataService['POWDIFF_test'] @@ -43,6 +49,7 @@ def updateInstrumentView(self): #receive signal that it has updated here, then print out new number? ws = simpleapi.AnalysisDataService['POWDIFF_test'] simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) + self.updateDetectorID() while self.dataListener.data: data = self.dataListener.data.popleft() index = 0 @@ -54,6 +61,7 @@ def updateInstrumentView(self): ws.dataX(index)[1] = x[1] ws.dataX(index)[2] = x[2] index+=1 + #self.updateDetectorID() class Plotter(object): def __init__(self, dataListener): From 50a3cd6e55a2b4f8b4ae89a7eefa9a3060c67227 Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Tue, 26 Apr 2016 17:27:46 +0200 Subject: [PATCH 18/22] checkpointing --- backend/backend_event_listener.py | 6 +--- backend/gather_spectra_transition.py | 30 +++++++++++++++---- backend/spectraview_publisher.py | 1 - picktabtest.py | 43 ++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 12 deletions(-) create mode 100644 picktabtest.py diff --git a/backend/backend_event_listener.py b/backend/backend_event_listener.py index e6dcb6f..ee69152 100644 --- a/backend/backend_event_listener.py +++ b/backend/backend_event_listener.py @@ -63,11 +63,7 @@ def _distribute_stream(self, what, data): split.append([]) for i in data: #make this into a separate function to be used by here and spectra_transition detector_id = int(i[0]) - #print "this is detector ID" - #print detector_id - target = detector_id % self._comm.size - #print "whatever i is:" - #print i + target = detector_id % self._comm.size #comm.size is number of processes, will be same in all ranks split[target].append(i) else: split = None diff --git a/backend/gather_spectra_transition.py b/backend/gather_spectra_transition.py index a84dde9..2ed5add 100644 --- a/backend/gather_spectra_transition.py +++ b/backend/gather_spectra_transition.py @@ -12,15 +12,33 @@ def __init__(self, parent): super(GatherSpectraTransition, self).__init__(parents=[parent]) def _do_transition(self, data): + #get knowledge of index required (not all processes know this?) index = int(self._spectra_id) + print "my rank is" + print self._comm.Get_rank() + print "the index i know is" + print index data = data[0].data - x = data.readX(index) - y = data.readY(index) - #if self._comm.Get_rank() == 0: #assuming rootrank - # self._comm.recv(index, source=MPI.ANY_SOURCE) #wants to receive the x,y data (and index?) - #else: - # self._comm.send(index, dest=0) #if it is the rank with the readX/Y data, it wants to send it to the root rank + x = data.readX(index) + y = data.readY(index) +# if self._comm.Get_rank() == 0: #assuming rootrank +# self._comm.recv(x, source=MPI.ANY_SOURCE) #wants to receive the x,y data (and index?) +# else: +# try: +# print "got here!" +# x = data.readX(index) +# y = data.readY(index) +# self._comm.send(x) +# except: +# pass +# + return x, y, index +#assume has knowledge of index of data +#find out which rank has the data assoc with the index +#specifically query that rank to get the data + + def set_spectra_id(self, spectra_id): self._spectra_id = spectra_id diff --git a/backend/spectraview_publisher.py b/backend/spectraview_publisher.py index e820438..260a16c 100644 --- a/backend/spectraview_publisher.py +++ b/backend/spectraview_publisher.py @@ -43,7 +43,6 @@ def _publish(self): x, y = self.eventListener._gather_spectra_transition.get_checkpoint().data index = self.eventListener._gather_spectra_transition._spectra_id packet = numpy.concatenate((x, y)) - print y header = self._create_header('spectraData', index) self.socket.send_json(header, flags=zmq.SNDMORE) self.socket.send(packet) diff --git a/picktabtest.py b/picktabtest.py new file mode 100644 index 0000000..744f6a2 --- /dev/null +++ b/picktabtest.py @@ -0,0 +1,43 @@ +from threading import Thread +import time +import random +import sys +import PyQt4 +import mantid.simpleapi as simpleapi +import mantidqtpython as mpy +import numpy + +ws = simpleapi.WorkspaceFactory.create("Workspace2D", 1000, 2,1) +simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) +simpleapi.LoadInstrument(Workspace=ws, Filename='data/POWDIFF_Definition.xml', RewriteSpectraMap=True) +ws = simpleapi.Rebin(ws, "0,0.1,2") +simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) +InstrumentWidget = mpy.MantidQt.MantidWidgets.InstrumentWidget +app = PyQt4.QtGui.QApplication(sys.argv) +iw = InstrumentWidget("POWDIFF_test") +iw.show() +picktab = iw.getTab("Pick") +picktab.__class__ = mpy.MantidQt.MantidWidgets.InstrumentWidgetPickTab +#picktab.get_currentPickID() +dir(picktab) + +x = 1 +def addtoY(): + x =1 + while 1: + time.sleep(1) + x+=1 + print x + + ws.dataY(2)[1]+=1 + ws.dataY(random.randint(1,999))[random.randint(1,9)] += random.randint(1,6) + simpleapi.AnalysisDataService.Instance().addOrReplace('POWDIFF_test', ws) #added this line + print "the detector ID is" + print picktab.get_currentPickID() + + +ws.dataY(1)[9] = 3 +ws.dataY(1)[8] = 9 +addThread = Thread(target=addtoY) +addThread.start() +app.exec_() From 0d5fd1c2c9c5af9fa7fdd7d50449e0ba7b11915a Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Mon, 9 May 2016 10:40:17 +0200 Subject: [PATCH 19/22] Working instrumentvisualiser and spectraview with MPI --- backend/gather_spectra_transition.py | 38 +++++++++++++++------------- backend/spectraview_publisher.py | 6 ++++- visualizer/visualizer.py | 13 +++++++--- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/backend/gather_spectra_transition.py b/backend/gather_spectra_transition.py index 2ed5add..11260be 100644 --- a/backend/gather_spectra_transition.py +++ b/backend/gather_spectra_transition.py @@ -8,30 +8,32 @@ class GatherSpectraTransition(Transition): def __init__(self, parent): self._comm = MPI.COMM_WORLD - self._spectra_id = '11' + self._spectra_id = '1' #also set as 1 in backendreducer super(GatherSpectraTransition, self).__init__(parents=[parent]) def _do_transition(self, data): - #get knowledge of index required (not all processes know this?) - index = int(self._spectra_id) - print "my rank is" - print self._comm.Get_rank() - print "the index i know is" - print index + index = int(self._spectra_id) #workspaceIndex + #print index + #index = 11 + target = index % self._comm.size #target now mpi process with det id information + #print target data = data[0].data x = data.readX(index) y = data.readY(index) -# if self._comm.Get_rank() == 0: #assuming rootrank -# self._comm.recv(x, source=MPI.ANY_SOURCE) #wants to receive the x,y data (and index?) -# else: -# try: -# print "got here!" -# x = data.readX(index) -# y = data.readY(index) -# self._comm.send(x) -# except: -# pass -# + if target is not 0: #to avoid the target being 0 and then requesting information for itself, which when using blocking sends and recieves would hang the program + if self._comm.Get_rank() == target: + self._comm.send(x, dest = 0) + self._comm.send(y, dest = 0) + if self._comm.Get_rank() == 0: + x = self._comm.recv(source=target) + y = self._comm.recv(source=target) +# if self._comm.Get_rank() == target: #assuming rootrank +# request = self._comm.Isend(x, dest=0) #as request object, use wait method on it later on, can lower case this for picking any python object +# request = self._comm.Isend(y, dest=0) + # if self._comm.Get_rank() == 0: +# x = self._comm.recv(source=target) +# y = self._comm.recv(source=target) + return x, y, index #assume has knowledge of index of data diff --git a/backend/spectraview_publisher.py b/backend/spectraview_publisher.py index 260a16c..e16b0b5 100644 --- a/backend/spectraview_publisher.py +++ b/backend/spectraview_publisher.py @@ -40,7 +40,11 @@ def _publish_clear(self): self.socket.send_json(header) def _publish(self): - x, y = self.eventListener._gather_spectra_transition.get_checkpoint().data + try: + x, y = self.eventListener._gather_spectra_transition.get_checkpoint().data + except TypeError: + time.sleep(5) + x, y = self.eventListener._gather_spectra_transition.get_checkpoint().data index = self.eventListener._gather_spectra_transition._spectra_id packet = numpy.concatenate((x, y)) header = self._create_header('spectraData', index) diff --git a/visualizer/visualizer.py b/visualizer/visualizer.py index ec57a05..f96ed0c 100644 --- a/visualizer/visualizer.py +++ b/visualizer/visualizer.py @@ -33,13 +33,20 @@ def __init__(self, dataListener): InstrumentWidget = mpy.MantidQt.MantidWidgets.InstrumentWidget self.iw = InstrumentWidget('POWDIFF_test') self.iw.show() + self.pickTab = self.iw.getTab("Pick") - self.pickTab.__class__ = mpy.MantidQt.MantidWidgets.InstrumentWidgetPickTab - dir(self.pickTab) + self.pickTab.__class__ = mpy.MantidQt.MantidWidgets.InstrumentWidgetPickTab + #while getTab does return an InstrumentWidgetPickTab class object, python assumes it is still just an InstrumentWidget object until it is specifically cast + dir(self.pickTab) + #after being cast, the class attributes have not yet been updated, calling dir() does this + self.pickID = self.pickTab.get_currentPickID() def updateDetectorID(self): - self.controller.set_parameter_value('BackendMantidReducer', 'spectra_id', self.pickTab.get_currentPickID()) + if self.pickID != self.pickTab.get_currentPickID(): + self.pickID = self.pickTab.get_currentPickID() + self.controller.set_parameter_value('BackendMantidReducer', 'spectra_id', self.pickID) #update this to be -1 and therefore null when mouse off spectragraph? + def clear(self): ws = simpleapi.AnalysisDataService['POWDIFF_test'] From fba6855af30390fe0bb74e49067807776c476f03 Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Mon, 9 May 2016 11:45:22 +0200 Subject: [PATCH 20/22] extracted out datasplitting function --- backend/MPIDataSplit.py | 6 ++++++ backend/backend_event_listener.py | 18 ++++-------------- backend/gather_spectra_transition.py | 7 ++----- 3 files changed, 12 insertions(+), 19 deletions(-) create mode 100644 backend/MPIDataSplit.py diff --git a/backend/MPIDataSplit.py b/backend/MPIDataSplit.py new file mode 100644 index 0000000..a63f9c9 --- /dev/null +++ b/backend/MPIDataSplit.py @@ -0,0 +1,6 @@ + + +def determine_data_split(index, num_processes): + target = index % num_processes + return target + diff --git a/backend/backend_event_listener.py b/backend/backend_event_listener.py index ee69152..16b0465 100644 --- a/backend/backend_event_listener.py +++ b/backend/backend_event_listener.py @@ -5,7 +5,7 @@ from logger import log from backend_worker import BackendWorker - +import MPIDataSplit # master connectes to main event stream and distributes it to all # packets are put in a queue @@ -57,14 +57,13 @@ def _distribute_stream(self, what, data): if what == 'meta_data': split = [data] * self._comm.size else: - #split = distribute_stream_split(self._comm.size, data) split = [] for i in range(self._comm.size): split.append([]) - for i in data: #make this into a separate function to be used by here and spectra_transition + for i in data: detector_id = int(i[0]) - target = detector_id % self._comm.size #comm.size is number of processes, will be same in all ranks - split[target].append(i) + target = MPIDataSplit.determine_data_split(detector_id, self._comm.size) + split[target].append(i) else: split = None what = self._comm.scatter([what]*self._comm.size, root=0) @@ -74,12 +73,3 @@ def _distribute_stream(self, what, data): else: return what, numpy.array(data) -#distribute_stream_split(comm_size, data): -# split = [] -# for i in range(comm.size): -# split.append([]) -# for i in data: -# detector_id = int(i[0]) -# target = detector_id % comm.size -# split[target].append(i) - diff --git a/backend/gather_spectra_transition.py b/backend/gather_spectra_transition.py index 11260be..9be562a 100644 --- a/backend/gather_spectra_transition.py +++ b/backend/gather_spectra_transition.py @@ -3,7 +3,7 @@ import random from transition import Transition from histogram_checkpoint import HistogramCheckpoint - +import MPIDataSplit class GatherSpectraTransition(Transition): def __init__(self, parent): @@ -13,10 +13,7 @@ def __init__(self, parent): def _do_transition(self, data): index = int(self._spectra_id) #workspaceIndex - #print index - #index = 11 - target = index % self._comm.size #target now mpi process with det id information - #print target + target = MPIDataSplit.determine_data_split(index, self._comm.size) data = data[0].data x = data.readX(index) y = data.readY(index) From c60b0fb490154a17140ec3e4204e62a25b33a759 Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Mon, 9 May 2016 15:15:03 +0200 Subject: [PATCH 21/22] Updated handling of mpi threads in spectra_transition --- backend/gather_spectra_transition.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/backend/gather_spectra_transition.py b/backend/gather_spectra_transition.py index 9be562a..723be07 100644 --- a/backend/gather_spectra_transition.py +++ b/backend/gather_spectra_transition.py @@ -17,25 +17,14 @@ def _do_transition(self, data): data = data[0].data x = data.readX(index) y = data.readY(index) - if target is not 0: #to avoid the target being 0 and then requesting information for itself, which when using blocking sends and recieves would hang the program - if self._comm.Get_rank() == target: - self._comm.send(x, dest = 0) - self._comm.send(y, dest = 0) - if self._comm.Get_rank() == 0: - x = self._comm.recv(source=target) - y = self._comm.recv(source=target) -# if self._comm.Get_rank() == target: #assuming rootrank -# request = self._comm.Isend(x, dest=0) #as request object, use wait method on it later on, can lower case this for picking any python object -# request = self._comm.Isend(y, dest=0) - # if self._comm.Get_rank() == 0: -# x = self._comm.recv(source=target) -# y = self._comm.recv(source=target) - - + if self._comm.Get_rank() == target: + packet = numpy.concatenate((x, y)) + requestXY = self._comm.isend(packet, dest=0) + requestXY.Wait() + if self._comm.Get_rank() == 0: + packet = self._comm.recv(source=target) + x,y = numpy.array_split(packet, 2) return x, y, index -#assume has knowledge of index of data -#find out which rank has the data assoc with the index -#specifically query that rank to get the data From 628b1a4bda9688327a5e956f1dc8eb338cbf09c2 Mon Sep 17 00:00:00 2001 From: Lottie Greenwood Date: Tue, 10 May 2016 11:22:36 +0200 Subject: [PATCH 22/22] added resultpublisher base class --- backend/__init__.py | 2 +- backend/histogram_publisher.py | 64 ++++++++++++++++++++++++++++++++++ backend/result_publisher.py | 56 ++++++++++------------------- backend/result_publisherM.py | 46 ------------------------ ports.py | 2 +- run_backend_reducer.py | 12 +++---- 6 files changed, 91 insertions(+), 91 deletions(-) create mode 100644 backend/histogram_publisher.py delete mode 100644 backend/result_publisherM.py diff --git a/backend/__init__.py b/backend/__init__.py index b753697..e1dc38c 100644 --- a/backend/__init__.py +++ b/backend/__init__.py @@ -1,6 +1,6 @@ from backend_event_listener import BackendEventListener from backend_mantid_reducer import BackendMantidReducer -from result_publisher import ResultPublisher +from histogram_publisher import HistogramPublisher from instrumentview_publisher import InstrumentViewPublisher from spectraview_publisher import SpectraViewPublisher from zmq_queue import ZMQQueueServer diff --git a/backend/histogram_publisher.py b/backend/histogram_publisher.py new file mode 100644 index 0000000..84a5d92 --- /dev/null +++ b/backend/histogram_publisher.py @@ -0,0 +1,64 @@ +import time +import numpy +import zmq + +from logger import log +import ports +from result_publisher import ResultPublisher + + +class HistogramPublisher(ResultPublisher): + def __init__(self, eventListener): + super(ResultPublisher, self).__init__(type(self).__name__) + self.eventListener = eventListener + self._update_rate = 1.0 + self.socket = None + self._last_count = 0 + + def run(self): + log.info("Starting ResultPublisher") + self.connect() + + self._publish_clear() + while True: + count = len(self.eventListener._gather_histogram_transition.get_checkpoint()) + if count != self._last_count: + self._publish_clear() + self._last_count = count + for i in range(count): + if self.eventListener._gather_histogram_transition.get_checkpoint()[i]: + self._publish(i) + time.sleep(self.update_rate) + + def connect(self): + context = zmq.Context() + self.socket = context.socket(zmq.PUB) + uri = 'tcp://*:{0:d}'.format(ports.result_stream) + self.socket.bind(uri) + log.info('Bound to ' + uri) + + def _create_header(self, command, index): + return { 'command':command, 'index':index } + + def _publish_clear(self): + header = self._create_header('clear', None) + self.socket.send_json(header) + + def _publish(self, index): + boundaries, values = self.eventListener._gather_histogram_transition.get_checkpoint()[index].data + packet = numpy.concatenate((boundaries, values)) +# print "this boundaries " +str(boundaries) + header = self._create_header('graphData', index) + self.socket.send_json(header, flags=zmq.SNDMORE) + self.socket.send(packet) + + def get_parameter_dict(self): + return {'update_rate':'float'} + + @property + def update_rate(self): + return self._update_rate + + @update_rate.setter + def update_rate(self, update_rate): + self._update_rate = update_rate diff --git a/backend/result_publisher.py b/backend/result_publisher.py index a347499..31a91f6 100644 --- a/backend/result_publisher.py +++ b/backend/result_publisher.py @@ -1,4 +1,3 @@ -import time import numpy import zmq @@ -6,54 +5,37 @@ import ports from controllable import Controllable - class ResultPublisher(Controllable): def __init__(self, eventListener): - super(ResultPublisher, self).__init__(type(self).__name__) - self.eventListener = eventListener - self._update_rate = 1.0 - self.socket = None - self._last_count = 0 + super(ResultPublisher, self).__init__(type(self).__name__) + self.eventListener = eventListener + self._update_rate = 1.0 + self.socket = None + self._last_count = 0 + self.ports = None def run(self): - log.info("Starting ResultPublisher") - self.connect() - - self._publish_clear() - while True: - count = len(self.eventListener._gather_histogram_transition.get_checkpoint()) - if count != self._last_count: - self._publish_clear() - self._last_count = count - for i in range(count): - if self.eventListener._gather_histogram_transition.get_checkpoint()[i]: - self._publish(i) - time.sleep(self.update_rate) + pass def connect(self): - context = zmq.Context() - self.socket = context.socket(zmq.PUB) - uri = 'tcp://*:{0:d}'.format(ports.result_stream) - self.socket.bind(uri) - log.info('Bound to ' + uri) + context = zmq.Context() + self.socket = context.socket(zmq.PUB) + uri = 'tcp://*:{0:d}'.format(self.ports) + self.socket.bind(uri) + log.info('Bound to ' + uri) - def _create_header(self, command, index): - return { 'command':command, 'index':index } + def _create_header(self,command,index): + return {'command':command, 'index':index} def _publish_clear(self): - header = self._create_header('clear', None) - self.socket.send_json(header) + header = self._create_header('clear', None) + self.socket.send_json(header) def _publish(self, index): - boundaries, values = self.eventListener._gather_histogram_transition.get_checkpoint()[index].data - packet = numpy.concatenate((boundaries, values)) -# print "this boundaries " +str(boundaries) - header = self._create_header('graphData', index) - self.socket.send_json(header, flags=zmq.SNDMORE) - self.socket.send(packet) + pass def get_parameter_dict(self): - return {'update_rate':'float'} + return {'update_rate':'float'} @property def update_rate(self): @@ -61,4 +43,4 @@ def update_rate(self): @update_rate.setter def update_rate(self, update_rate): - self._update_rate = update_rate + self._update_rate = update_rate diff --git a/backend/result_publisherM.py b/backend/result_publisherM.py deleted file mode 100644 index 31a91f6..0000000 --- a/backend/result_publisherM.py +++ /dev/null @@ -1,46 +0,0 @@ -import numpy -import zmq - -from logger import log -import ports -from controllable import Controllable - -class ResultPublisher(Controllable): - def __init__(self, eventListener): - super(ResultPublisher, self).__init__(type(self).__name__) - self.eventListener = eventListener - self._update_rate = 1.0 - self.socket = None - self._last_count = 0 - self.ports = None - - def run(self): - pass - - def connect(self): - context = zmq.Context() - self.socket = context.socket(zmq.PUB) - uri = 'tcp://*:{0:d}'.format(self.ports) - self.socket.bind(uri) - log.info('Bound to ' + uri) - - def _create_header(self,command,index): - return {'command':command, 'index':index} - - def _publish_clear(self): - header = self._create_header('clear', None) - self.socket.send_json(header) - - def _publish(self, index): - pass - - def get_parameter_dict(self): - return {'update_rate':'float'} - - @property - def update_rate(self): - return self._update_rate - - @update_rate.setter - def update_rate(self, update_rate): - self._update_rate = update_rate diff --git a/ports.py b/ports.py index d218f36..dc71da6 100644 --- a/ports.py +++ b/ports.py @@ -1,7 +1,7 @@ event_stream = 10001 streamer_control = 10002 result_stream = 10003 -result_publisher_control = 10004 +histogram_publisher_control = 10004 rebin_control = 10005 instrumentview_result_stream = 10006 spectra_result_stream = 10007 diff --git a/run_backend_reducer.py b/run_backend_reducer.py index ea6f6d4..551ba43 100644 --- a/run_backend_reducer.py +++ b/run_backend_reducer.py @@ -1,6 +1,6 @@ from logger import setup_global_logger from backend import BackendMantidReducer -from backend import ResultPublisher +from backend import HistogramPublisher from backend import InstrumentViewPublisher from backend import SpectraViewPublisher from backend import ZMQQueueServer @@ -38,17 +38,17 @@ reducer_controller_thread = threading.Thread(target=reducer_controller.run) reducer_controller_thread.start() - resultPublisher = ResultPublisher(reducer) - resultPublisher_thread = threading.Thread(target=resultPublisher.run) - resultPublisher_thread.start() + histogramPublisher = HistogramPublisher(reducer) + histogramPublisher_thread = threading.Thread(target=histogramPublisher.run) + histogramPublisher_thread.start() instrumentViewPublisher = InstrumentViewPublisher(reducer) instrumentViewPublisher_thread = threading.Thread(target=instrumentViewPublisher.run) instrumentViewPublisher_thread.start() spectraViewPublisher = SpectraViewPublisher(reducer) spectraViewPublisher_thread = threading.Thread(target=spectraViewPublisher.run) spectraViewPublisher_thread.start() - parameterController = ParameterControlServer(controllees=[instrumentViewPublisher], port=ports.result_publisher_control) - parameterController = ParameterControlServer(controllees=[resultPublisher], port=ports.result_publisher_control) + parameterController = ParameterControlServer(controllees=[instrumentViewPublisher], port=ports.histogram_publisher_control) + parameterController = ParameterControlServer(controllees=[histogramPublisher], port=ports.histogram_publisher_control) parameterController_thread = threading.Thread(target=parameterController.run) parameterController_thread.start()