From a72fbbe39190d848fd050adb2cbae1c234237aa2 Mon Sep 17 00:00:00 2001 From: Spandan Naik Date: Wed, 25 Jun 2025 16:36:46 +0000 Subject: [PATCH 1/4] Adding Integ test for NeuronCore Utilization Bug Fix --- .../resources/dummy-neuron-monitor/Dockerfile | 1 + .../dummy_neuron_monitor.py | 602 +- .../neuron-monitor-output.json | 28166 ++++++++++++++++ terraform/eks/daemon/awsneuron/main.tf | 2 +- test/awsneuron/neuron_metrics_test.go | 1 + test/metric/container_insights_util.go | 47 + util/awsservice/cloudwatchlogs.go | 35 + 7 files changed, 28255 insertions(+), 599 deletions(-) create mode 100644 docs/resources/dummy-neuron-monitor/neuron-monitor-output.json diff --git a/docs/resources/dummy-neuron-monitor/Dockerfile b/docs/resources/dummy-neuron-monitor/Dockerfile index 47b06ba3..decbc7cb 100644 --- a/docs/resources/dummy-neuron-monitor/Dockerfile +++ b/docs/resources/dummy-neuron-monitor/Dockerfile @@ -27,6 +27,7 @@ RUN apt-get update \ && rm -rf /tmp/tmp* \ && apt-get clean +COPY neuron-monitor-output.json /opt/aws/neuron/bin/neuron-monitor-output.json COPY dummy_neuron_monitor.py /opt/aws/neuron/bin/dummy_neuron_monitor.py RUN chmod 755 /opt/aws/neuron/bin/dummy_neuron_monitor.py RUN pip3 install prometheus_client boto3 requests \ No newline at end of file diff --git a/docs/resources/dummy-neuron-monitor/dummy_neuron_monitor.py b/docs/resources/dummy-neuron-monitor/dummy_neuron_monitor.py index 22ccf7e7..73ab4dae 100644 --- a/docs/resources/dummy-neuron-monitor/dummy_neuron_monitor.py +++ b/docs/resources/dummy-neuron-monitor/dummy_neuron_monitor.py @@ -252,607 +252,13 @@ def signal_handler(*_): original_file_hash = {"certfile":certfile_hash,"keyfile":keyfile_hash} while running: - line = ('{"neuron_runtime_data":[{"pid":457402,"neuron_runtime_tag":"367","error":"","report":{' - '"execution_stats":{"period":4.999666547,"error_summary":{"generic":2,"numerical":0,"transient":0,' - '"model":0,"runtime":0,"hardware":0},"execution_summary":{"completed":2,"completed_with_err":0,' - '"completed_with_num_err":0,"timed_out":0,"incorrect_input":0,"failed_to_queue":0},"latency_stats":{' - '"total_latency":null,"device_latency":null},"error":""},"memory_used":{"period":4.999671285,' - '"neuron_runtime_used_bytes":{"host":9043968,"neuron_device":3541303936,"usage_breakdown":{"host":{' - '"application_memory":655360,"constants":0,"dma_buffers":8388608,"tensors":0},' - '"neuroncore_memory_usage":{"0":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"1":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"2":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"3":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852},"4":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"5":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"6":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"7":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852},"8":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"9":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"10":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"11":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852},"12":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"13":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"14":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"15":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852},"16":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"17":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"18":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"19":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852},"20":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"21":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"22":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"23":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852},"24":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"25":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"26":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"27":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852},"28":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"29":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"30":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"31":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852}}}},"loaded_models":[{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10019,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":5}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10005,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":5}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10007,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":10}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10013,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":10}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10029,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":2}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10032,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":2}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10004,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":0}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10012,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":0}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10001,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":3}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10016,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":3}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10022,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":11}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10024,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":11}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10026,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":13}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10031,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":13}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10025,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":15}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10021,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":15}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10006,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":12}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10011,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":12}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10010,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":6}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10015,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":6}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10008,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":1}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10027,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":1}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10018,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":14}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10030,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":14}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10017,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":9}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10003,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":9}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10002,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":4}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10009,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":4}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10020,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":7}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10028,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":7}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10014,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":8}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10023,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":8}}}],"error":""},"neuron_runtime_vcpu_usage":{"period":4.999647382,' - '"vcpu_usage":{"user":0,"system":0},"error":"open/proc/457402/stat:nosuchfileordirectory"},' - '"neuroncore_counters":{"period":4.999667932,"neuroncores_in_use":{"0":{"neuroncore_utilization":0},' - '"1":{"neuroncore_utilization":0},"2":{"neuroncore_utilization":0},"3":{"neuroncore_utilization":0},' - '"4":{"neuroncore_utilization":0},"5":{"neuroncore_utilization":0},"6":{"neuroncore_utilization": 41.4},' - '"7":{"neuroncore_utilization":0},"8":{"neuroncore_utilization":0},"9":{"neuroncore_utilization":0},' - '"10":{"neuroncore_utilization":0},"11":{"neuroncore_utilization":0},' - '"12":{"neuroncore_utilization":0},"13":{"neuroncore_utilization":0},' - '"14":{"neuroncore_utilization":0},"15":{"neuroncore_utilization":0},' - '"16":{"neuroncore_utilization":0},"17":{"neuroncore_utilization":0},' - '"18":{"neuroncore_utilization":0},"19":{"neuroncore_utilization":0},' - '"20":{"neuroncore_utilization":0},"21":{"neuroncore_utilization":0},' - '"22":{"neuroncore_utilization":0},"23":{"neuroncore_utilization":0},' - '"24":{"neuroncore_utilization":0},"25":{"neuroncore_utilization":0},' - '"26":{"neuroncore_utilization":0},"27":{"neuroncore_utilization":0},' - '"28":{"neuroncore_utilization":0},"29":{"neuroncore_utilization":0},' - '"30":{"neuroncore_utilization":0},"31":{"neuroncore_utilization":0}},"error":""}}}, {"pid":457402,"neuron_runtime_tag":"123","error":"","report":{' - '"execution_stats":{"period":4.999666547,"error_summary":{"generic":2,"numerical":0,"transient":0,' - '"model":0,"runtime":0,"hardware":0},"execution_summary":{"completed":2,"completed_with_err":0,' - '"completed_with_num_err":0,"timed_out":0,"incorrect_input":0,"failed_to_queue":0},"latency_stats":{' - '"total_latency":null,"device_latency":null},"error":""},"memory_used":{"period":4.999671285,' - '"neuron_runtime_used_bytes":{"host":9043968,"neuron_device":3541303936,"usage_breakdown":{"host":{' - '"application_memory":655360,"constants":0,"dma_buffers":8388608,"tensors":0},' - '"neuroncore_memory_usage":{"0":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"1":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"2":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"3":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852},"4":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"5":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"6":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"7":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852},"8":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"9":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"10":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"11":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852},"12":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"13":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"14":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"15":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852},"16":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"17":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"18":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"19":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852},"20":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"21":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"22":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"23":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852},"24":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"25":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"26":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"27":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852},"28":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,' - '"runtime_memory":0,"tensors":9912852},"29":{"constants":0,"model_code":100752896,' - '"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},"30":{"constants":0,' - '"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,"tensors":9912852},' - '"31":{"constants":0,"model_code":100752896,"model_shared_scratchpad":0,"runtime_memory":0,' - '"tensors":9912852}}}},"loaded_models":[{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10019,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":5}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10005,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":5}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10007,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":10}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10013,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":10}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10029,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":2}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10032,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":2}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10004,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":0}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10012,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":0}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10001,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":3}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10016,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":3}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10022,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":11}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10024,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":11}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10026,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":13}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10031,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":13}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10025,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":15}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10021,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":15}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10006,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":12}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10011,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":12}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10010,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":6}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10015,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":6}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10008,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":1}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10027,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":1}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10018,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":14}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10030,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":14}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10017,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":9}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10003,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":9}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10002,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":4}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10009,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":4}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10020,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":7}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10028,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":7}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10014,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":1,' - '"neuron_device_index":8}}},{"name":"2.2.0.73+0af5a171c-/neuronxcc-y_d15g2g",' - '"uuid":"302e6ea0c77b11ee846d26b89fc7ffab","model_id":10023,"is_running":false,"subgraphs":{"sg_00":{' - '"memory_used_bytes":{"host":20480,"neuron_device":24064,"usage_breakdown":{"host":{' - '"application_memory":20480,"constants":0,"dma_buffers":0,"tensors":0},"neuron_device":{' - '"constants":0,"model_code":24064,"runtime_memory":0,"tensors":0}}},"neuroncore_index":0,' - '"neuron_device_index":8}}}],"error":""},"neuron_runtime_vcpu_usage":{"period":4.999647382,' - '"vcpu_usage":{"user":0,"system":0},"error":"open/proc/457402/stat:nosuchfileordirectory"},' - '"neuroncore_counters":{"period":4.999667932,"neuroncores_in_use":{"0":{"neuroncore_utilization":0},' - '"1":{"neuroncore_utilization":0},"2":{"neuroncore_utilization":0},"3":{"neuroncore_utilization":0},' - '"4":{"neuroncore_utilization":0},"5":{"neuroncore_utilization":0},"6":{"neuroncore_utilization":0},' - '"7":{"neuroncore_utilization":0},"8":{"neuroncore_utilization":0},"9":{"neuroncore_utilization":0},' - '"10":{"neuroncore_utilization":0},"11":{"neuroncore_utilization":0},' - '"12":{"neuroncore_utilization":0},"13":{"neuroncore_utilization":0},' - '"14":{"neuroncore_utilization":0},"15":{"neuroncore_utilization":0},' - '"16":{"neuroncore_utilization":0},"17":{"neuroncore_utilization":0},' - '"18":{"neuroncore_utilization":0},"19":{"neuroncore_utilization":0},' - '"20":{"neuroncore_utilization":0},"21":{"neuroncore_utilization":0},' - '"22":{"neuroncore_utilization":0},"23":{"neuroncore_utilization":0},' - '"24":{"neuroncore_utilization":0},"25":{"neuroncore_utilization":0},' - '"26":{"neuroncore_utilization":0},"27":{"neuroncore_utilization":0},' - '"28":{"neuroncore_utilization":0},"29":{"neuroncore_utilization":0},' - '"30":{"neuroncore_utilization":0},"31":{"neuroncore_utilization":0}},"error":""}}}],"system_data":{' - '"memory_info":{"period":4.9997283150000005,"memory_total_bytes":532523487232,' - '"memory_used_bytes":81207975936,"swap_total_bytes":0,"swap_used_bytes":0,"error":""},"vcpu_usage":{' - '"period":4.999737702,"average_usage":{"user":19.66,"nice":0,"system":1.67,"idle":78.67,"io_wait":0,' - '"irq":0,"soft_irq":0},"usage_data":{"0":{"user":51.31,"nice":0,"system":0,"idle":48.69,"io_wait":0,' - '"irq":0,"soft_irq":0},"1":{"user":52.91,"nice":0,"system":6.21,"idle":40.88,"io_wait":0,"irq":0,' - '"soft_irq":0},"2":{"user":25.6,"nice":0,"system":0,"idle":74.4,"io_wait":0,"irq":0,"soft_irq":0},' - '"3":{"user":0.6,"nice":0,"system":0,"idle":99.4,"io_wait":0,"irq":0,"soft_irq":0},"4":{"user":0.2,' - '"nice":0,"system":0,"idle":99.8,"io_wait":0,"irq":0,"soft_irq":0},"5":{"user":0.8,"nice":0,' - '"system":0,"idle":99.2,"io_wait":0,"irq":0,"soft_irq":0},"6":{"user":1,"nice":0,"system":2.99,' - '"idle":96.01,"io_wait":0,"irq":0,"soft_irq":0},"7":{"user":0,"nice":0,"system":0.2,"idle":99.8,' - '"io_wait":0,"irq":0,"soft_irq":0},"8":{"user":1.8,"nice":0,"system":0,"idle":98.2,"io_wait":0,' - '"irq":0,"soft_irq":0},"9":{"user":0.4,"nice":0,"system":0.8,"idle":98.8,"io_wait":0,"irq":0,' - '"soft_irq":0},"10":{"user":0.2,"nice":0,"system":0,"idle":99.8,"io_wait":0,"irq":0,"soft_irq":0},' - '"11":{"user":0,"nice":0,"system":0,"idle":100,"io_wait":0,"irq":0,"soft_irq":0},"12":{"user":0,' - '"nice":0,"system":0,"idle":100,"io_wait":0,"irq":0,"soft_irq":0},"13":{"user":0,"nice":0,"system":0,' - '"idle":100,"io_wait":0,"irq":0,"soft_irq":0},"14":{"user":0,"nice":0,"system":0.2,"idle":99.8,' - '"io_wait":0,"irq":0,"soft_irq":0},"15":{"user":0.2,"nice":0,"system":0,"idle":99.8,"io_wait":0,' - '"irq":0,"soft_irq":0},"16":{"user":0.2,"nice":0,"system":0,"idle":99.8,"io_wait":0,"irq":0,' - '"soft_irq":0},"17":{"user":0.2,"nice":0,"system":0.2,"idle":99.6,"io_wait":0,"irq":0,"soft_irq":0},' - '"18":{"user":0,"nice":0,"system":0,"idle":100,"io_wait":0,"irq":0,"soft_irq":0},"19":{"user":0,' - '"nice":0,"system":0,"idle":100,"io_wait":0,"irq":0,"soft_irq":0},"20":{"user":0.4,"nice":0,' - '"system":0.4,"idle":99.2,"io_wait":0,"irq":0,"soft_irq":0},"21":{"user":0.2,"nice":0,"system":0.2,' - '"idle":99.6,"io_wait":0,"irq":0,"soft_irq":0},"22":{"user":0.2,"nice":0,"system":0,"idle":99.8,' - '"io_wait":0,"irq":0,"soft_irq":0},"23":{"user":0,"nice":0,"system":0.2,"idle":99.8,"io_wait":0,' - '"irq":0,"soft_irq":0},"24":{"user":0.2,"nice":0,"system":0.2,"idle":99.6,"io_wait":0,"irq":0,' - '"soft_irq":0},"25":{"user":0,"nice":0,"system":0,"idle":100,"io_wait":0,"irq":0,"soft_irq":0},' - '"26":{"user":0.2,"nice":0,"system":0.2,"idle":99.6,"io_wait":0,"irq":0,"soft_irq":0},' - '"27":{"user":0.2,"nice":0,"system":0.4,"idle":99.4,"io_wait":0,"irq":0,"soft_irq":0},' - '"28":{"user":0.2,"nice":0,"system":0.8,"idle":99,"io_wait":0,"irq":0,"soft_irq":0},"29":{"user":0.2,' - '"nice":0,"system":0,"idle":99.8,"io_wait":0,"irq":0,"soft_irq":0},"30":{"user":0,"nice":0,' - '"system":0.4,"idle":99.6,"io_wait":0,"irq":0,"soft_irq":0},"31":{"user":0,"nice":0,"system":0,' - '"idle":100,"io_wait":0,"irq":0,"soft_irq":0},"32":{"user":0.2,"nice":0,"system":1,"idle":98.8,' - '"io_wait":0,"irq":0,"soft_irq":0},"33":{"user":1.81,"nice":0,"system":4.44,"idle":93.75,"io_wait":0,' - '"irq":0,"soft_irq":0},"34":{"user":5.22,"nice":0,"system":0.2,"idle":94.58,"io_wait":0,"irq":0,' - '"soft_irq":0},"35":{"user":22,"nice":0,"system":0,"idle":78,"io_wait":0,"irq":0,"soft_irq":0},' - '"36":{"user":47.31,"nice":0,"system":2.79,"idle":49.9,"io_wait":0,"irq":0,"soft_irq":0},' - '"37":{"user":0.2,"nice":0,"system":0.2,"idle":99.6,"io_wait":0,"irq":0,"soft_irq":0},' - '"38":{"user":5.2,"nice":0,"system":3.8,"idle":91,"io_wait":0,"irq":0,"soft_irq":0},' - '"39":{"user":72.69,"nice":0,"system":5.42,"idle":21.89,"io_wait":0,"irq":0,"soft_irq":0},' - '"40":{"user":75.85,"nice":0,"system":2.4,"idle":21.76,"io_wait":0,"irq":0,"soft_irq":0},' - '"41":{"user":1,"nice":0,"system":3.6,"idle":95.4,"io_wait":0,"irq":0,"soft_irq":0},' - '"42":{"user":4.58,"nice":0,"system":3.78,"idle":91.63,"io_wait":0,"irq":0,"soft_irq":0},' - '"43":{"user":7.62,"nice":0,"system":5.21,"idle":87.17,"io_wait":0,"irq":0,"soft_irq":0},' - '"44":{"user":6.22,"nice":0,"system":2.81,"idle":90.96,"io_wait":0,"irq":0,"soft_irq":0},' - '"45":{"user":1.81,"nice":0,"system":4.62,"idle":93.57,"io_wait":0,"irq":0,"soft_irq":0},' - '"46":{"user":1.8,"nice":0,"system":6.21,"idle":91.98,"io_wait":0,"irq":0,"soft_irq":0},' - '"47":{"user":1.6,"nice":0,"system":5,"idle":93.4,"io_wait":0,"irq":0,"soft_irq":0},"48":{"user":0,' - '"nice":0,"system":0.2,"idle":99.8,"io_wait":0,"irq":0,"soft_irq":0},"49":{"user":0.2,"nice":0,' - '"system":0.2,"idle":99.6,"io_wait":0,"irq":0,"soft_irq":0},"50":{"user":0,"nice":0,"system":1.4,' - '"idle":98.6,"io_wait":0,"irq":0,"soft_irq":0},"51":{"user":76.15,"nice":0,"system":2.81,' - '"idle":21.04,"io_wait":0,"irq":0,"soft_irq":0},"52":{"user":8.2,"nice":0,"system":3.4,"idle":88.4,' - '"io_wait":0,"irq":0,"soft_irq":0},"53":{"user":8.62,"nice":0,"system":3.61,"idle":87.78,"io_wait":0,' - '"irq":0,"soft_irq":0},"54":{"user":7.62,"nice":0,"system":1,"idle":91.38,"io_wait":0,"irq":0,' - '"soft_irq":0},"55":{"user":75.3,"nice":0,"system":0.6,"idle":24.1,"io_wait":0,"irq":0,"soft_irq":0},' - '"56":{"user":0,"nice":0,"system":0,"idle":100,"io_wait":0,"irq":0,"soft_irq":0},"57":{"user":0,' - '"nice":0,"system":0,"idle":100,"io_wait":0,"irq":0,"soft_irq":0},"58":{"user":0,"nice":0,"system":0,' - '"idle":100,"io_wait":0,"irq":0,"soft_irq":0},"59":{"user":75.2,"nice":0,"system":0.6,"idle":24.2,' - '"io_wait":0,"irq":0,"soft_irq":0},"60":{"user":70.46,"nice":0,"system":0,"idle":29.54,"io_wait":0,' - '"irq":0,"soft_irq":0},"61":{"user":70.34,"nice":0,"system":0,"idle":29.66,"io_wait":0,"irq":0,' - '"soft_irq":0},"62":{"user":72.8,"nice":0,"system":0,"idle":27.2,"io_wait":0,"irq":0,"soft_irq":0},' - '"63":{"user":73.2,"nice":0,"system":3,"idle":23.8,"io_wait":0,"irq":0,"soft_irq":0},' - '"64":{"user":19.8,"nice":0,"system":1,"idle":79.2,"io_wait":0,"irq":0,"soft_irq":0},' - '"65":{"user":0.8,"nice":0,"system":0,"idle":99.2,"io_wait":0,"irq":0,"soft_irq":0},"66":{"user":0.2,' - '"nice":0,"system":0,"idle":99.8,"io_wait":0,"irq":0,"soft_irq":0},"67":{"user":0.2,"nice":0,' - '"system":0,"idle":99.8,"io_wait":0,"irq":0,"soft_irq":0},"68":{"user":6.24,"nice":0,"system":1.61,' - '"idle":92.15,"io_wait":0,"irq":0,"soft_irq":0},"69":{"user":0.2,"nice":0,"system":0,"idle":99.8,' - '"io_wait":0,"irq":0,"soft_irq":0},"70":{"user":0.6,"nice":0,"system":2.59,"idle":96.81,"io_wait":0,' - '"irq":0,"soft_irq":0},"71":{"user":2.79,"nice":0,"system":5.38,"idle":91.83,"io_wait":0,"irq":0,' - '"soft_irq":0},"72":{"user":1.6,"nice":0,"system":6.01,"idle":92.38,"io_wait":0,"irq":0,' - '"soft_irq":0},"73":{"user":0.2,"nice":0,"system":0.2,"idle":99.6,"io_wait":0,"irq":0,"soft_irq":0},' - '"74":{"user":0.2,"nice":0,"system":0,"idle":99.8,"io_wait":0,"irq":0,"soft_irq":0},"75":{"user":0.2,' - '"nice":0,"system":0,"idle":99.8,"io_wait":0,"irq":0,"soft_irq":0},"76":{"user":0.2,"nice":0,' - '"system":0.2,"idle":99.6,"io_wait":0,"irq":0,"soft_irq":0},"77":{"user":2.2,"nice":0,"system":5.4,' - '"idle":92.4,"io_wait":0,"irq":0,"soft_irq":0},"78":{"user":1.4,"nice":0,"system":5.01,"idle":93.59,' - '"io_wait":0,"irq":0,"soft_irq":0},"79":{"user":0.4,"nice":0,"system":0,"idle":99.6,"io_wait":0,' - '"irq":0,"soft_irq":0},"80":{"user":0,"nice":0,"system":0.4,"idle":99.6,"io_wait":0,"irq":0,' - '"soft_irq":0},"81":{"user":2,"nice":0,"system":5.59,"idle":92.42,"io_wait":0,"irq":0,"soft_irq":0},' - '"82":{"user":2.6,"nice":0,"system":6.4,"idle":91,"io_wait":0,"irq":0,"soft_irq":0},' - '"83":{"user":2.79,"nice":0,"system":6.37,"idle":90.84,"io_wait":0,"irq":0,"soft_irq":0},' - '"84":{"user":2.2,"nice":0,"system":5.2,"idle":92.6,"io_wait":0,"irq":0,"soft_irq":0},' - '"85":{"user":0.2,"nice":0,"system":1.4,"idle":98.4,"io_wait":0,"irq":0,"soft_irq":0},' - '"86":{"user":2.4,"nice":0,"system":5.21,"idle":92.38,"io_wait":0,"irq":0,"soft_irq":0},' - '"87":{"user":2.4,"nice":0,"system":6.61,"idle":90.98,"io_wait":0,"irq":0,"soft_irq":0},' - '"88":{"user":2.2,"nice":0,"system":6.61,"idle":91.18,"io_wait":0,"irq":0,"soft_irq":0},' - '"89":{"user":0,"nice":0,"system":0.4,"idle":99.6,"io_wait":0,"irq":0,"soft_irq":0},"90":{"user":2.4,' - '"nice":0,"system":5,"idle":92.6,"io_wait":0,"irq":0,"soft_irq":0},"91":{"user":0.2,"nice":0,' - '"system":0,"idle":99.8,"io_wait":0,"irq":0,"soft_irq":0},"92":{"user":0,"nice":0,"system":0.2,' - '"idle":99.8,"io_wait":0,"irq":0,"soft_irq":0},"93":{"user":0,"nice":0,"system":0,"idle":100,' - '"io_wait":0,"irq":0,"soft_irq":0},"94":{"user":0,"nice":0,"system":0.2,"idle":99.8,"io_wait":0,' - '"irq":0,"soft_irq":0},"95":{"user":2,"nice":0,"system":5.19,"idle":92.81,"io_wait":0,"irq":0,' - '"soft_irq":0},"96":{"user":76.2,"nice":0,"system":6.8,"idle":17,"io_wait":0,"irq":0,"soft_irq":0},' - '"97":{"user":75.15,"nice":0,"system":1.6,"idle":23.25,"io_wait":0,"irq":0,"soft_irq":0},' - '"98":{"user":1.4,"nice":0,"system":4.81,"idle":93.79,"io_wait":0,"irq":0,"soft_irq":0},' - '"99":{"user":1.39,"nice":0,"system":4.78,"idle":93.82,"io_wait":0,"irq":0,"soft_irq":0},' - '"100":{"user":2.19,"nice":0,"system":5.38,"idle":92.43,"io_wait":0,"irq":0,"soft_irq":0},' - '"101":{"user":75.05,"nice":0,"system":0.6,"idle":24.35,"io_wait":0,"irq":0,"soft_irq":0},' - '"102":{"user":77.4,"nice":0,"system":3.8,"idle":18.8,"io_wait":0,"irq":0,"soft_irq":0},' - '"103":{"user":0,"nice":0,"system":0,"idle":100,"io_wait":0,"irq":0,"soft_irq":0},"104":{"user":0,' - '"nice":0,"system":0,"idle":100,"io_wait":0,"irq":0,"soft_irq":0},"105":{"user":74.95,"nice":0,' - '"system":1.2,"idle":23.85,"io_wait":0,"irq":0,"soft_irq":0},"106":{"user":76.8,"nice":0,' - '"system":0.4,"idle":22.8,"io_wait":0,"irq":0,"soft_irq":0},"107":{"user":76.95,"nice":0,' - '"system":1.2,"idle":21.84,"io_wait":0,"irq":0,"soft_irq":0},"108":{"user":78.71,"nice":0,' - '"system":7.43,"idle":13.86,"io_wait":0,"irq":0,"soft_irq":0},"109":{"user":75.05,"nice":0,' - '"system":0.4,"idle":24.55,"io_wait":0,"irq":0,"soft_irq":0},"110":{"user":75.15,"nice":0,' - '"system":0.4,"idle":24.45,"io_wait":0,"irq":0,"soft_irq":0},"111":{"user":75.15,"nice":0,' - '"system":0.6,"idle":24.25,"io_wait":0,"irq":0,"soft_irq":0},"112":{"user":75.15,"nice":0,' - '"system":0.6,"idle":24.25,"io_wait":0,"irq":0,"soft_irq":0},"113":{"user":74.85,"nice":0,' - '"system":1.2,"idle":23.95,"io_wait":0,"irq":0,"soft_irq":0},"114":{"user":74.85,"nice":0,"system":1,' - '"idle":24.15,"io_wait":0,"irq":0,"soft_irq":0},"115":{"user":0,"nice":0,"system":0,"idle":100,' - '"io_wait":0,"irq":0,"soft_irq":0},"116":{"user":77.84,"nice":0,"system":0.8,"idle":21.36,' - '"io_wait":0,"irq":0,"soft_irq":0},"117":{"user":78.2,"nice":0,"system":0.4,"idle":21.4,"io_wait":0,' - '"irq":0,"soft_irq":0},"118":{"user":77.8,"nice":0,"system":1,"idle":21.2,"io_wait":0,"irq":0,' - '"soft_irq":0},"119":{"user":0.4,"nice":0,"system":0,"idle":99.6,"io_wait":0,"irq":0,"soft_irq":0},' - '"120":{"user":75.2,"nice":0,"system":0.4,"idle":24.4,"io_wait":0,"irq":0,"soft_irq":0},' - '"121":{"user":75.15,"nice":0,"system":0.6,"idle":24.25,"io_wait":0,"irq":0,"soft_irq":0},' - '"122":{"user":74.8,"nice":0,"system":1,"idle":24,"io_wait":0,"irq":0,"soft_irq":0.2},' - '"123":{"user":0.2,"nice":0,"system":0,"idle":99.8,"io_wait":0,"irq":0,"soft_irq":0},"124":{"user":0,' - '"nice":0,"system":0.2,"idle":99.8,"io_wait":0,"irq":0,"soft_irq":0},"125":{"user":0.2,"nice":0,' - '"system":0,"idle":99.8,"io_wait":0,"irq":0,"soft_irq":0},"126":{"user":0,"nice":0,"system":0.2,' - '"idle":99.8,"io_wait":0,"irq":0,"soft_irq":0},"127":{"user":1.2,"nice":0,"system":3,"idle":95.8,' - '"io_wait":0,"irq":0,"soft_irq":0}},"context_switch_count":171386,"error":""},"neuron_hw_counters":{' - '"period":1.000142057,"neuron_devices":[{"neuron_device_index":0,"mem_ecc_corrected":1,' - '"mem_ecc_uncorrected":0,"sram_ecc_uncorrected":0,"sram_ecc_corrected":0},{"neuron_device_index":1,' - '"mem_ecc_corrected":0,"mem_ecc_uncorrected":0,"sram_ecc_uncorrected":0,"sram_ecc_corrected":0},' - '{"neuron_device_index":2,"mem_ecc_corrected":1,"mem_ecc_uncorrected":0,"sram_ecc_uncorrected":0,' - '"sram_ecc_corrected":0},{"neuron_device_index":3,"mem_ecc_corrected":2,"mem_ecc_uncorrected":0,' - '"sram_ecc_uncorrected":0,"sram_ecc_corrected":0},{"neuron_device_index":4,"mem_ecc_corrected":0,' - '"mem_ecc_uncorrected":0,"sram_ecc_uncorrected":0,"sram_ecc_corrected":0},{"neuron_device_index":5,' - '"mem_ecc_corrected":1,"mem_ecc_uncorrected":0,"sram_ecc_uncorrected":0,"sram_ecc_corrected":0},' - '{"neuron_device_index":6,"mem_ecc_corrected":0,"mem_ecc_uncorrected":1,"sram_ecc_uncorrected":0,' - '"sram_ecc_corrected":0},{"neuron_device_index":7,"mem_ecc_corrected":0,"mem_ecc_uncorrected":0,' - '"sram_ecc_uncorrected":1,"sram_ecc_corrected":0},{"neuron_device_index":8,"mem_ecc_corrected":0,' - '"mem_ecc_uncorrected":0,"sram_ecc_uncorrected":0,"sram_ecc_corrected":1},{"neuron_device_index":9,' - '"mem_ecc_corrected":0,"mem_ecc_uncorrected":1,"sram_ecc_uncorrected":0,"sram_ecc_corrected":0},' - '{"neuron_device_index":10,"mem_ecc_corrected":0,"mem_ecc_uncorrected":0,"sram_ecc_uncorrected":0,' - '"sram_ecc_corrected":0},{"neuron_device_index":11,"mem_ecc_corrected":0,"mem_ecc_uncorrected":0,' - '"sram_ecc_uncorrected":0,"sram_ecc_corrected":0},{"neuron_device_index":12,"mem_ecc_corrected":0,' - '"mem_ecc_uncorrected":0,"sram_ecc_uncorrected":1,"sram_ecc_corrected":0},{"neuron_device_index":13,' - '"mem_ecc_corrected":0,"mem_ecc_uncorrected":0,"sram_ecc_uncorrected":0,"sram_ecc_corrected":1},' - '{"neuron_device_index":14,"mem_ecc_corrected":0,"mem_ecc_uncorrected":1,"sram_ecc_uncorrected":1,' - '"sram_ecc_corrected":0},{"neuron_device_index":15,"mem_ecc_corrected":0,"mem_ecc_uncorrected":1,' - '"sram_ecc_uncorrected":0,"sram_ecc_corrected":0}],"error":""}},"instance_info":{' - '"instance_name":"DummyNodeName",' - '"instance_id":"i-09db9b55e0095612f","instance_type":"trn1n.32xlarge",' - '"instance_availability_zone":"us-east-1c","instance_availability_zone_id":"use1-az6",' - '"instance_region":"us-east-1","ami_id":"ami-030686a4e905e98d3",' - '"subnet_id":"subnet-06a7754948e8a000f","error":""},"neuron_hardware_info":{"neuron_device_count":16,' - '"neuroncore_per_device_count":2,"error":""}}') - if len(line) == 0: - continue if original_file_hash: _watch_file_and_update_ssl_cxt(original_file_hash, certfile=certfile, keyfile=keyfile) try: - monitor_data = json.loads(line) + monitor_data = {} + with open('/opt/aws/neuron/bin/neuron-monitor-output.json', 'r') as file: + json_data = json.load(file) + monitor_data = json_data["1"] except Exception as exc: print('Unable to decode JSON {}'.format(exc)) continue diff --git a/docs/resources/dummy-neuron-monitor/neuron-monitor-output.json b/docs/resources/dummy-neuron-monitor/neuron-monitor-output.json new file mode 100644 index 00000000..7cd2b8af --- /dev/null +++ b/docs/resources/dummy-neuron-monitor/neuron-monitor-output.json @@ -0,0 +1,28166 @@ +{ + "1": { + "neuron_runtime_data": [ + { + "pid": 784315, + "neuron_runtime_tag": "535", + "error": "", + "report": { + "execution_stats": { + "period": 0.007951199, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.007955515, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534166027, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "0": { + "constants": 1007473416, + "model_code": 767175872, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "1": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ccf89b50-20ca-47dd-ab0c-2c83ebd32782/model.MODULE_949767491799162040+c7db2df4.neff", + "uuid": "4a0f3929d5364365907772b4dc505064", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7822084, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7822080, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/faff9412-ee9c-489b-8c9f-2e7231c57ba5/model.MODULE_10131425082988973763+c7db2df4.neff", + "uuid": "91bd9b84410e4bd9a7ab2225cce8f0a7", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 28003136, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27937600, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8f28bd9c-7368-4ace-bfa2-575ee8ef7bf4/model.MODULE_624408017509569431+c7db2df4.neff", + "uuid": "519b4ea069584cd489772c390b840f62", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27313668, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27248128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 0 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.007952473, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784315/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.00796773, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784318, + "neuron_runtime_tag": "538", + "error": "", + "report": { + "execution_stats": { + "period": 0.005847536, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.00587363, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9532596107, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "0": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "1": { + "constants": 1007473416, + "model_code": 765605952, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 0 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.005173345, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784318/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.005908631, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 1, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784319, + "neuron_runtime_tag": "539", + "error": "", + "report": { + "execution_stats": { + "period": 0.010269021, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.010306091, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534054283, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "2": { + "constants": 1007473416, + "model_code": 767064128, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "3": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 1 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.010206474, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784319/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.010259128, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 2, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784321, + "neuron_runtime_tag": "541", + "error": "", + "report": { + "execution_stats": { + "period": 0.00938813, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.00939062, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9532724107, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "2": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "3": { + "constants": 1007473416, + "model_code": 765733952, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 1 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.009399908, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784321/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.009376896, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 3, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784323, + "neuron_runtime_tag": "543", + "error": "", + "report": { + "execution_stats": { + "period": 0.007558421, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.007566375, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534060427, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "4": { + "constants": 1007473416, + "model_code": 767070272, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "5": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 2 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.007616398, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784323/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.007753573, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 4, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784324, + "neuron_runtime_tag": "544", + "error": "", + "report": { + "execution_stats": { + "period": 0.004883886, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.004895039, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9549358987, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "4": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "5": { + "constants": 1024250632, + "model_code": 765591616, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 2 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.004899835, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784324/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.004879433, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 5, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784325, + "neuron_runtime_tag": "545", + "error": "", + "report": { + "execution_stats": { + "period": 0.01413949, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.01417754, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534060427, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "6": { + "constants": 1007473416, + "model_code": 767070272, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "7": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 3 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.014205578, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784325/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.014155536, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 6, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784328, + "neuron_runtime_tag": "548", + "error": "", + "report": { + "execution_stats": { + "period": 0.014294063, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.014381727, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9549488011, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "6": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "7": { + "constants": 1024250632, + "model_code": 765720640, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 3 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.014410118, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784328/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.014384148, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 7, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784329, + "neuron_runtime_tag": "549", + "error": "", + "report": { + "execution_stats": { + "period": 0.010111463, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.010136266, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9549599755, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "8": { + "constants": 1024250632, + "model_code": 765832384, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "9": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ccf89b50-20ca-47dd-ab0c-2c83ebd32782/model.MODULE_949767491799162040+c7db2df4.neff", + "uuid": "4a0f3929d5364365907772b4dc505064", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7822084, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7822080, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/faff9412-ee9c-489b-8c9f-2e7231c57ba5/model.MODULE_10131425082988973763+c7db2df4.neff", + "uuid": "91bd9b84410e4bd9a7ab2225cce8f0a7", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 28003136, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27937600, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8f28bd9c-7368-4ace-bfa2-575ee8ef7bf4/model.MODULE_624408017509569431+c7db2df4.neff", + "uuid": "519b4ea069584cd489772c390b840f62", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27313668, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27248128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 4 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.010111908, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784329/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.010138449, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 8, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784330, + "neuron_runtime_tag": "550", + "error": "", + "report": { + "execution_stats": { + "period": 0.006087428, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.006123003, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534060427, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "8": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "9": { + "constants": 1007473416, + "model_code": 767070272, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 4 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.005131034, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784330/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.006194258, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 9, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784333, + "neuron_runtime_tag": "553", + "error": "", + "report": { + "execution_stats": { + "period": 0.008302207, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.008521502, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9549358987, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "10": { + "constants": 1024250632, + "model_code": 765591616, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "11": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 5 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.00823582, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784333/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.008592634, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 10, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784334, + "neuron_runtime_tag": "554", + "error": "", + "report": { + "execution_stats": { + "period": 0.010435697, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.010963099, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534060427, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "10": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "11": { + "constants": 1007473416, + "model_code": 767070272, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 5 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.01046205, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784334/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.010824863, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 11, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784335, + "neuron_runtime_tag": "555", + "error": "", + "report": { + "execution_stats": { + "period": 0.010068644, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.010074572, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9532724107, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "12": { + "constants": 1007473416, + "model_code": 765733952, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "13": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 6 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.009855462, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784335/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.010077468, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 12, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784338, + "neuron_runtime_tag": "558", + "error": "", + "report": { + "execution_stats": { + "period": 0.008129331, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.008132716, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534054283, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "12": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "13": { + "constants": 1007473416, + "model_code": 767064128, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 6 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.008065739, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784338/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.008135758, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 13, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784339, + "neuron_runtime_tag": "559", + "error": "", + "report": { + "execution_stats": { + "period": 0.008254164, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.008258892, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9532596107, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "14": { + "constants": 1007473416, + "model_code": 765605952, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "15": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 7 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.008166499, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784339/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.008269464, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 14, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784340, + "neuron_runtime_tag": "560", + "error": "", + "report": { + "execution_stats": { + "period": 0.010446477, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.010440191, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534054283, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "14": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "15": { + "constants": 1007473416, + "model_code": 767064128, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 7 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.010487281, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784340/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.010453421, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 15, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784342, + "neuron_runtime_tag": "562", + "error": "", + "report": { + "execution_stats": { + "period": 0.008179872, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.008181648, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9549470731, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "16": { + "constants": 1024250632, + "model_code": 765703360, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "17": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8f28bd9c-7368-4ace-bfa2-575ee8ef7bf4/model.MODULE_624408017509569431+c7db2df4.neff", + "uuid": "519b4ea069584cd489772c390b840f62", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27313668, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27248128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ccf89b50-20ca-47dd-ab0c-2c83ebd32782/model.MODULE_949767491799162040+c7db2df4.neff", + "uuid": "4a0f3929d5364365907772b4dc505064", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7822084, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7822080, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/faff9412-ee9c-489b-8c9f-2e7231c57ba5/model.MODULE_10131425082988973763+c7db2df4.neff", + "uuid": "91bd9b84410e4bd9a7ab2225cce8f0a7", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 28003136, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27937600, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 8 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.008182225, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784342/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.008183381, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 16, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784344, + "neuron_runtime_tag": "564", + "error": "", + "report": { + "execution_stats": { + "period": 0.00991838, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.009938192, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534060427, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "16": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "17": { + "constants": 1007473416, + "model_code": 767070272, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 8 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.009918149, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784344/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.009968444, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 17, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784345, + "neuron_runtime_tag": "565", + "error": "", + "report": { + "execution_stats": { + "period": 0.012694878, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.012695781, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9549488011, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "18": { + "constants": 1024250632, + "model_code": 765720640, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "19": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 9 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.012755498, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784345/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.012721635, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 18, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784347, + "neuron_runtime_tag": "567", + "error": "", + "report": { + "execution_stats": { + "period": 0.008155696, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.008163837, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534060427, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "18": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "19": { + "constants": 1007473416, + "model_code": 767070272, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 9 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.008185894, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784347/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.008148421, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 19, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784349, + "neuron_runtime_tag": "569", + "error": "", + "report": { + "execution_stats": { + "period": 0.014116873, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.014252603, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9532596107, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "20": { + "constants": 1007473416, + "model_code": 765605952, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "21": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 10 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.014171425, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784349/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.014178395, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 20, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784350, + "neuron_runtime_tag": "570", + "error": "", + "report": { + "execution_stats": { + "period": 0.008734573, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.008731275, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534054283, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "20": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "21": { + "constants": 1007473416, + "model_code": 767064128, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 10 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.008671648, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784350/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.008685416, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 21, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784353, + "neuron_runtime_tag": "573", + "error": "", + "report": { + "execution_stats": { + "period": 0.007808881, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.007192795, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9532724107, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "22": { + "constants": 1007473416, + "model_code": 765733952, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "23": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 11 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.007626478, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784353/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.007775212, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 22, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784354, + "neuron_runtime_tag": "574", + "error": "", + "report": { + "execution_stats": { + "period": 0.010079358, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.010066605, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534054283, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "22": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "23": { + "constants": 1007473416, + "model_code": 767064128, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 11 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.009683313, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784354/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.01005129, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 23, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784355, + "neuron_runtime_tag": "575", + "error": "", + "report": { + "execution_stats": { + "period": 0.00787167, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.00766832, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534166027, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "24": { + "constants": 1007473416, + "model_code": 767175872, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "25": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/faff9412-ee9c-489b-8c9f-2e7231c57ba5/model.MODULE_10131425082988973763+c7db2df4.neff", + "uuid": "91bd9b84410e4bd9a7ab2225cce8f0a7", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 28003136, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27937600, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ccf89b50-20ca-47dd-ab0c-2c83ebd32782/model.MODULE_949767491799162040+c7db2df4.neff", + "uuid": "4a0f3929d5364365907772b4dc505064", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7822084, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7822080, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8f28bd9c-7368-4ace-bfa2-575ee8ef7bf4/model.MODULE_624408017509569431+c7db2df4.neff", + "uuid": "519b4ea069584cd489772c390b840f62", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27313668, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27248128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 12 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.007697022, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784355/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.007756478, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 24, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784358, + "neuron_runtime_tag": "578", + "error": "", + "report": { + "execution_stats": { + "period": 0.010015775, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.00984638, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9532724107, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "24": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "25": { + "constants": 1007473416, + "model_code": 765733952, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 12 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.010095425, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784358/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.010067691, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 25, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784359, + "neuron_runtime_tag": "579", + "error": "", + "report": { + "execution_stats": { + "period": 0.008194223, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.008216282, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534054283, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "26": { + "constants": 1007473416, + "model_code": 767064128, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "27": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 13 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.008131361, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784359/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.008202071, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 26, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784360, + "neuron_runtime_tag": "580", + "error": "", + "report": { + "execution_stats": { + "period": 0.011242371, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.011359077, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9532596107, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "26": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "27": { + "constants": 1007473416, + "model_code": 765605952, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 13 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.011239344, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784360/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.011368491, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 27, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784363, + "neuron_runtime_tag": "583", + "error": "", + "report": { + "execution_stats": { + "period": 0.008013407, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.008014578, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534060427, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "28": { + "constants": 1007473416, + "model_code": 767070272, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "29": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 14 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.008040184, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784363/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.008090202, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 28, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784364, + "neuron_runtime_tag": "584", + "error": "", + "report": { + "execution_stats": { + "period": 0.008756984, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.008765672, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9549488011, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "28": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "29": { + "constants": 1024250632, + "model_code": 765720640, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 14 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.008751198, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784364/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.00874738, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 29, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784366, + "neuron_runtime_tag": "586", + "error": "", + "report": { + "execution_stats": { + "period": 0.011305847, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.011078307, + "neuron_runtime_used_bytes": { + "host": 7191684, + "neuron_device": 9534060427, + "usage_breakdown": { + "host": { + "application_memory": 474244, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "30": { + "constants": 1007473416, + "model_code": 767070272, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + }, + "31": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": true, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 0, + "neuron_device_index": 15 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.011361849, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784366/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.011344389, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 30, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + }, + { + "pid": 784368, + "neuron_runtime_tag": "588", + "error": "", + "report": { + "execution_stats": { + "period": 0.009573865, + "error_summary": { + "generic": 0, + "numerical": 0, + "transient": 0, + "model": 0, + "runtime": 0, + "hardware": 0 + }, + "execution_summary": { + "completed": 0, + "completed_with_err": 0, + "completed_with_num_err": 0, + "timed_out": 0, + "incorrect_input": 0, + "failed_to_queue": 0 + }, + "latency_stats": { + "total_latency": null, + "device_latency": null + }, + "error": "" + }, + "memory_used": { + "period": 0.009647114, + "neuron_runtime_used_bytes": { + "host": 7224452, + "neuron_device": 9549358987, + "usage_breakdown": { + "host": { + "application_memory": 507012, + "constants": 0, + "dma_buffers": 6717440, + "tensors": 0 + }, + "neuroncore_memory_usage": { + "30": { + "constants": 0, + "model_code": 0, + "model_shared_scratchpad": 0, + "runtime_memory": 0, + "tensors": 0 + }, + "31": { + "constants": 1024250632, + "model_code": 765591616, + "model_shared_scratchpad": 1073742080, + "runtime_memory": 1088, + "tensors": 6685773571 + } + } + } + }, + "loaded_models": [ + { + "name": "/tmp/neuroncc_compile_workdir/ad87d1b9-d981-46ca-ac7d-da426fa303c2/model.MODULE_8779599195420024690+c7db2df4.neff", + "uuid": "13f9f3e19a8d47379c2dc022dfd6b607", + "model_id": 10007, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 33088, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 33088, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/3379feee-78e0-4e01-b382-3882473b8475/model.MODULE_855895338905798074+c7db2df4.neff", + "uuid": "497517d38ea045daa2a4fb50ffc5115f", + "model_id": 10009, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 61163072, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 163840, + "model_code": 60999232, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/0a0c7988-51a5-4f65-8759-f8c59e68bedd/model.MODULE_3195886093958684488+c7db2df4.neff", + "uuid": "bc3df6888abe412fa34650ea23c2aa8d", + "model_id": 10012, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 164278912, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 164049536, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/9f0aff00-7e3e-4a3c-8870-d58fef43f310/model.MODULE_1001400978678458940+c7db2df4.neff", + "uuid": "547fe8b6b789431180beecd8ff20a9fd", + "model_id": 10014, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 169212416, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 229376, + "model_code": 168983040, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/cd4224e3-b276-4b70-b853-8b0a3f60eda2/model.MODULE_1017057454460023801+c7db2df4.neff", + "uuid": "71bf7d89bf8e44f7b01a9e4b5ce105d9", + "model_id": 10016, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 7747716, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 4, + "model_code": 7747712, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/b99b09e8-d2a9-4be2-807f-2a047991fbc1/model.MODULE_11673304993061181424+c7db2df4.neff", + "uuid": "088f86ade1a24d2ebc77d4ef384f9d4b", + "model_id": 10020, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27993280, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 27927744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "neff", + "uuid": "c94aae5c116749b5ae35b78bb7e17fad", + "model_id": 10001, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 26432, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 26176, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/69645bcc-7d1d-4e80-95b5-1cfc215ee300/model.MODULE_11046142791348916433+c7db2df4.neff", + "uuid": "fe31dd2f6e7849748c6028032f78f7d3", + "model_id": 10006, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 54784, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 54784, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/36022a0c-3a54-4e79-97c2-cc07ff0fc51e/model.MODULE_509369269099815655+c7db2df4.neff", + "uuid": "fbdf37f96ace4d23aaeec8a400c67d68", + "model_id": 10017, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 27299972, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 65540, + "model_code": 27234432, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f7200a6e-d9ea-459a-9678-d0b6e6ed2bbc/model.MODULE_3962016099826517868+c7db2df4.neff", + "uuid": "4ab99ea7c8364b9bbb573e0c40ada23a", + "model_id": 10018, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31744, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31744, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/a4415e5e-77a7-4d77-b14b-4ef12ba8f750/model.MODULE_14657009528611371363+c7db2df4.neff", + "uuid": "e78543316c0945058e4b3bbbea2f646c", + "model_id": 10004, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/598d76c6-1538-4910-ab8f-ca932a4f967d/model.MODULE_15885530732822698094+c7db2df4.neff", + "uuid": "88c787dd202e4a5f91a5d79cb4904bc4", + "model_id": 10010, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/94ca98c2-7979-416f-9c2c-3a0ebfc94ac1/model.MODULE_10373091242957097222+c7db2df4.neff", + "uuid": "b38ed1e8367f4b888a0bbebc22b57de1", + "model_id": 10008, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/6b3b87d8-a022-4eef-bc6d-ccccc261d9d3/model.MODULE_13320881343942844190+c7db2df4.neff", + "uuid": "cd409b2ed0f743efbf84c2bcf5dd7590", + "model_id": 10011, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 8462272, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 8388608, + "model_code": 73664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/e8846f72-37f6-45f6-913f-c5525eef6f8b/model.MODULE_17527677555922634669+c7db2df4.neff", + "uuid": "9eac925e5b4a41feaf5a0c34604ef24f", + "model_id": 10013, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 24128, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 24128, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f4008a4e-7783-4a66-8300-9980accf3f0a/model.MODULE_13734722552091714043+c7db2df4.neff", + "uuid": "2f15e7be2d754d7e984f1ce6fdea1b5b", + "model_id": 10015, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 273472, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 65536, + "model_code": 207936, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/8632cd7e-2c90-4f65-aed7-57384eb78713/model.MODULE_8668685380708903824+c7db2df4.neff", + "uuid": "2fc8480a37124291a8b5f240bd19bdf0", + "model_id": 10019, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 282624, + "neuron_device": 4471680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 262144, + "tensors": 0 + }, + "neuron_device": { + "constants": 16384, + "model_code": 4455296, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f79df32c-6edd-4eef-9620-c98949fe0237/model.MODULE_13435372550914585992+c7db2df4.neff", + "uuid": "a37fc03d56544980944bb1e97cc9dfb7", + "model_id": 10002, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 544768, + "neuron_device": 31680, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 524288, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 31680, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/f6d2879c-002d-456b-abbd-4b8f84a64c96/model.MODULE_8400150938565577423+c7db2df4.neff", + "uuid": "df8bfb04367047fb852481d31ce0b944", + "model_id": 10003, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + }, + { + "name": "/tmp/neuroncc_compile_workdir/bdbdba23-4a50-4f04-8f22-f62d25eb6337/model.MODULE_10907652914944028636+c7db2df4.neff", + "uuid": "09aeaddeaa924c83a1c0c77860b8fb04", + "model_id": 10005, + "is_running": false, + "subgraphs": { + "sg_00": { + "memory_used_bytes": { + "host": 53248, + "neuron_device": 121664, + "usage_breakdown": { + "host": { + "application_memory": 20480, + "constants": 0, + "dma_buffers": 32768, + "tensors": 0 + }, + "neuron_device": { + "constants": 0, + "model_code": 121664, + "runtime_memory": 0, + "tensors": 0 + } + } + }, + "neuroncore_index": 1, + "neuron_device_index": 15 + } + } + } + ], + "error": "" + }, + "neuron_runtime_vcpu_usage": { + "period": 0.009671582, + "vcpu_usage": { + "user": 0, + "system": 0 + }, + "error": "open /proc/784368/stat: no such file or directory" + }, + "neuroncore_counters": { + "period": 0.009843913, + "neuroncores_in_use": { + "0": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "1": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "10": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "11": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "12": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "13": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "14": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "15": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "16": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "17": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "18": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "19": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "2": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "20": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "21": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "22": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "23": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "24": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "25": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "26": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "27": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "28": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "29": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "3": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "30": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "31": { + "neuroncore_utilization": 31, + "flops": 0, + "effective_flops": 0 + }, + "4": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "5": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "6": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "7": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "8": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + }, + "9": { + "neuroncore_utilization": 0, + "flops": 0, + "effective_flops": 0 + } + }, + "error": "" + } + } + } + ], + "system_data": { + "memory_info": { + "period": 4.9997283150000005, + "memory_total_bytes": 532523487232, + "memory_used_bytes": 81207975936, + "swap_total_bytes": 0, + "swap_used_bytes": 0, + "error": "" + }, + "vcpu_usage": { + "period": 4.999737702, + "average_usage": { + "user": 19.66, + "nice": 0, + "system": 1.67, + "idle": 78.67, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "usage_data": { + "0": { + "user": 51.31, + "nice": 0, + "system": 0, + "idle": 48.69, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "1": { + "user": 52.91, + "nice": 0, + "system": 6.21, + "idle": 40.88, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "2": { + "user": 25.6, + "nice": 0, + "system": 0, + "idle": 74.4, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "3": { + "user": 0.6, + "nice": 0, + "system": 0, + "idle": 99.4, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "4": { + "user": 0.2, + "nice": 0, + "system": 0, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "5": { + "user": 0.8, + "nice": 0, + "system": 0, + "idle": 99.2, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "6": { + "user": 1, + "nice": 0, + "system": 2.99, + "idle": 96.01, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "7": { + "user": 0, + "nice": 0, + "system": 0.2, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "8": { + "user": 1.8, + "nice": 0, + "system": 0, + "idle": 98.2, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "9": { + "user": 0.4, + "nice": 0, + "system": 0.8, + "idle": 98.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "10": { + "user": 0.2, + "nice": 0, + "system": 0, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "11": { + "user": 0, + "nice": 0, + "system": 0, + "idle": 100, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "12": { + "user": 0, + "nice": 0, + "system": 0, + "idle": 100, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "13": { + "user": 0, + "nice": 0, + "system": 0, + "idle": 100, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "14": { + "user": 0, + "nice": 0, + "system": 0.2, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "15": { + "user": 0.2, + "nice": 0, + "system": 0, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "16": { + "user": 0.2, + "nice": 0, + "system": 0, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "17": { + "user": 0.2, + "nice": 0, + "system": 0.2, + "idle": 99.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "18": { + "user": 0, + "nice": 0, + "system": 0, + "idle": 100, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "19": { + "user": 0, + "nice": 0, + "system": 0, + "idle": 100, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "20": { + "user": 0.4, + "nice": 0, + "system": 0.4, + "idle": 99.2, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "21": { + "user": 0.2, + "nice": 0, + "system": 0.2, + "idle": 99.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "22": { + "user": 0.2, + "nice": 0, + "system": 0, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "23": { + "user": 0, + "nice": 0, + "system": 0.2, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "24": { + "user": 0.2, + "nice": 0, + "system": 0.2, + "idle": 99.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "25": { + "user": 0, + "nice": 0, + "system": 0, + "idle": 100, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "26": { + "user": 0.2, + "nice": 0, + "system": 0.2, + "idle": 99.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "27": { + "user": 0.2, + "nice": 0, + "system": 0.4, + "idle": 99.4, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "28": { + "user": 0.2, + "nice": 0, + "system": 0.8, + "idle": 99, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "29": { + "user": 0.2, + "nice": 0, + "system": 0, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "30": { + "user": 0, + "nice": 0, + "system": 0.4, + "idle": 99.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "31": { + "user": 0, + "nice": 0, + "system": 0, + "idle": 100, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "32": { + "user": 0.2, + "nice": 0, + "system": 1, + "idle": 98.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "33": { + "user": 1.81, + "nice": 0, + "system": 4.44, + "idle": 93.75, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "34": { + "user": 5.22, + "nice": 0, + "system": 0.2, + "idle": 94.58, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "35": { + "user": 22, + "nice": 0, + "system": 0, + "idle": 78, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "36": { + "user": 47.31, + "nice": 0, + "system": 2.79, + "idle": 49.9, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "37": { + "user": 0.2, + "nice": 0, + "system": 0.2, + "idle": 99.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "38": { + "user": 5.2, + "nice": 0, + "system": 3.8, + "idle": 91, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "39": { + "user": 72.69, + "nice": 0, + "system": 5.42, + "idle": 21.89, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "40": { + "user": 75.85, + "nice": 0, + "system": 2.4, + "idle": 21.76, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "41": { + "user": 1, + "nice": 0, + "system": 3.6, + "idle": 95.4, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "42": { + "user": 4.58, + "nice": 0, + "system": 3.78, + "idle": 91.63, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "43": { + "user": 7.62, + "nice": 0, + "system": 5.21, + "idle": 87.17, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "44": { + "user": 6.22, + "nice": 0, + "system": 2.81, + "idle": 90.96, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "45": { + "user": 1.81, + "nice": 0, + "system": 4.62, + "idle": 93.57, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "46": { + "user": 1.8, + "nice": 0, + "system": 6.21, + "idle": 91.98, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "47": { + "user": 1.6, + "nice": 0, + "system": 5, + "idle": 93.4, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "48": { + "user": 0, + "nice": 0, + "system": 0.2, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "49": { + "user": 0.2, + "nice": 0, + "system": 0.2, + "idle": 99.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "50": { + "user": 0, + "nice": 0, + "system": 1.4, + "idle": 98.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "51": { + "user": 76.15, + "nice": 0, + "system": 2.81, + "idle": 21.04, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "52": { + "user": 8.2, + "nice": 0, + "system": 3.4, + "idle": 88.4, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "53": { + "user": 8.62, + "nice": 0, + "system": 3.61, + "idle": 87.78, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "54": { + "user": 7.62, + "nice": 0, + "system": 1, + "idle": 91.38, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "55": { + "user": 75.3, + "nice": 0, + "system": 0.6, + "idle": 24.1, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "56": { + "user": 0, + "nice": 0, + "system": 0, + "idle": 100, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "57": { + "user": 0, + "nice": 0, + "system": 0, + "idle": 100, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "58": { + "user": 0, + "nice": 0, + "system": 0, + "idle": 100, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "59": { + "user": 75.2, + "nice": 0, + "system": 0.6, + "idle": 24.2, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "60": { + "user": 70.46, + "nice": 0, + "system": 0, + "idle": 29.54, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "61": { + "user": 70.34, + "nice": 0, + "system": 0, + "idle": 29.66, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "62": { + "user": 72.8, + "nice": 0, + "system": 0, + "idle": 27.2, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "63": { + "user": 73.2, + "nice": 0, + "system": 3, + "idle": 23.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "64": { + "user": 19.8, + "nice": 0, + "system": 1, + "idle": 79.2, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "65": { + "user": 0.8, + "nice": 0, + "system": 0, + "idle": 99.2, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "66": { + "user": 0.2, + "nice": 0, + "system": 0, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "67": { + "user": 0.2, + "nice": 0, + "system": 0, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "68": { + "user": 6.24, + "nice": 0, + "system": 1.61, + "idle": 92.15, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "69": { + "user": 0.2, + "nice": 0, + "system": 0, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "70": { + "user": 0.6, + "nice": 0, + "system": 2.59, + "idle": 96.81, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "71": { + "user": 2.79, + "nice": 0, + "system": 5.38, + "idle": 91.83, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "72": { + "user": 1.6, + "nice": 0, + "system": 6.01, + "idle": 92.38, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "73": { + "user": 0.2, + "nice": 0, + "system": 0.2, + "idle": 99.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "74": { + "user": 0.2, + "nice": 0, + "system": 0, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "75": { + "user": 0.2, + "nice": 0, + "system": 0, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "76": { + "user": 0.2, + "nice": 0, + "system": 0.2, + "idle": 99.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "77": { + "user": 2.2, + "nice": 0, + "system": 5.4, + "idle": 92.4, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "78": { + "user": 1.4, + "nice": 0, + "system": 5.01, + "idle": 93.59, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "79": { + "user": 0.4, + "nice": 0, + "system": 0, + "idle": 99.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "80": { + "user": 0, + "nice": 0, + "system": 0.4, + "idle": 99.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "81": { + "user": 2, + "nice": 0, + "system": 5.59, + "idle": 92.42, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "82": { + "user": 2.6, + "nice": 0, + "system": 6.4, + "idle": 91, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "83": { + "user": 2.79, + "nice": 0, + "system": 6.37, + "idle": 90.84, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "84": { + "user": 2.2, + "nice": 0, + "system": 5.2, + "idle": 92.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "85": { + "user": 0.2, + "nice": 0, + "system": 1.4, + "idle": 98.4, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "86": { + "user": 2.4, + "nice": 0, + "system": 5.21, + "idle": 92.38, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "87": { + "user": 2.4, + "nice": 0, + "system": 6.61, + "idle": 90.98, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "88": { + "user": 2.2, + "nice": 0, + "system": 6.61, + "idle": 91.18, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "89": { + "user": 0, + "nice": 0, + "system": 0.4, + "idle": 99.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "90": { + "user": 2.4, + "nice": 0, + "system": 5, + "idle": 92.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "91": { + "user": 0.2, + "nice": 0, + "system": 0, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "92": { + "user": 0, + "nice": 0, + "system": 0.2, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "93": { + "user": 0, + "nice": 0, + "system": 0, + "idle": 100, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "94": { + "user": 0, + "nice": 0, + "system": 0.2, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "95": { + "user": 2, + "nice": 0, + "system": 5.19, + "idle": 92.81, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "96": { + "user": 76.2, + "nice": 0, + "system": 6.8, + "idle": 17, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "97": { + "user": 75.15, + "nice": 0, + "system": 1.6, + "idle": 23.25, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "98": { + "user": 1.4, + "nice": 0, + "system": 4.81, + "idle": 93.79, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "99": { + "user": 1.39, + "nice": 0, + "system": 4.78, + "idle": 93.82, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "100": { + "user": 2.19, + "nice": 0, + "system": 5.38, + "idle": 92.43, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "101": { + "user": 75.05, + "nice": 0, + "system": 0.6, + "idle": 24.35, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "102": { + "user": 77.4, + "nice": 0, + "system": 3.8, + "idle": 18.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "103": { + "user": 0, + "nice": 0, + "system": 0, + "idle": 100, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "104": { + "user": 0, + "nice": 0, + "system": 0, + "idle": 100, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "105": { + "user": 74.95, + "nice": 0, + "system": 1.2, + "idle": 23.85, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "106": { + "user": 76.8, + "nice": 0, + "system": 0.4, + "idle": 22.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "107": { + "user": 76.95, + "nice": 0, + "system": 1.2, + "idle": 21.84, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "108": { + "user": 78.71, + "nice": 0, + "system": 7.43, + "idle": 13.86, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "109": { + "user": 75.05, + "nice": 0, + "system": 0.4, + "idle": 24.55, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "110": { + "user": 75.15, + "nice": 0, + "system": 0.4, + "idle": 24.45, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "111": { + "user": 75.15, + "nice": 0, + "system": 0.6, + "idle": 24.25, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "112": { + "user": 75.15, + "nice": 0, + "system": 0.6, + "idle": 24.25, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "113": { + "user": 74.85, + "nice": 0, + "system": 1.2, + "idle": 23.95, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "114": { + "user": 74.85, + "nice": 0, + "system": 1, + "idle": 24.15, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "115": { + "user": 0, + "nice": 0, + "system": 0, + "idle": 100, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "116": { + "user": 77.84, + "nice": 0, + "system": 0.8, + "idle": 21.36, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "117": { + "user": 78.2, + "nice": 0, + "system": 0.4, + "idle": 21.4, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "118": { + "user": 77.8, + "nice": 0, + "system": 1, + "idle": 21.2, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "119": { + "user": 0.4, + "nice": 0, + "system": 0, + "idle": 99.6, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "120": { + "user": 75.2, + "nice": 0, + "system": 0.4, + "idle": 24.4, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "121": { + "user": 75.15, + "nice": 0, + "system": 0.6, + "idle": 24.25, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "122": { + "user": 74.8, + "nice": 0, + "system": 1, + "idle": 24, + "io_wait": 0, + "irq": 0, + "soft_irq": 0.2 + }, + "123": { + "user": 0.2, + "nice": 0, + "system": 0, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "124": { + "user": 0, + "nice": 0, + "system": 0.2, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "125": { + "user": 0.2, + "nice": 0, + "system": 0, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "126": { + "user": 0, + "nice": 0, + "system": 0.2, + "idle": 99.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + }, + "127": { + "user": 1.2, + "nice": 0, + "system": 3, + "idle": 95.8, + "io_wait": 0, + "irq": 0, + "soft_irq": 0 + } + }, + "context_switch_count": 171386, + "error": "" + }, + "neuron_hw_counters": { + "period": 1.000142057, + "neuron_devices": [ + { + "neuron_device_index": 0, + "mem_ecc_corrected": 1, + "mem_ecc_uncorrected": 0, + "sram_ecc_uncorrected": 0, + "sram_ecc_corrected": 0 + }, + { + "neuron_device_index": 1, + "mem_ecc_corrected": 0, + "mem_ecc_uncorrected": 0, + "sram_ecc_uncorrected": 0, + "sram_ecc_corrected": 0 + }, + { + "neuron_device_index": 2, + "mem_ecc_corrected": 1, + "mem_ecc_uncorrected": 0, + "sram_ecc_uncorrected": 0, + "sram_ecc_corrected": 0 + }, + { + "neuron_device_index": 3, + "mem_ecc_corrected": 2, + "mem_ecc_uncorrected": 0, + "sram_ecc_uncorrected": 0, + "sram_ecc_corrected": 0 + }, + { + "neuron_device_index": 4, + "mem_ecc_corrected": 0, + "mem_ecc_uncorrected": 0, + "sram_ecc_uncorrected": 0, + "sram_ecc_corrected": 0 + }, + { + "neuron_device_index": 5, + "mem_ecc_corrected": 1, + "mem_ecc_uncorrected": 0, + "sram_ecc_uncorrected": 0, + "sram_ecc_corrected": 0 + }, + { + "neuron_device_index": 6, + "mem_ecc_corrected": 0, + "mem_ecc_uncorrected": 1, + "sram_ecc_uncorrected": 0, + "sram_ecc_corrected": 0 + }, + { + "neuron_device_index": 7, + "mem_ecc_corrected": 0, + "mem_ecc_uncorrected": 0, + "sram_ecc_uncorrected": 1, + "sram_ecc_corrected": 0 + }, + { + "neuron_device_index": 8, + "mem_ecc_corrected": 0, + "mem_ecc_uncorrected": 0, + "sram_ecc_uncorrected": 0, + "sram_ecc_corrected": 1 + }, + { + "neuron_device_index": 9, + "mem_ecc_corrected": 0, + "mem_ecc_uncorrected": 1, + "sram_ecc_uncorrected": 0, + "sram_ecc_corrected": 0 + }, + { + "neuron_device_index": 10, + "mem_ecc_corrected": 0, + "mem_ecc_uncorrected": 0, + "sram_ecc_uncorrected": 0, + "sram_ecc_corrected": 0 + }, + { + "neuron_device_index": 11, + "mem_ecc_corrected": 0, + "mem_ecc_uncorrected": 0, + "sram_ecc_uncorrected": 0, + "sram_ecc_corrected": 0 + }, + { + "neuron_device_index": 12, + "mem_ecc_corrected": 0, + "mem_ecc_uncorrected": 0, + "sram_ecc_uncorrected": 1, + "sram_ecc_corrected": 0 + }, + { + "neuron_device_index": 13, + "mem_ecc_corrected": 0, + "mem_ecc_uncorrected": 0, + "sram_ecc_uncorrected": 0, + "sram_ecc_corrected": 1 + }, + { + "neuron_device_index": 14, + "mem_ecc_corrected": 0, + "mem_ecc_uncorrected": 1, + "sram_ecc_uncorrected": 1, + "sram_ecc_corrected": 0 + }, + { + "neuron_device_index": 15, + "mem_ecc_corrected": 0, + "mem_ecc_uncorrected": 1, + "sram_ecc_uncorrected": 0, + "sram_ecc_corrected": 0 + } + ], + "error": "" + } + }, + "instance_info": { + "instance_name": "DummyNodeName", + "instance_id": "i-09db9b55e0095612f", + "instance_type": "trn1n.32xlarge", + "instance_availability_zone": "us-east-1c", + "instance_availability_zone_id": "use1-az6", + "instance_region": "us-east-1", + "ami_id": "ami-030686a4e905e98d3", + "subnet_id": "subnet-06a7754948e8a000f", + "error": "" + }, + "neuron_hardware_info": { + "neuron_device_count": 16, + "neuroncore_per_device_count": 2, + "error": "" + } + } +} \ No newline at end of file diff --git a/terraform/eks/daemon/awsneuron/main.tf b/terraform/eks/daemon/awsneuron/main.tf index 1fb5383b..e815aa46 100644 --- a/terraform/eks/daemon/awsneuron/main.tf +++ b/terraform/eks/daemon/awsneuron/main.tf @@ -395,7 +395,7 @@ resource "kubernetes_daemonset" "neuron_monitor" { } container { name = "neuron-monitor-prometheus" - image = "506463145083.dkr.ecr.us-west-2.amazonaws.com/mocked-neuron-monitor:v2" + image = "506463145083.dkr.ecr.us-west-2.amazonaws.com/mocked-neuron-monitor:v4" port { container_port = 8000 } diff --git a/test/awsneuron/neuron_metrics_test.go b/test/awsneuron/neuron_metrics_test.go index da9ac577..40377827 100644 --- a/test/awsneuron/neuron_metrics_test.go +++ b/test/awsneuron/neuron_metrics_test.go @@ -40,6 +40,7 @@ func (t *AwsNeuronTestRunner) Validate() status.TestGroupResult { testResults = append(testResults, metric.ValidateMetrics(t.env, awsNeuronMetricIndicator, expectedDimsToMetrics)...) testResults = append(testResults, metric.ValidateLogs(t.env)) testResults = append(testResults, metric.ValidateLogsFrequency(t.env)) + testResults = append(testResults, metric.ValidateNeuronCoreUtilizationValuesLogs(t.env)) return status.TestGroupResult{ Name: t.GetTestName(), TestResults: testResults, diff --git a/test/metric/container_insights_util.go b/test/metric/container_insights_util.go index 1b6669d5..fd1e54f3 100644 --- a/test/metric/container_insights_util.go +++ b/test/metric/container_insights_util.go @@ -10,8 +10,10 @@ import ( "errors" "fmt" "log" + "math" "math/rand" "sort" + "strconv" "strings" "time" @@ -296,3 +298,48 @@ func ValidateLogsFrequency(env *environment.MetaData) status.TestResult { testResult.Status = status.SUCCESSFUL return testResult } + +func ValidateNeuronCoreUtilizationValuesLogs(env *environment.MetaData) status.TestResult { + + testResult := status.TestResult{ + Name: "emf-logs-neuron-core-utilization", + Status: status.FAILED, + } + + end := time.Now().Add(time.Duration(-2) * time.Minute).Truncate(time.Minute) + start := end.Add(time.Duration(-1) * time.Minute) + group := fmt.Sprintf("/aws/containerinsights/%s/performance", env.EKSClusterName) + + // need to get the instances used for the EKS cluster + eKSInstances, err := awsservice.GetEKSInstances(env.EKSClusterName) + if err != nil { + log.Println("failed to get EKS instances", err) + return testResult + } + + for _, instance := range eKSInstances { + stream := *instance.InstanceName + coreMap, err := awsservice.GetNeuronCoreUtilizationPerCore(group, stream, &start, &end) + + if err != nil { + log.Printf("log validation (%s/%s) failed: %v, start time : %s, error is : %s", group, stream, err, start, err) + return testResult + } + + // Check if coreMap has the expected core utilization values + for coreKey, actualValue := range coreMap { + if strings.HasPrefix(coreKey, "core") { + coreNumStr := strings.TrimPrefix(coreKey, "core") + expectedValue, err := strconv.Atoi(coreNumStr) + if err != nil || math.Round(actualValue) != float64(expectedValue) { + log.Printf("Core utilization validation failed: expected %s:%d, got %v", + coreKey, expectedValue, actualValue) + return testResult + } + } + } + } + + testResult.Status = status.SUCCESSFUL + return testResult +} diff --git a/util/awsservice/cloudwatchlogs.go b/util/awsservice/cloudwatchlogs.go index 1882f7e9..58c30bb0 100644 --- a/util/awsservice/cloudwatchlogs.go +++ b/util/awsservice/cloudwatchlogs.go @@ -431,3 +431,38 @@ func CountMetricsInEMFLogs(logGroupName string) (int, error) { return totalMetrics, nil } + +func GetNeuronCoreUtilizationPerCore(logGroup, logStream string, since, until *time.Time) (map[string]float64, error) { + var coreUtilization = make(map[string]float64) + var data map[string]interface{} + + events, err := GetLogsSince(logGroup, logStream, since, until) + + // if there is an error, return the empty map + if err != nil { + return coreUtilization, err + } + + for _, event := range events { + message := *event.Message + + var eksClusterType EKSClusterType + innerErr := json.Unmarshal([]byte(message), &eksClusterType) + if innerErr != nil || !strings.Contains(eksClusterType.Type, "NodeAWSNeuronCore") { + continue + } + + err := json.Unmarshal([]byte(message), &data) + if err != nil { + return coreUtilization, err + } + + if core, ok := data["NeuronCore"].(string); ok { + if util, ok := data["node_neuroncore_utilization"].(float64); ok { + coreUtilization[core] = util + } + } + } + + return coreUtilization, nil +} From 8b02c7efca7083c2e30b6a02163345f596302ff8 Mon Sep 17 00:00:00 2001 From: Spandan Naik Date: Tue, 8 Jul 2025 10:39:27 +0000 Subject: [PATCH 2/4] second revision to resolve comments. --- test/metric/container_insights_util.go | 27 ++++++++++++++++++++------ util/awsservice/cloudwatchlogs.go | 2 +- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/test/metric/container_insights_util.go b/test/metric/container_insights_util.go index fd1e54f3..87277c20 100644 --- a/test/metric/container_insights_util.go +++ b/test/metric/container_insights_util.go @@ -300,14 +300,15 @@ func ValidateLogsFrequency(env *environment.MetaData) status.TestResult { } func ValidateNeuronCoreUtilizationValuesLogs(env *environment.MetaData) status.TestResult { - + const core = "core" testResult := status.TestResult{ Name: "emf-logs-neuron-core-utilization", Status: status.FAILED, } + var testFailed = false - end := time.Now().Add(time.Duration(-2) * time.Minute).Truncate(time.Minute) - start := end.Add(time.Duration(-1) * time.Minute) + end := time.Now().Add(-2 * time.Minute).Truncate(time.Minute) + start := end.Add(-1 * time.Minute) group := fmt.Sprintf("/aws/containerinsights/%s/performance", env.EKSClusterName) // need to get the instances used for the EKS cluster @@ -326,20 +327,34 @@ func ValidateNeuronCoreUtilizationValuesLogs(env *environment.MetaData) status.T return testResult } + // We expect 32 Cores from the current test, anything less or more is a bug + if len(coreMap) != 32 { + log.Printf("32 Cores not found") + var coreMapStr strings.Builder + for k, v := range coreMap { + coreMapStr.WriteString(fmt.Sprintf("%s: %f, ", k, v)) + } + log.Printf("coreMap: %s", coreMapStr.String()) + testFailed = true + } + // Check if coreMap has the expected core utilization values for coreKey, actualValue := range coreMap { - if strings.HasPrefix(coreKey, "core") { - coreNumStr := strings.TrimPrefix(coreKey, "core") + if strings.HasPrefix(coreKey, core) { + coreNumStr := strings.TrimPrefix(coreKey, core) expectedValue, err := strconv.Atoi(coreNumStr) if err != nil || math.Round(actualValue) != float64(expectedValue) { log.Printf("Core utilization validation failed: expected %s:%d, got %v", coreKey, expectedValue, actualValue) - return testResult + testFailed = true } } } } testResult.Status = status.SUCCESSFUL + if testFailed { + testResult.Status = status.FAILED + } return testResult } diff --git a/util/awsservice/cloudwatchlogs.go b/util/awsservice/cloudwatchlogs.go index 58c30bb0..6624e403 100644 --- a/util/awsservice/cloudwatchlogs.go +++ b/util/awsservice/cloudwatchlogs.go @@ -454,7 +454,7 @@ func GetNeuronCoreUtilizationPerCore(logGroup, logStream string, since, until *t err := json.Unmarshal([]byte(message), &data) if err != nil { - return coreUtilization, err + continue } if core, ok := data["NeuronCore"].(string); ok { From 24b66142aa237f209adeb3d8dbcd7951d977cdd9 Mon Sep 17 00:00:00 2001 From: Spandan Naik Date: Tue, 8 Jul 2025 13:26:52 +0000 Subject: [PATCH 3/4] small fixes --- test/metric/container_insights_util.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/metric/container_insights_util.go b/test/metric/container_insights_util.go index 87277c20..3b1e3e61 100644 --- a/test/metric/container_insights_util.go +++ b/test/metric/container_insights_util.go @@ -335,7 +335,7 @@ func ValidateNeuronCoreUtilizationValuesLogs(env *environment.MetaData) status.T coreMapStr.WriteString(fmt.Sprintf("%s: %f, ", k, v)) } log.Printf("coreMap: %s", coreMapStr.String()) - testFailed = true + return testResult } // Check if coreMap has the expected core utilization values @@ -352,9 +352,8 @@ func ValidateNeuronCoreUtilizationValuesLogs(env *environment.MetaData) status.T } } - testResult.Status = status.SUCCESSFUL - if testFailed { - testResult.Status = status.FAILED + if !testFailed { + testResult.Status = status.SUCCESSFUL } return testResult } From b668929acc5d4d1a63d26841a49b9986e366ca8d Mon Sep 17 00:00:00 2001 From: Spandan Naik Date: Tue, 8 Jul 2025 13:30:13 +0000 Subject: [PATCH 4/4] Minor changes cover all edge cases. --- test/metric/container_insights_util.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/metric/container_insights_util.go b/test/metric/container_insights_util.go index 3b1e3e61..1ba1f20b 100644 --- a/test/metric/container_insights_util.go +++ b/test/metric/container_insights_util.go @@ -303,7 +303,7 @@ func ValidateNeuronCoreUtilizationValuesLogs(env *environment.MetaData) status.T const core = "core" testResult := status.TestResult{ Name: "emf-logs-neuron-core-utilization", - Status: status.FAILED, + Status: status.SUCCESSFUL, } var testFailed = false @@ -315,6 +315,7 @@ func ValidateNeuronCoreUtilizationValuesLogs(env *environment.MetaData) status.T eKSInstances, err := awsservice.GetEKSInstances(env.EKSClusterName) if err != nil { log.Println("failed to get EKS instances", err) + testResult.Status = status.FAILED return testResult } @@ -324,6 +325,7 @@ func ValidateNeuronCoreUtilizationValuesLogs(env *environment.MetaData) status.T if err != nil { log.Printf("log validation (%s/%s) failed: %v, start time : %s, error is : %s", group, stream, err, start, err) + testResult.Status = status.FAILED return testResult } @@ -335,6 +337,7 @@ func ValidateNeuronCoreUtilizationValuesLogs(env *environment.MetaData) status.T coreMapStr.WriteString(fmt.Sprintf("%s: %f, ", k, v)) } log.Printf("coreMap: %s", coreMapStr.String()) + testResult.Status = status.FAILED return testResult } @@ -352,8 +355,8 @@ func ValidateNeuronCoreUtilizationValuesLogs(env *environment.MetaData) status.T } } - if !testFailed { - testResult.Status = status.SUCCESSFUL + if testFailed { + testResult.Status = status.FAILED } return testResult }