path: root/python/openvino/runtime/python_demos/OpenVINO_benchmark_app/main.py

# Copyright (C) 2018-2023 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

import os
import sys
from datetime import datetime

from openvino.runtime import Dimension,properties

import benchmark as openvino_benchmark
from openvino.tools.benchmark.parameters import parse_args
from openvino.tools.benchmark.utils.constants import MULTI_DEVICE_NAME, \
    CPU_DEVICE_NAME, GPU_DEVICE_NAME, \
    BIN_EXTENSION, AUTO_DEVICE_NAME
from openvino.tools.benchmark.utils.inputs_filling import get_input_data
from openvino.tools.benchmark.utils.logging import logger
from openvino.tools.benchmark.utils.utils import next_step, get_number_iterations, pre_post_processing, \
    process_help_inference_string, print_perf_counters, print_perf_counters_sort, dump_exec_graph, get_duration_in_milliseconds, \
    get_command_line_arguments, parse_value_per_device, parse_devices, get_inputs_info, \
    print_inputs_and_outputs_info, get_network_batch_size, load_config, dump_config, get_latency_groups, \
    check_for_static, can_measure_as_static, parse_value_for_virtual_device, is_virtual_device, is_virtual_device_found
from openvino.tools.benchmark.utils.statistics_report import StatisticsReport, JsonStatisticsReport, CsvStatisticsReport, \
    averageCntReport, detailedCntReport

def parse_and_check_command_line():
    def arg_not_empty(arg_value,empty_value):
        return not arg_value is None and not arg_value == empty_value

    parser = parse_args()
    args = parser.parse_args()

    if args.latency_percentile < 1 or args.latency_percentile > 100:
        parser.print_help()
        raise RuntimeError("The percentile value is incorrect. The applicable values range is [1, 100].")

    if not args.perf_hint == "none" and (arg_not_empty(args.number_streams, "") or arg_not_empty(args.number_threads, 0) or arg_not_empty(args.infer_threads_pinning, "")):
        raise Exception("-nstreams, -nthreads and -pin options are fine tune options. To use them you " \
                        "should explicitely set -hint option to none. This is not OpenVINO limitation " \
                        "(those options can be used in OpenVINO together), but a benchmark_app UI rule.")

    if args.report_type == "average_counters" and MULTI_DEVICE_NAME in args.target_device:
        raise Exception("only detailed_counters report type is supported for MULTI device")

    if args.number_infer_requests != 1 and "FPGA" in args.target_device:
        logger.warning(f"If the target FPGA design uses JTAG to access the CSRs on the FPGA AI Suite IP "\
                       "(e.g. the Agilex 5E Premium Development Kit JTAG Design Example), "\
                       "then the number of inference request must be 1.")

    _, ext = os.path.splitext(args.path_to_model)
    is_network_compiled = True if ext == BIN_EXTENSION else False

    return args, is_network_compiled

def main():
    statistics = None
    try:
        # ------------------------------ 1. Parsing and validating input arguments ------------------------------
        next_step()
        logger.info("Parsing input parameters")
        args, is_network_compiled = parse_and_check_command_line()

        command_line_arguments = get_command_line_arguments(sys.argv)
        if args.report_type:
            _statistics_class = JsonStatisticsReport if args.json_stats else CsvStatisticsReport
            statistics = _statistics_class(StatisticsReport.Config(args.report_type, args.report_folder))
            statistics.add_parameters(StatisticsReport.Category.COMMAND_LINE_PARAMETERS, command_line_arguments)

        def is_flag_set_in_command_line(flag):
            return any(x.strip('-') == flag for x, y in command_line_arguments)

        device_name = args.target_device

        devices = parse_devices(device_name)
        device_number_streams = parse_value_per_device(devices, args.number_streams, "nstreams")
        device_infer_precision = parse_value_per_device(devices, args.infer_precision, "infer_precision")

        config = {}
        if args.load_config:
            load_config(args.load_config, config)

        if is_network_compiled:
            logger.info("Model is compiled")

        # ------------------------------ 2. Loading OpenVINO Runtime -------------------------------------------
        next_step(step_id=2)

        benchmark = openvino_benchmark.Benchmark(args.target_device, args.number_infer_requests,
                              args.number_iterations, args.time, args.api_type, args.inference_only)

        if args.extensions:
            benchmark.add_extension(path_to_extensions=args.extensions)

        ## GPU (clDNN) Extensions
        if GPU_DEVICE_NAME in device_name and args.path_to_cldnn_config:
            if GPU_DEVICE_NAME not in config.keys():
                config[GPU_DEVICE_NAME] = {}
            config[GPU_DEVICE_NAME]['CONFIG_FILE'] = args.path_to_cldnn_config

        if GPU_DEVICE_NAME in config.keys() and 'CONFIG_FILE' in config[GPU_DEVICE_NAME].keys():
            cldnn_config = config[GPU_DEVICE_NAME]['CONFIG_FILE']
            benchmark.add_extension(path_to_cldnn_config=cldnn_config)

        benchmark.print_version_info()

        # --------------------- 3. Setting device configuration --------------------------------------------------------
        next_step()

        def set_performance_hint(device):
            perf_hint = properties.hint.PerformanceMode.UNDEFINED
            supported_properties = benchmark.core.get_property(device, properties.supported_properties())
            if properties.hint.performance_mode() in supported_properties:
                if is_flag_set_in_command_line('hint'):
                    if args.perf_hint == "throughput" or args.perf_hint == "tput":
                        perf_hint = properties.hint.PerformanceMode.THROUGHPUT
                    elif args.perf_hint == "latency":
                        perf_hint = properties.hint.PerformanceMode.LATENCY
                    elif args.perf_hint == "cumulative_throughput" or args.perf_hint == "ctput":
                        perf_hint = properties.hint.PerformanceMode.CUMULATIVE_THROUGHPUT
                    elif args.perf_hint=='none':
                        perf_hint = properties.hint.PerformanceMode.UNDEFINED
                    else:
                        raise RuntimeError("Incorrect performance hint. Please set -hint option to"
                            "`throughput`(tput), `latency', 'cumulative_throughput'(ctput) value or 'none'.")
                else:
                    perf_hint = properties.hint.PerformanceMode.THROUGHPUT if benchmark.api_type == "async" else properties.hint.PerformanceMode.LATENCY
                    logger.warning(f"Performance hint was not explicitly specified in command line. " +
                    f"Device({device}) performance hint will be set to {perf_hint}.")
                if perf_hint != properties.hint.PerformanceMode.UNDEFINED:
                    config[device][properties.hint.performance_mode()] = perf_hint
            else:
                logger.warning(f"Device {device} does not support performance hint property(-hint).")


        def get_device_type_from_name(name) :
            new_name = str(name)
            new_name = new_name.split(".", 1)[0]
            new_name = new_name.split("(", 1)[0]
            return new_name

        ## Set default values from dumped config
        default_devices = set()
        for device in devices:
            device_type = get_device_type_from_name(device)
            if device_type in config and device not in config:
                config[device] = config[device_type].copy()
                default_devices.add(device_type)

        for def_device in default_devices:
            config.pop(def_device)

        perf_counts = False
        # check if using the virtual device
        hw_devices_list = devices.copy()
        # Remove the hardware devices if AUTO/MULTI/HETERO appears in the devices list.
        is_virtual = is_virtual_device_found(devices)
        if is_virtual:
            devices.clear()
            # Parse out the currect virtual device as the target device.
            virtual_device = device_name.partition(":")[0]
            hw_devices_list.remove(virtual_device)
            devices.append(virtual_device)
            parse_value_for_virtual_device(virtual_device, device_number_streams)
            parse_value_for_virtual_device(virtual_device, device_infer_precision)

        for device in devices:
            supported_properties = benchmark.core.get_property(device, properties.supported_properties())
            if device not in config.keys():
                config[device] = {}
            ## high-level performance modes
            # The orginial OV 2022.3 Python API fails with the pc flag, so we comment it out
            # for both the HETERO and FPGA devices in our patched version of the Python demos
            if device in ['HETERO', 'FPGA']:
                continue
            set_performance_hint(device)

            if is_flag_set_in_command_line('nireq'):
                config[device][properties.hint.num_requests()] = str(args.number_infer_requests)

            ## Set performance counter
            if is_flag_set_in_command_line('pc'):
                ## set to user defined value
                config[device][properties.enable_profiling()] = True if args.perf_counts else False
            elif properties.enable_profiling() in config[device].keys() and config[device][properties.enable_profiling()] == True:
                logger.warning(f"Performance counters for {device} device is turned on. " +
                               "To print results use -pc option.")
            elif args.report_type in [ averageCntReport, detailedCntReport ]:
                logger.warning(f"Turn on performance counters for {device} device " +
                               f"since report type is {args.report_type}.")
                config[device][properties.enable_profiling()] = True
            elif args.exec_graph_path is not None:
                logger.warning(f"Turn on performance counters for {device} device " +
                               "due to execution graph dumping.")
                config[device][properties.enable_profiling()] = True
            elif is_flag_set_in_command_line('pcsort'):
                ## set to default value
                logger.warning(f"Turn on performance counters for {device} device " +
                               f"since pcsort value is {args.perf_counts_sort}.")
                config[device][properties.enable_profiling()] = True if args.perf_counts_sort else False
            else:
                ## set to default value
                config[device][properties.enable_profiling()] = args.perf_counts
            perf_counts = True if config[device][properties.enable_profiling()] == True else perf_counts

            ## insert or append property into hw device properties list
            def update_configs(hw_device, property_name, property_value):
                (key, value) = properties.device.properties({hw_device:{property_name:property_value}})
                # add property into hw device properties list.
                if key not in config[device].keys():
                    config[device][key] = value
                else:
                    current_config = config[device][key].get()
                    if hw_device not in current_config.keys():
                        current_config.update(value.get())
                    else:
                        current_device_config = current_config[hw_device]
                        for prop in value.get().items():
                            current_device_config.update(prop[1])
                        current_config[hw_device].update(current_device_config)
                    config[device][key].set(current_config)

            def update_device_config_for_virtual_device(value, config, key):
                # check if the element contains the hardware device property
                if len(value.split(':')) == 1:
                    config[device][key] = device_infer_precision[device]
                else:
                    # set device nstreams properties in the AUTO/MULTI plugin
                    value_vec = value[value.find('{') + 1:value.rfind('}')].split(',')
                    device_properties  = {value_vec[i].split(':')[0] : value_vec[i].split(':')[1] for i in range(0, len(value_vec))}
                    for hw_device in device_properties.keys():
                        update_configs(hw_device, key, device_properties[hw_device])

            ## infer precision
            def set_infer_precision():
                key = properties.hint.inference_precision()
                if device in device_infer_precision.keys():
                    ## set to user defined value
                    if key in supported_properties:
                        config[device][key] = device_infer_precision[device]
                    elif is_virtual_device(device):
                        update_device_config_for_virtual_device(device_infer_precision[device], config, key)
                    else:
                        raise Exception(f"Device {device} doesn't support config key INFERENCE_PRECISION_HINT!" \
                                        " Please specify -infer_precision for correct devices in format" \
                                        " <dev1>:<infer_precision1>,<dev2>:<infer_precision2> or via configuration file.")
                return

            ## the rest are individual per-device settings (overriding the values the device will deduce from perf hint)
            def set_throughput_streams():
                key = get_device_type_from_name(device) + "_THROUGHPUT_STREAMS"
                if device in device_number_streams.keys():
                    ## set to user defined value
                    if key in supported_properties:
                        config[device][key] = device_number_streams[device]
                    elif properties.streams.num() in supported_properties:
                        key = properties.streams.num()
                        config[device][key] = device_number_streams[device]
                    elif is_virtual_device(device):
                        key = properties.streams.num()
                        update_device_config_for_virtual_device(device_number_streams[device], config, key)
                    else:
                        raise Exception(f"Device {device} doesn't support config key '{key}'! " +
                                        "Please specify -nstreams for correct devices in format  <dev1>:<nstreams1>,<dev2>:<nstreams2>")
                elif key not in config[device].keys() and args.api_type == "async" and key not in config[device].keys() \
                    and 'PERFORMANCE_HINT' in config[device].keys() and config[device]['PERFORMANCE_HINT'] == '':
                    ## set the _AUTO value for the #streams
                    logger.warning(f"-nstreams default value is determined automatically for {device} device. " +
                                   "Although the automatic selection usually provides a reasonable performance, "
                                   "but it still may be non-optimal for some cases, for more information look at README.")
                    if key in supported_properties:
                        config[device][key] = get_device_type_from_name(device) + "_THROUGHPUT_AUTO"
                    elif properties.streams.Num() in supported_properties:
                        key = properties.streams.Num()
                        config[device][key] = "-1"  # Set AUTO mode for streams number
                    elif is_virtual_device(device):
                        # Set nstreams to default value auto if no nstreams specified from cmd line.
                        for hw_device in hw_devices_list:
                            hw_supported_properties = benchmark.core.get_property(hw_device, properties.supported_properties())
                            key = get_device_type_from_name(hw_device) + "_THROUGHPUT_STREAMS"
                            value = get_device_type_from_name(hw_device) + "_THROUGHPUT_AUTO"
                            if key not in hw_supported_properties:
                                key = properties.streams.Num()
                                value = properties.streams.Num.AUTO
                            if key in hw_supported_properties:
                                update_configs(hw_device, key, value)
                if key in config[device].keys():
                    device_number_streams[device] = config[device][key]
                return

            def set_nthreads_pin(property_name, property_value):
                if property_name == properties.affinity():
                    if property_value == "YES":
                        property_value = properties.Affinity.CORE
                    elif property_value == "NO":
                        property_value = properties.Affinity.NONE
                if property_name in supported_properties or device_name == AUTO_DEVICE_NAME:
                    # create nthreads/pin primary property for HW device or AUTO if -d is AUTO directly.
                    config[device][property_name] = property_value
                elif is_virtual:
                    # Create secondary property of -nthreads/-pin only for CPU if CPU device appears in the devices
                    # list specified by -d.
                    if CPU_DEVICE_NAME in hw_devices_list:
                        update_configs(CPU_DEVICE_NAME, property_name, property_value)
                return

            if args.number_threads and is_flag_set_in_command_line("nthreads"):
                # limit threading for CPU portion of inference
                set_nthreads_pin(properties.inference_num_threads(), str(args.number_threads))

            if is_flag_set_in_command_line('pin'):
                ## set for CPU to user defined value
                set_nthreads_pin(properties.affinity(), args.infer_threads_pinning)

            set_throughput_streams()
            set_infer_precision()

            if is_virtual_device(device):
                if device in device_number_streams.keys():
                    del device_number_streams[device]

        device_config = {}
        for device in config:
            if benchmark.device.find(device) == 0:
                device_config = config[device]
        if args.cache_dir:
            benchmark.set_cache_dir(args.cache_dir)

        ## If set batch size, disable the auto batching
        if args.batch_size:
            logger.warning("Batch size is set. Auto batching will be disabled")
            device_config["ALLOW_AUTO_BATCHING"] = False

        topology_name = ""
        load_from_file_enabled = is_flag_set_in_command_line('load_from_file') or is_flag_set_in_command_line('lfile')
        if load_from_file_enabled and not is_network_compiled:
            if args.mean_values or args.scale_values:
                raise RuntimeError("--mean_values and --scale_values aren't supported with --load_from_file. "
                    "The values can be set via model_optimizer while generating xml")
            next_step()
            print("Skipping the step for loading model from file")
            next_step()
            print("Skipping the step for loading model from file")
            next_step()
            print("Skipping the step for loading model from file")

            # --------------------- 7. Loading the model to the device -------------------------------------------------
            next_step()

            start_time = datetime.utcnow()
            compiled_model = benchmark.core.compile_model(args.path_to_model, benchmark.device, device_config)
            duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}"
            logger.info(f"Compile model took {duration_ms} ms")
            if statistics:
                statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                          [
                                              ('compile model time (ms)', duration_ms)
                                          ])
            app_inputs_info, _ = get_inputs_info(args.shape, args.data_shape, args.layout, args.batch_size, args.scale_values, args.mean_values, compiled_model.inputs)
            batch_size = get_network_batch_size(app_inputs_info)
        elif not is_network_compiled:
            # --------------------- 4. Read the Intermediate Representation of the network -----------------------------
            next_step()

            logger.info("Loading model files")

            start_time = datetime.utcnow()
            model = benchmark.read_model(args.path_to_model)
            topology_name = model.get_name()
            duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}"
            logger.info(f"Read model took {duration_ms} ms")
            logger.info("Original model I/O parameters:")
            print_inputs_and_outputs_info(model)

            if statistics:
                statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                          [
                                              ('read model time (ms)', duration_ms)
                                          ])

            # --------------------- 5. Resizing network to match image sizes and given batch ---------------------------
            next_step()

            app_inputs_info, reshape = get_inputs_info(args.shape, args.data_shape, args.layout, args.batch_size, args.scale_values, args.mean_values, model.inputs)

            # use batch size according to provided layout and shapes
            batch_size = get_network_batch_size(app_inputs_info)
            logger.info(f'Model batch size: {batch_size}')

            if reshape:
                start_time = datetime.utcnow()
                shapes = { info.name : info.partial_shape for info in app_inputs_info }
                logger.info(
                    'Reshaping model: {}'.format(', '.join("'{}': {}".format(k, str(v)) for k, v in shapes.items())))
                model.reshape(shapes)
                duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}"
                logger.info(f"Reshape model took {duration_ms} ms")
                if statistics:
                    statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                              [
                                                  ('reshape model time (ms)', duration_ms)
                                              ])

            # --------------------- 6. Configuring inputs and outputs of the model --------------------------------------------------
            next_step()

            pre_post_processing(model, app_inputs_info, args.input_precision, args.output_precision, args.input_output_precision)
            print_inputs_and_outputs_info(model)

            # --------------------- 7. Loading the model to the device -------------------------------------------------
            next_step()
            start_time = datetime.utcnow()
            compiled_model = benchmark.core.compile_model(model, benchmark.device, device_config)

            duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}"
            logger.info(f"Compile model took {duration_ms} ms")
            if statistics:
                statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                          [
                                              ('compile model time (ms)', duration_ms)
                                          ])
        else:
            if args.mean_values or args.scale_values:
                raise RuntimeError("--mean_values and --scale_values aren't supported for compiled model. "
                    "The values can be set via model_optimizer while generating xml")
            next_step()
            print("Skipping the step for compiled model")
            next_step()
            print("Skipping the step for compiled model")
            next_step()
            print("Skipping the step for compiled model")

            # --------------------- 7. Loading the model to the device -------------------------------------------------
            next_step()

            start_time = datetime.utcnow()
            try:
                with open(args.path_to_model, "rb") as model_stream:
                    model_bytes = model_stream.read()
                compiled_model = benchmark.core.import_model(model_bytes, device_name)
                duration_ms = f"{(datetime.utcnow() - start_time).total_seconds() * 1000:.2f}"
                logger.info(f"Import model took {duration_ms} ms")
                if statistics:
                    statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                            [
                                                ('import model time (ms)', duration_ms)
                                            ])
                app_inputs_info, _ = get_inputs_info(args.shape, args.data_shape, args.layout, args.batch_size, args.scale_values, args.mean_values, compiled_model.inputs)
                batch_size = get_network_batch_size(app_inputs_info)
            except Exception as e:
                raise RuntimeError(f"Cannot open or import compiled model file: {args.path_to_model}. Error: {str(e)}")

        # --------------------- 8. Querying optimal runtime parameters --------------------------------------------------
        next_step()

        ## actual device-deduced settings
        keys = compiled_model.get_property(properties.supported_properties())
        logger.info("Model:")
        for k in keys:
            skip_keys = ('SUPPORTED_METRICS', 'SUPPORTED_CONFIG_KEYS', properties.supported_properties())
            if k not in skip_keys:
                value = compiled_model.get_property(k)
                if k == properties.device.properties():
                    for device_key in value.keys():
                        logger.info(f'  {device_key}:')
                        for k2, value2 in value.get(device_key).items():
                            if k2 not in skip_keys:
                                logger.info(f'    {k2}: {value2}')
                else:
                    logger.info(f'  {k}: {value}')

        # Update number of streams
        for device in device_number_streams.keys():
            try:
                key = get_device_type_from_name(device) + '_THROUGHPUT_STREAMS'
                device_number_streams[device] = compiled_model.get_property(key)
            except:
                key = 'NUM_STREAMS'
                device_number_streams[device] = compiled_model.get_property(key)

        # ------------------------------------ 9. Creating infer requests and preparing input data ----------------------
        next_step()

        # Create infer requests
        requests = benchmark.create_infer_requests(compiled_model)

        # Prepare input data
        paths_to_input = list()
        if args.paths_to_input:
            for path in args.paths_to_input:
                if ":" in next(iter(path), ""):
                    paths_to_input.extend(path)
                else:
                    paths_to_input.append(os.path.abspath(*path))

        data_queue = get_input_data(paths_to_input, app_inputs_info)

        static_mode = check_for_static(app_inputs_info)
        allow_inference_only_or_sync = can_measure_as_static(app_inputs_info)
        if not allow_inference_only_or_sync and benchmark.api_type == 'sync':
            raise Exception("Benchmarking of the model with dynamic shapes is available for async API only. "
                            "Please use -api async -hint latency -nireq 1 to emulate sync behavior.")

        if benchmark.inference_only == None:
            if static_mode:
                benchmark.inference_only = True
            else:
                benchmark.inference_only = False
        elif benchmark.inference_only and not allow_inference_only_or_sync:
            raise Exception("Benchmarking dynamic model available with input filling in measurement loop only!")

        # update batch size in case dynamic network with one data_shape
        if allow_inference_only_or_sync and batch_size.is_dynamic:
            batch_size = Dimension(data_queue.batch_sizes[data_queue.current_group_id])

        benchmark.latency_groups = get_latency_groups(app_inputs_info)

        if len(benchmark.latency_groups) > 1:
            logger.info(f"Defined {len(benchmark.latency_groups)} tensor groups:")
            for group in benchmark.latency_groups:
                logger.info(f"\t{str(group)}")

        # Iteration limit
        benchmark.niter = get_number_iterations(benchmark.niter, benchmark.nireq, max(len(info.shapes) for info in app_inputs_info), benchmark.api_type)

        # Set input tensors before first inference
        for request in requests:
            data_tensors = data_queue.get_next_input()
            for port, data_tensor in data_tensors.items():
                input_tensor = request.get_input_tensor(port)
                if not static_mode:
                    input_tensor.shape = data_tensor.shape
                if not len(input_tensor.shape):
                    input_tensor.data.flat[:] = data_tensor.data
                else:
                    input_tensor.data[:] = data_tensor.data

        if statistics:
            statistics.add_parameters(StatisticsReport.Category.RUNTIME_CONFIG,
                                      [
                                          ('topology', topology_name),
                                          ('target device', device_name),
                                          ('API', args.api_type),
                                          ('inference_only', benchmark.inference_only),
                                          ('precision', "UNSPECIFIED"),
                                          ('batch size', str(batch_size)),
                                          ('number of iterations', str(benchmark.niter)),
                                          ('number of parallel infer requests', str(benchmark.nireq)),
                                          ('duration (ms)', str(get_duration_in_milliseconds(benchmark.duration_seconds))),
                                       ])

            for nstreams in device_number_streams.items():
                statistics.add_parameters(StatisticsReport.Category.RUNTIME_CONFIG,
                                         [
                                            (f"number of {nstreams[0]} streams", str(nstreams[1])),
                                         ])

        # ------------------------------------ 10. Measuring performance -----------------------------------------------

        output_string = process_help_inference_string(benchmark, device_number_streams)

        next_step(additional_info=output_string)

        if benchmark.inference_only:
            logger.info("Benchmarking in inference only mode (inputs filling are not included in measurement loop).")
        else:
            logger.info("Benchmarking in full mode (inputs filling are included in measurement loop).")
        duration_ms = f"{benchmark.first_infer(requests):.2f}"
        logger.info(f"First inference took {duration_ms} ms")
        if statistics:
            statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                    [
                                        ('first inference time (ms)', duration_ms)
                                    ])

        pcseq = args.pcseq
        if static_mode or len(benchmark.latency_groups) == 1:
            pcseq = False

        fps, median_latency_ms, avg_latency_ms, min_latency_ms, max_latency_ms, total_duration_sec, iteration = benchmark.main_loop(requests, data_queue, batch_size, args.latency_percentile, pcseq)

        # ------------------------------------ 11. Dumping statistics report -------------------------------------------
        next_step()

        if args.dump_config:
            dump_config(args.dump_config, config)
            logger.info(f"OpenVINO configuration settings were dumped to {args.dump_config}")

        if args.exec_graph_path:
            dump_exec_graph(compiled_model, args.exec_graph_path)

        if perf_counts:
            perfs_count_list = []
            for request in requests:
                perfs_count_list.append(request.profiling_info)

            if args.perf_counts_sort:
                total_sorted_list = print_perf_counters_sort(perfs_count_list,sort_flag=args.perf_counts_sort)
                if statistics:
                    statistics.dump_performance_counters_sorted(total_sorted_list)

            elif args.perf_counts:
                print_perf_counters(perfs_count_list)

            if statistics:
                # if not args.perf_counts_sort:
                statistics.dump_performance_counters(perfs_count_list)

        if statistics:
            statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                      [
                                          ('total execution time (ms)', f'{get_duration_in_milliseconds(total_duration_sec):.2f}'),
                                          ('total number of iterations', str(iteration)),
                                      ])
            if MULTI_DEVICE_NAME not in device_name:
                latency_prefix = None
                if args.latency_percentile == 50:
                    latency_prefix = 'latency (ms)'
                elif args.latency_percentile != 50:
                    latency_prefix = 'latency (' + str(args.latency_percentile) + ' percentile) (ms)'
                if latency_prefix:
                    statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                            [
                                                (latency_prefix, f'{median_latency_ms:.2f}'),
                                            ])
                statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                          [
                                              ("avg latency", f'{avg_latency_ms:.2f}'),
                                          ])
                statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                          [
                                              ("min latency", f'{min_latency_ms:.2f}'),
                                          ])
                statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                          [
                                              ("max latency", f'{max_latency_ms:.2f}'),
                                          ])
                if pcseq:
                    for group in benchmark.latency_groups:
                        statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                          [
                                              ("group", str(group)),
                                          ])
                        statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                          [
                                              ("avg latency", f'{group.avg:.2f}'),
                                          ])
                        statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                          [
                                              ("min latency", f'{group.min:.2f}'),
                                          ])
                        statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                          [
                                              ("max latency", f'{group.max:.2f}'),
                                          ])
            statistics.add_parameters(StatisticsReport.Category.EXECUTION_RESULTS,
                                      [
                                          ('throughput', f'{fps:.2f}'),
                                      ])
            statistics.dump()

        try:
            exeDevice = compiled_model.get_property("EXECUTION_DEVICES")
            logger.info(f'Execution Devices:{exeDevice}')
        except:
            exeDevice = None
        logger.info(f'Count:            {iteration} iterations')
        logger.info(f'Duration:         {get_duration_in_milliseconds(total_duration_sec):.2f} ms')
        if MULTI_DEVICE_NAME not in device_name:
            logger.info('Latency:')
            if args.latency_percentile == 50:
                logger.info(f'   Median:        {median_latency_ms:.2f} ms')
            elif args.latency_percentile != 50:
                logger.info(f'   {args.latency_percentile} percentile:     {median_latency_ms:.2f} ms')
            logger.info(f'   Average:       {avg_latency_ms:.2f} ms')
            logger.info(f'   Min:           {min_latency_ms:.2f} ms')
            logger.info(f'   Max:           {max_latency_ms:.2f} ms')

            if pcseq:
                logger.info("Latency for each data shape group:")
                for idx,group in enumerate(benchmark.latency_groups):
                    logger.info(f"{idx+1}.{str(group)}")
                    if args.latency_percentile == 50:
                        logger.info(f'   Median:     {group.median:.2f} ms')
                    elif args.latency_percentile != 50:
                        logger.info(f'   {args.latency_percentile} percentile:     {group.median:.2f} ms')
                    logger.info(f'   Average:    {group.avg:.2f} ms')
                    logger.info(f'   Min:        {group.min:.2f} ms')
                    logger.info(f'   Max:        {group.max:.2f} ms')

        logger.info(f'Throughput:   {fps:.2f} FPS')

        del compiled_model

        next_step.step_id = 0
    except Exception as e:
        logger.exception(e)

        if statistics:
            statistics.add_parameters(
                StatisticsReport.Category.EXECUTION_RESULTS,
                [('error', str(e))]
            )
            statistics.dump()
        sys.exit(1)