diff options
Diffstat (limited to 'python/openvino/runtime/dla_benchmark/main.cpp')
| -rw-r--r-- | python/openvino/runtime/dla_benchmark/main.cpp | 1575 |
1 files changed, 1575 insertions, 0 deletions
diff --git a/python/openvino/runtime/dla_benchmark/main.cpp b/python/openvino/runtime/dla_benchmark/main.cpp new file mode 100644 index 0000000..9d9055d --- /dev/null +++ b/python/openvino/runtime/dla_benchmark/main.cpp @@ -0,0 +1,1575 @@ +// Copyright (C) 2018-2023 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +// Description: Main file of DLA benchmark. Entry point of DLA for just in time, ahead of time execution +// and any use case of DLA performing inference. This file is responsible for the end to end flow of DLA, +// from reading user input arguments, creating input tensors, compiling models, running inference +// dumping results. DLA benchmark is loosely based off of OpenVINO's sample benchmark app. +// For future OpenVINO uplifts viewing their sample app is a good place to start. +// Ref: [openvinotoolkit/openvino › samples/cpp/benchmark_app/main.cpp] + +#include <algorithm> +#include <chrono> +#include <cstddef> +#include <map> +#include <memory> +#include <stdexcept> +#include <string> +#include <utility> +#include <vector> +#if defined(_WIN32) || defined(_WIN64) +#include <io.h> +#define NOMINMAX +#include <Windows.h> +#else +#include <dirent.h> +#include <unistd.h> +#endif +#include <stdio.h> +#include <sys/stat.h> +#include <fstream> +#include <regex> + +#include <samples/args_helper.hpp> +#include <samples/common.hpp> +#include <samples/slog.hpp> + +// DLA utils +#include "dla_stl_utils.h" +#include "dla_defines.h" + +// DLA benchmark +#include "average_precision.hpp" +#include "dla_benchmark.hpp" +#include "dla_plugin_config.hpp" +#include "infer_request_wrap.hpp" +#include "inputs_filling.hpp" +#include "progress_bar.hpp" +#include "statistics_report.hpp" +#include "top1_top5.hpp" +#include "utils.hpp" + +using DebugNetworkData = std::map<std::string, uint64_t>; +using LSUCounterData = std::map<std::string, uint64_t>; + +static const size_t progressBarDefaultTotalCount = 1000; + +// Get value from env variable named 'name', if it exists. +// If not, returns provided default value. +template <class T> +T GetEnvOrDefault(const char* name, T default_value) { + char* str_val = std::getenv(name); + T result = default_value; + if (str_val != NULL) { + std::stringstream ss; + ss << str_val; + ss >> result; + } + return result; +} + +bool ExistsTest(const std::string& name) { + struct stat buffer; + return (stat(name.c_str(), &buffer) == 0); +} + +bool isFile(const std::string& path) { +#if defined(_WIN32) || defined(_WIN64) + std::cout << "Windows-specific implementation for checking if something is a file" << std::endl; + // Windows-specific implementation + DWORD fileAttr = GetFileAttributesA(path.c_str()); + if (fileAttr == INVALID_FILE_ATTRIBUTES) { + // The path does not exist or an error occurred. + return false; + } + // Check if it's not a directory. + return !(fileAttr & FILE_ATTRIBUTE_DIRECTORY); +#else + // UNIX-specific implementation + struct stat buffer; + if (stat(path.c_str(), &buffer) == 0) { + return S_ISREG(buffer.st_mode); + } + return false; +#endif +} + +// This function appears in dla_aot_splitter/src/main.cpp too +bool DirOpenTest(const std::string& name) { +#if (!defined(_WIN32) && !defined(_WIN64)) + // If we can open the directory then return true + DIR* dp = opendir(name.c_str()); + if (dp != nullptr) { + closedir(dp); + return true; + } +#endif // !_WIN32 && !_WIN64 + struct stat sb; + if (stat(name.c_str(), &sb) == 0) { + if ((sb.st_mode & S_IFMT) != S_IFREG) { + slog::err << "File " << name << " cannot be opened!" << slog::endl; + throw std::logic_error("File cannot be opened!"); + } + } + return true; +} + +// Define a custom comparison function to sort based on ASCII names +bool CompareOutputNodeNames(const ov::Output<const ov::Node>& node1, const ov::Output<const ov::Node>& node2) { + return node1.get_any_name() < node2.get_any_name(); +} + +// copy arguments into a new array to split the '-i=<arg>' into +// two arguments (i.e. '-i' and '<arg>') to overcome a bug +// parseInputFilesArguments function where is doesn't recognize +// the -i=<arg> format +void ParseCommandLine(int argc, char** argv) { + int num_args = argc; + // allocated enough memory in case we needed to split the -i argument into two + char** arguments = new char*[num_args + 1]; + for (int i = 0, j = 0; j < argc; ++i, ++j) { + if (strstr(argv[j], "-i=")) { + // number of arguments will increase by one after splitting + num_args++; + arguments[i] = new char[3]; + strcpy(arguments[i++], "-i"); + // copy the reset of the argument (i.e. post "-i=") + arguments[i] = new char[strlen(argv[j]) - 2]; + strcpy(arguments[i], argv[j] + 3); + continue; + } + arguments[i] = new char[strlen(argv[j]) + 1]; + strcpy(arguments[i], argv[j]); + } + // the parse function is modifying the arguments point so we need to keep + // a copy of the original pointer value to delete it properly + char** orig_arg_ptr = arguments; + gflags::ParseCommandLineNonHelpFlags(&num_args, &arguments, true); + // delete the allocated memory + for (int i = 0; i < num_args; ++i) { + delete[] orig_arg_ptr[i]; + } + delete[] orig_arg_ptr; +} + +bool CheckAndSetPluginsPath(const char* coredla_root) { + // plugins_xml_file should probably be removed in the future + if (!FLAGS_plugins_xml_file.empty()) { + FLAGS_plugins = FLAGS_plugins_xml_file; + slog::warn << "====================================================================" << slog::endl; + slog::warn << "Warning: -plugins_xml_file option is deprecated, please use -plugins." << slog::endl; + slog::warn << "====================================================================" << slog::endl; + } + + const char* coredla_work = std::getenv("COREDLA_WORK"); + std::string coredla_root_str = coredla_root; + if (FLAGS_plugins.empty()) { + if (coredla_work == nullptr) { + FLAGS_plugins = coredla_root_str + "/runtime/plugins.xml"; + } else { + std::string coredla_work_str = coredla_work; + FLAGS_plugins = coredla_work_str + "/runtime/plugins.xml"; + } + + if (ExistsTest(FLAGS_plugins)) { + slog::info << "Using default plugins xml file - " << FLAGS_plugins << slog::endl; + return true; + } + } + + if (ExistsTest(FLAGS_plugins) && isFile(FLAGS_plugins)) { + slog::info << "Using custom plugins xml file - " << FLAGS_plugins << slog::endl; + return true; + } + // Check if user wants a shortcut to software emulation xml file if a path does not exist + if (FLAGS_plugins.find("emulation") != std::string::npos) { + // Potential paths for the plugins_emulation.xml file + std::string deployed_loc_plugins = coredla_root_str + "/bin/plugins_emulation.xml"; + std::string developer_loc_plugins = coredla_root_str + "/build/coredla/dla/bin/plugins_emulation.xml"; + + if (ExistsTest(deployed_loc_plugins)) + FLAGS_plugins = deployed_loc_plugins; + else if (ExistsTest(developer_loc_plugins)) + FLAGS_plugins = developer_loc_plugins; + } else { + // if user didn't specify emulation and user did not pass any xml file, raise an error + throw std::invalid_argument("Invalid argument for -plugins. Use 'emulation' or a path to custom xml file"); + } + + if (ExistsTest(FLAGS_plugins)) { + slog::info << "Using custom emulation xml file - " << FLAGS_plugins << slog::endl; + return true; + } + + return false; +} + +bool ParseAndCheckCommandLine(int argc, char* argv[], size_t& net_size) { + // ---------------------------Parsing and validating input arguments-------------------------------------- + slog::info << "Parsing input parameters" << slog::endl; + + // Check for any flags that are missing their preceding dashes + // GFlags quietly ignores any flags missing their dashes, which can cause + // dla_benchmark to run with settings other than what the user intended + + // GFlags supports two different styles of flag: + // 1. --<flag> + // 2. -<flag> + // It also supports two different ways of specifying values for flags which + // take values: + // 1. --<flag>=<value> + // 2. --<flag> <value> + + // If we are not expecting a flag, we are expecting a value for the + // preceding flag + bool expecting_flag = true; + // Start at 1 to skip the command itself + for (int i = 1; i < argc; i++) { + if (expecting_flag) { + // A flag is always denoted by the first char being '-' + if (argv[i][0] != '-') { + slog::err << "Argument " << argv[i] << " is invalid. You" + << " may have forgotten a preceding '-'." << slog::endl; + throw std::logic_error("One or more invalid arguments"); + } + + char* flag_name_start = (argv[i][1] == '-') ? &argv[i][2] : &argv[i][1]; + std::string flag_name; + + gflags::CommandLineFlagInfo flag_info; + if (strstr(flag_name_start, "=")) { + flag_name = std::string(flag_name_start, size_t(strstr(flag_name_start, "=") - flag_name_start)); + } else { + flag_name = std::string(flag_name_start); + } + + // We expect a flag in the next argv if the current flag is a bool, + // because bool flags do not take a value. + // If GetCommandLineFlagInfo returns false, we assume the current + // flag is a boolean because boolean flags can be specified as + // -no<flag>, which is equivalent to -<flag>=false, or the flag + // simply being omitted. However, "no<flag>" is not recognized by + // GetCommandLineFlagInfo. + // Therefore, if the name is not recognized either the flag is a + // boolean flag or doesn't exist. In the latter case, gflags errors + // when we call ParseCommandLine so we can assume here it's a bool. + if (!GetCommandLineFlagInfo(flag_name.c_str(), &flag_info) || strstr(argv[i], "=") || flag_info.type == "bool") { + expecting_flag = true; + } else { + expecting_flag = false; + } + } else { + // If we were expecting a value, doesn't matter what it is + // gflags will check all values are the correct type, and + // dla_benchmark checks if the values received are sane + expecting_flag = true; + } + } + + ParseCommandLine(argc, argv); + + if (FLAGS_help || FLAGS_h) { + ShowUsage(); + // CoreDLA: Version 2020.3 of OpenVINO assumes that the PAC board with OPAE on it + // is an OpenCL/DLAv1 device. Since it is not, it then errors-out when the device + // does not response as expected to the OpenCL query. + // showAvailableDevices(); + std::cout << "\n"; + return false; + } + + if (FLAGS_hidden_help) { + PrintHiddenHelp(); + return false; + } + + if (FLAGS_cm.empty()) { + std::string network_file_flag; + if (!FLAGS_m.empty()) { + if (!FLAGS_network_file.empty()) { + throw std::invalid_argument( + "Both --network-file and -m are specified. Please only use one of the two arguments."); + } + network_file_flag = FLAGS_m; + } else if (!FLAGS_network_file.empty()) { + network_file_flag = FLAGS_network_file; + } else { + throw std::logic_error("Model is required but not set. Please set -m option."); + } + + std::vector<std::string> m_paths = split(network_file_flag, MULTIGRAPH_SEP); + net_size = m_paths.size(); + slog::info << "Found " << net_size << " graph" << (net_size == 1 ? "" : "s") << slog::endl; + for (auto& m_path : m_paths) { + if (!ExistsTest(m_path)) { + slog::err << "network file: " << m_path << " doesn't exist. Please provide a valid path with -m." << slog::endl; + throw std::logic_error("Model file path does not exist."); + } + } + } else { + std::vector<std::string> m_paths = split(FLAGS_cm, MULTIGRAPH_SEP); + net_size = m_paths.size(); + slog::info << "Found " << net_size << " compiled graph" << (net_size == 1 ? "" : "s") << slog::endl; + for (auto& m_path : m_paths) { + if (!ExistsTest(m_path)) { + slog::err << "compiled model file: " << FLAGS_cm << " doesn't exist. Please provide a valid path with -cm." + << slog::endl; + throw std::logic_error("Compiled model file path does not exist."); + } + } + } + + if (FLAGS_api != "async" && FLAGS_api != "sync") { + throw std::logic_error("Incorrect API. Please set -api option to `sync` or `async` value."); + } + + if (FLAGS_niter <= 0) { + throw std::logic_error("-niter is a required flag and its value must be positive"); + } + + const char* coredla_root = std::getenv("COREDLA_ROOT"); + if (coredla_root == nullptr) { + slog::err << "ERROR: COREDLA_ROOT environment variable is not set." << slog::endl; + throw std::logic_error("Please set up correct environment variables first"); + } + + if (!CheckAndSetPluginsPath(coredla_root)) { + slog::err << "plugins_xml file: " << FLAGS_plugins_xml_file << " doesn't exist. Please provide a valid path." + << slog::endl; + throw std::logic_error("plugins_xml file path does not exist."); + } + + // Checks required arguments for the mAP calculation subroutine. + if (FLAGS_enable_object_detection_ap) { + if (!FLAGS_yolo_version.size() || !is_yolo_supported(FLAGS_yolo_version)) { + slog::err << "Please specify the version of your YOLO graph by setting the -yolo_version option to " + "`yolo-v3-tiny-tf` or `yolo-v3-tf` value." + << slog::endl; + throw std::logic_error("Incorrect YOLO version."); + } + } + + // Checks if output directory exists and can be opened + if (!FLAGS_output_dir.empty()) { + if (!ExistsTest(FLAGS_output_dir)) { + slog::err << "Specified output directory: " << FLAGS_output_dir << " does not exist" << slog::endl; + throw std::logic_error("Output directory does not exist"); + } + // Test whether the path can be opened if it's a directory + DirOpenTest(FLAGS_output_dir); + } + + return true; +} + +static void next_step(const std::string additional_info = "") { + static size_t step_id = 0; + static const std::map<size_t, std::string> step_names = {{1, "Parsing and validating input arguments"}, + {2, "Loading OpenVINO Runtime"}, + {3, "Setting device configuration"}, + {4, "Reading the Intermediate Representation network"}, + {5, "Resizing network to match image sizes and given batch"}, + {6, "Configuring input of the model"}, + {7, "Loading the model to the device"}, + {8, "Setting optimal runtime parameters"}, + {9, "Creating infer requests and preparing input tensors"}, + {10, "Measuring performance"}, + {11, "Dumping statistics report"}, + {12, "Dumping the output values"}}; + + step_id++; + if (step_names.count(step_id) == 0) + THROW_IE_EXCEPTION << "Step ID " << step_id << " is out of total steps number " << step_names.size(); + + std::cout << "[Step " << step_id << "/" << step_names.size() << "] " << step_names.at(step_id) + << (additional_info.empty() ? "" : " (" + additional_info + ")") << std::endl; +} + +template <typename T> +T GetMedianValue(const std::vector<T>& vec) { + std::vector<T> sorted_vec(vec); + std::sort(sorted_vec.begin(), sorted_vec.end()); + return (sorted_vec.size() % 2 != 0) + ? sorted_vec[sorted_vec.size() / 2ULL] + : (sorted_vec[sorted_vec.size() / 2ULL] + sorted_vec[sorted_vec.size() / 2ULL - 1ULL]) / + static_cast<T>(2.0); +} + +void ReadDebugNetworkInfo(ov::Core core) { + if (FLAGS_debug_network) { + // On hardware timeout exception, fetch Debug CSR values from all modules attached to the Debug Network + std::vector<DebugNetworkData> debug_csr_return = + core.get_property("FPGA", "COREDLA_DEBUG_NETWORK_INFO").as<std::vector<DebugNetworkData>>(); + slog::info << "Dumping Debug Network profiling counters" << slog::endl; + for (auto i = 0U; i < debug_csr_return.size(); i++) { + std::cout << "---------- CoreDLA instance " << i << " ----------" << std::endl; + // Print debug info for all instances + for (auto& instance_csr_return : debug_csr_return[i]) { + std::cout << instance_csr_return.first << ": " << instance_csr_return.second << std::endl; + } + } + } +} + +void PrintLSUCounterInfo(ov::Core core) { + std::vector<LSUCounterData> lsu_counter_vec = + core.get_property("FPGA", "COREDLA_LSU_ACCESS_COUNT").as<std::vector<LSUCounterData>>(); + slog::info << "Dumping LSU memory access counters" << slog::endl; + for (auto i = 0U; i < lsu_counter_vec.size(); i++) { + std::cout << "---------- CoreDLA instance " << i << " ----------" << std::endl; + for (const auto& entry : lsu_counter_vec.at(i)) { + std::cout << entry.first <<": " << entry.second << std::endl; + } + } +} + +// Returns true if last char of csv is a comma +bool is_last_char_comma(FILE* file) { + if (file == nullptr) return 0; + + int i = -1; + std::vector<char> white_space_chars = {'\n', ' ', '\t', '\r', '\f', '\v'}; + char last_char[1]; + do { + if (std::fseek(file, i, SEEK_END) != 0) { + return 0; + } + if (std::fread(last_char, 1, 1, file) == 0) { + return 0; + } + i--; + } while (std::count(white_space_chars.begin(), white_space_chars.end(), last_char[0]) != 0); + + return last_char[0] == ','; +} + +bool fileExists(std::string& path) { + struct stat buffer; + return (stat(path.c_str(), &buffer) == 0); +} + +void append_value_if_incomplete_to_csv(std::string path, double value) { + try { + if (!fileExists(path)) { + return; + } + + FILE* data_file = fopen(path.c_str(), "rb"); + if (data_file == nullptr) { + return; + } + bool is_comma = is_last_char_comma(data_file); + fclose(data_file); + + if (is_comma) { + FILE* append_file = fopen(path.c_str(), "a"); + if (append_file == nullptr) { + return; + } + fprintf(append_file, "%f\n", value); + fclose(append_file); + } + } catch (...) { + return; + } +} + +/** + * @brief The entry point of the dla benchmark + */ +int main(int argc, char* argv[]) { + std::shared_ptr<StatisticsReport> statistics; + try { + // Declaring the CompiledModel object as a pointer to workaround the segfault + // that occurs when destructing the object. Now that it's declared as a pointer + // the complier won't automatically call the destructor of the object at the end + // of this scope and we won't delete the allocated memory either + std::vector<ov::CompiledModel*> compiled_models; + size_t net_size = 0; // parse the size of networks for arguments check + + size_t return_code = 0; // universal return code, return this value after dumping out Debug info + + // ----------------- 1. Parsing and validating input arguments ------------------------------------------------- + next_step(); + + if (!ParseAndCheckCommandLine(argc, argv, net_size)) { + return 0; + } + + bool is_model_compiled = !FLAGS_cm.empty(); + if (is_model_compiled) { + slog::info << "Model is compiled" << slog::endl; + } + + std::string arch_file_flag; + if (!FLAGS_arch_file.empty()) { + if (!FLAGS_arch.empty()) { + throw std::invalid_argument( + "Both --arch and -arch_file are specified. Please only use one of the two arguments."); + } + arch_file_flag = FLAGS_arch_file; + } else if (!FLAGS_arch.empty()) { + arch_file_flag = FLAGS_arch; + } + + bool flag_b_default = gflags::GetCommandLineFlagInfoOrDie("b").is_default; + bool flag_batch_size_default = gflags::GetCommandLineFlagInfoOrDie("batch_size").is_default; + + size_t batch_size_flag; + if (!flag_b_default) { + if (!flag_batch_size_default) { + throw std::invalid_argument( + "Both --batch-size and -b are specified. Please only use one of the two arguments."); + } + batch_size_flag = FLAGS_b; + } else { + batch_size_flag = FLAGS_batch_size; + } + + if (batch_size_flag > 10000 || batch_size_flag <= 0) { + throw std::invalid_argument( + "Batch size is too big (>10000) or not a postive number (<=0). Specify the batch size within the specified " + "range."); + } + + std::string network_file_flag; + if (!FLAGS_m.empty()) { + if (!FLAGS_network_file.empty()) { + throw std::invalid_argument( + "Both --network-file and -m are specified. Please only use one of the two arguments."); + } + network_file_flag = FLAGS_m; + } else if (!FLAGS_network_file.empty()) { + network_file_flag = FLAGS_network_file; + } + + // langsu: ideally use boost to create a sub-folder for ddrfree files + // but ed4 toolchain doesn't have boost yet. + std::string output_dir; + std::string parameter_rom_output_dir; + std::string separator = dla::util::path_separator; + if (!FLAGS_output_dir.empty()) { + output_dir = FLAGS_output_dir + separator; + parameter_rom_output_dir = output_dir; + } else { + output_dir = "." + separator; + parameter_rom_output_dir = output_dir; + } + + // The set of arguments printed is meant to be a useful summary to the + // user, rather than all of the arguments to dla_benchmark + slog::info << "Printing summary of arguments being used by dla_benchmark" << slog::endl + << "API (-api) ........................... " << FLAGS_api << slog::endl + << "Device (-d) .......................... " << FLAGS_d << slog::endl + << "Batch size (-b) ...................... " << batch_size_flag << slog::endl + << (!FLAGS_cm.empty() ? "Compiled model (-cm) ................. " + : "Model (-m) ........................... ") + << (!FLAGS_cm.empty() ? FLAGS_cm : network_file_flag) << slog::endl + << "Num iterations (-niter) .............. " + << (FLAGS_niter > 0 ? std::to_string(FLAGS_niter) : "Not specified") << slog::endl + << "Input images directory (-i) .......... " + << (!FLAGS_i.empty() ? FLAGS_i : "Not specified, will use randomly-generated images") << slog::endl + << "Num CPU threads (-nthreads) .......... " + << (FLAGS_nthreads > 0 ? std::to_string(FLAGS_nthreads) : "Not specified") << slog::endl + << "Architecture file (-arch_file) ....... " << arch_file_flag << slog::endl + << "Num inference requests (-nireq) ...... " + << (FLAGS_nireq > 0 ? std::to_string(FLAGS_nireq) : "Not specified") << slog::endl + << "Plugins file (-plugins) ..... " << FLAGS_plugins << slog::endl + << "Groundtruth file (-groundtruth_loc) .. " + << (!FLAGS_groundtruth_loc.empty() ? FLAGS_groundtruth_loc : "Not specified") << slog::endl + << "Reverse input image channels (-bgr) .. " << (FLAGS_bgr ? "True" : "False") << slog::endl + << "EA features " << (FLAGS_enable_early_access ? "enabled." : "disabled.") << slog::endl; + + if (FLAGS_save_run_summary) { + std::vector<gflags::CommandLineFlagInfo> flags; + StatisticsReport::Parameters command_line_arguments; + gflags::GetAllFlags(&flags); + + for (auto& flag : flags) { + if (!flag.is_default) { + command_line_arguments.push_back({flag.name, flag.current_value}); + } + } + + if (!FLAGS_pcsort.empty() && + (FLAGS_pcsort != "simple_sort" && FLAGS_pcsort != "sort" && FLAGS_pcsort != "no_sort")) { + slog::err << "Invalid -pcsort option: " << FLAGS_pcsort << ". Please use one of sort, simple_sort, no_sort." + << slog::endl; + return 1; + } + + statistics = + std::make_shared<StatisticsReport>(StatisticsReport::Config{FLAGS_save_run_summary, FLAGS_report_folder}); + statistics->addParameters(StatisticsReport::Category::COMMAND_LINE_PARAMETERS, command_line_arguments); + } + + /** This vector stores paths to the processed images **/ + auto multi_input_files = VectorMap<std::vector<std::string>>( + SplitMultiInputFilesArguments(net_size), // get input directory list + [&](const std::vector<std::string>& input_args) mutable { + std::vector<std::string> files; + for (auto& input_arg : input_args) { + // Test if the path exists + if (!ExistsTest(input_arg)) { + slog::err << "Specified image path: " << input_arg << " does not exist" << slog::endl; + throw std::logic_error("Image path does not exist"); + } + // Test whether the path can be opened if it's a directory + DirOpenTest(input_arg); + readInputFilesArguments(files, input_arg); + } + return files; + }); + + if (multi_input_files.size() == 0) { + // failed to read input files + slog::err << "Failed to read input files" << slog::endl; + return 1; + } + + if (FLAGS_nstreams.empty()) { + slog::warn << "-nstreams default value is determined automatically for a device. " << slog::endl; + std::cout << "\tAlthough the automatic selection usually provides a reasonable performance, \n" + << "\tbut it still may be non-optimal for some cases, for more information look at README." + << std::endl; + } + +#ifdef DISABLE_JIT + if (!network_file_flag.empty()) { + slog::err << "Runtime compiled without support for Just-in-Time (JIT) execution!" << slog::endl + << "Either specify a compiled model using -cm <compiled_model.bin> " + << "or recompile the runtime without the -disable_jit flag." << slog::endl; + return 1; + } +#endif + + uint32_t num_batches = 1; + + // ----------------- 2. Loading OpenVINO Runtime/Inference Engine + // ----------------------------------------------------------- + next_step(); + + // Get optimal runtime parameters for device + std::string device_name = FLAGS_d; + if (is_model_compiled) { + auto compiled_graph_paths = split(FLAGS_cm, MULTIGRAPH_SEP); // separate each AOT file path + for (auto& compiled_graph : compiled_graph_paths) { + std::filebuf obj_file_buf; + // There does not seem to be a way to get the device from the OpenVINO executable network + // Instead we manually read through the xml header in the AOT graph to get the device name (an ugly hack + // unfortunately) + obj_file_buf.open(compiled_graph.c_str(), std::ios::in | std::ios::binary); + std::istream obj_istream(&obj_file_buf); + std::string xml_header, current_device; + getline(obj_istream, xml_header); // retrieve xml header from AOT bin file + if (xml_header.find("TARGET_FALLBACK") != std::string::npos) { // uses hetero plugin + int start_index = xml_header.find("TARGET_FALLBACK") + 24; + int end_index = xml_header.find("</hetero_config>") - 3; + current_device = + "HETERO:" + xml_header.substr(start_index, end_index - start_index); // get device from xml header + } else { + current_device = "FPGA"; + } + if (device_name == "") { // device flag not specified in AOT flow + device_name = current_device; + } else { + if (current_device != device_name) { // print error for non-matching devices + throw std::logic_error( + "The AOT file does not target the expected device. " + "The device specified to dla_benchmark using the -d flag must be the same as the " + "device specified to dla_compiler using the --fplugin flag."); + } + } + } + } else { + if (device_name == "") device_name = "CPU"; // default device for JIT flow is CPU + } + ov::Core core(FLAGS_plugins); + + if (device_name.find("CPU") != std::string::npos) { + core.set_property("FPGA", {{DLIAPlugin::properties::cpu_used.name(), true}}); + } + + if (arch_file_flag != "" && device_name.find("FPGA") != std::string::npos) { + core.set_property("FPGA", {{DLIAPlugin::properties::arch_path.name(), arch_file_flag}}); + if (!ExistsTest(arch_file_flag)) { + slog::err << "architecture file: " << arch_file_flag << " doesn't exist. Please provide a valid path." + << slog::endl; + throw std::logic_error("architecture file path does not exist."); + } + if (FLAGS_encryption_key != "") { + core.set_property("FPGA", {{DLIAPlugin::properties::encryption_key.name(), FLAGS_encryption_key}}); + } + if (FLAGS_encryption_iv != "") { + core.set_property("FPGA", {{DLIAPlugin::properties::encryption_iv.name(), FLAGS_encryption_iv}}); + } + // If emulator is used, do not perform decryption of compiled results in the import step + if (FLAGS_emulator_decryption) { + core.set_property("FPGA", {{DLIAPlugin::properties::emulator_decryption.name(), CONFIG_VALUE(YES)}}); + } + if (FLAGS_min_subgraph_layers < 1) { + slog::err << "-min-subgraph-layers must be >= 1" << slog::endl; + return 1; + } + core.set_property("FPGA", {{DLIAPlugin::properties::min_subgraph_layers.name(), FLAGS_min_subgraph_layers}}); + } + + if (device_name.find("CPU") != std::string::npos && !FLAGS_l.empty()) { + // CPU extensions is loaded as a shared library and passed as a pointer to base extension + core.add_extension(FLAGS_l); + slog::info << "CPU extensions is loaded " << FLAGS_l << slog::endl; + } + + slog::info << "OpenVINO: " << ov::get_openvino_version() << slog::endl; + slog::info << "Device info: " << core.get_versions(device_name) << slog::endl; + + // ----------------- 3. Setting device configuration ----------------------------------------------------------- + next_step(); + + auto devices = ParseDevices(device_name); + std::map<std::string, uint32_t> device_nstreams = ParseNStreamsValuePerDevice(devices, FLAGS_nstreams); + for (auto& pair : device_nstreams) { + auto key = std::string(pair.first + "_THROUGHPUT_STREAMS"); + std::vector<std::string> supported_config_keys = + core.get_property(pair.first, METRIC_KEY(SUPPORTED_CONFIG_KEYS)).as<std::vector<std::string>>(); + if (std::find(supported_config_keys.begin(), supported_config_keys.end(), key) == supported_config_keys.end()) { + throw std::logic_error( + "Device " + pair.first + " doesn't support config key '" + key + "'! " + + "Please specify -nstreams for correct devices in format <dev1>:<nstreams1>,<dev2>:<nstreams2>"); + } + } + + // pc is for CPU only at the moment + bool perf_count = FLAGS_pc; + std::string perf_count_sort = FLAGS_pcsort; + for (auto& device : devices) { + if (device == "CPU") { // CPU supports few special performance-oriented keys + if (perf_count || !perf_count_sort.empty()) { + core.set_property("CPU", {{CONFIG_KEY(PERF_COUNT), CONFIG_VALUE(YES)}}); + } + // limit threading for CPU portion of inference + if (FLAGS_nthreads != 0) + core.set_property(device, {{CONFIG_KEY(CPU_THREADS_NUM), std::to_string(FLAGS_nthreads)}}); + core.set_property(device, {{CONFIG_KEY(CPU_BIND_THREAD), FLAGS_pin}}); + // Set CPU to optimize throughput + core.set_property(device, ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)); + // for CPU execution, more throughput-oriented execution via streams + if (FLAGS_api == "async") { + core.set_property( + device, + ov::streams::num(device_nstreams.count(device) > 0 ? ov::streams::Num(device_nstreams.at(device)) + : ov::streams::AUTO)); + } + device_nstreams[device] = core.get_property(device, ov::streams::num); + } else if (device == ("GPU")) { + if (FLAGS_api == "async") { + core.set_property( + device, + ov::streams::num(device_nstreams.count(device) > 0 ? ov::streams::Num(device_nstreams.at(device)) + : ov::streams::AUTO)); + } + device_nstreams[device] = core.get_property(device, ov::streams::num); + } + } + + auto double_to_string = [](const double number) { + std::stringstream ss; + ss << std::fixed << std::setprecision(4) << number; + return ss.str(); + }; + auto get_total_ms_time = [](Time::time_point& start_time) { + return std::chrono::duration_cast<ns>(Time::now() - start_time).count() * 0.000001; + }; + + size_t batch_size = batch_size_flag; + std::vector<std::string> topology_names; + ov::element::Type precision = ov::element::undefined; + // Vector stores which model (multigraph), InputsInfo is a map of input names and its respctive + // input information + std::vector<dla_benchmark::InputsInfo> input_infos; + if (!is_model_compiled) { +#ifndef DISABLE_JIT + // We choose to ifdef out this block of code because it's more readable than + // pulling the block in the "else" out using ifdefs + // ----------------- 4. Reading the Intermediate Representation network ---------------------------------------- + next_step(); + + LOG_AND_PRINT(Logger::INFO, "Loading network files\n"); + + auto start_time_read = Time::now(); + // get list of graphs + std::vector<std::shared_ptr<ov::Model>> models = + VectorMap<std::shared_ptr<ov::Model>>(split(network_file_flag, MULTIGRAPH_SEP), [&](const std::string& m) { + std::shared_ptr<ov::Model> model = core.read_model(m); + // Assign rt info IMMEDIATELY when DLA benchmark reads the model. + // Applying transformations or reshaping may change node names. + // Mixed Precision is an EA only feature for 2024.2 + if (FLAGS_enable_early_access) { + for (auto&& node : model->get_ops()) { + if (dla::util::NodeTypeUsesPE(node->get_type_name())) { + node->get_rt_info()[DLA_PE_PRECISION_MODE] = + dla::util::ParseNodeForRTInfo(node->get_friendly_name(), DLA_PE_PRECISION_MODE); + } + } + } + printInputAndOutputsInfoShort(*model); + return model; + }); + + auto duration_ms = double_to_string(get_total_ms_time(start_time_read)); + slog::info << "Read network(s) took " << duration_ms << " ms" << slog::endl; + if (statistics) + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, + {{"read network time (ms)", duration_ms}}); + + // ----------------- 5. Resizing network to match image sizes and given batch ---------------------------------- + next_step(); + + for (size_t i = 0; i < models.size(); i++) { + const auto& model_inputs = std::const_pointer_cast<const ov::Model>(models[i])->inputs(); + bool reshape = false; + input_infos.push_back( + GetInputsInfo(batch_size, model_inputs, reshape, FLAGS_bin_data, FLAGS_mean_values, FLAGS_scale_values)); + if (reshape) { + dla_benchmark::PartialShapes shapes = {}; + for (auto& item : input_infos.back()) shapes[item.first] = item.second.partial_shape; + slog::info << "Reshaping model to batch: " << batch_size << slog::endl; + models[i]->reshape(shapes); + } + topology_names.push_back(models[i]->get_friendly_name()); + } + + // ----------------- 6. Configuring input and output + // ---------------------------------------------------------------------- + next_step(); + // Set input layouts for all models and their inputs + size_t input_info_idx = 0; + for (std::shared_ptr<ov::Model> model : models) { + auto preproc = ov::preprocess::PrePostProcessor(model); + const auto& inputs = model->inputs(); + for (size_t i = 0; i < inputs.size(); i++) { + ov::preprocess::InputInfo& input_info = preproc.input(i); + const size_t input_rank = inputs[i].get_partial_shape().size(); + const ov::Layout& layout = ov::Layout(dla::util::getTensorLayout(input_rank)); + const ov::element::Type_t type = input_infos[input_info_idx].at(inputs[i].get_any_name()).type; + input_info.tensor().set_element_type(type).set_layout(layout); + } + + const auto& outputs = model->outputs(); + for (size_t i = 0; i < outputs.size(); i++) { + const size_t output_rank = outputs[i].get_partial_shape().size(); + const ov::Layout& layout = ov::Layout(dla::util::getTensorLayout(output_rank)); + preproc.output(i).tensor().set_element_type(ov::element::f32).set_layout(layout); + } + // Once the build() method is called, the pre(post)processing steps + // for layout and precision conversions are inserted automatically + model = preproc.build(); + input_info_idx++; + } + // ----------------- 7. Loading the model to the device -------------------------------------------------------- + next_step(); + + // Get the value from the command line arguments (if the command line argument wasn't + // used by the user the default value set in dla_benchmark.hpp will be used) + int folding_option = FLAGS_folding_option; + bool fold_preprocessing = FLAGS_fold_preprocessing; + bool estimate_per_layer = FLAGS_estimate_per_layer_latencies; + bool enable_early_access = FLAGS_enable_early_access; + // TODO(arooney): Remove this once LT hang is fixed. + bool multi_infer_req = false; + if (FLAGS_nireq > 1 && FLAGS_api == "async") { + multi_infer_req = true; + } + + core.set_property("FPGA", {{DLIAPlugin::properties::folding_option.name(), std::to_string(folding_option)}}); + core.set_property("FPGA", + {{DLIAPlugin::properties::fold_preprocessing.name(), fold_preprocessing}}); + core.set_property("FPGA", + {{DLIAPlugin::properties::per_layer_estimation.name(), estimate_per_layer}}); + core.set_property("FPGA", + {{DLIAPlugin::properties::enable_early_access.name(), enable_early_access}}); + core.set_property("FPGA", + {{DLIAPlugin::properties::multiple_inferences.name(), multi_infer_req}}); + core.set_property("FPGA", {{DLIAPlugin::properties::streaming_input_pipe.name(), FLAGS_streaming_input_pipe}}); + + auto start_time = Time::now(); + auto individual_start_time = Time::now(); // timer for each individual graph loading + compiled_models = VectorMap<ov::CompiledModel*>(models, [&](std::shared_ptr<ov::Model> model) { + // Apply Low Precision transformations to handle quantized graphs + // Mohamed_I: currently, this only works if the entire graph fits on the FPGA + // because the CPU plugin calls common_optimizations again which has some transformations + // that cause the graph to fail (I suspect it's the ConvolutionMultiplyFusion, but I + // cannot disable it from the CPU) + + bool FPGA_used = device_name.find("FPGA") != std::string::npos; + bool CPU_used = device_name.find("CPU") != std::string::npos; + + ov::AnyMap config; + config.emplace(DLIAPlugin::properties::cpu_used.name(), CPU_used); + config.emplace(DLIAPlugin::properties::export_dir.name(), output_dir); + config.emplace(DLIAPlugin::properties::parameter_rom_export_dir.name(), parameter_rom_output_dir); + + for (auto&& node : model->get_ops()) { + if (std::string("FakeQuantize") == node->get_type_name()) { + config.emplace(DLIAPlugin::properties::apply_low_precision_transforms.name(), true); + if (CPU_used && FPGA_used) { + std::cerr << "ERROR: Quantized graphs only supported through HETERO:FPGA or CPU." << std::endl; + throw std::logic_error("HETERO:FPGA,CPU plugin is not supported for quantization."); + } + } + } + + auto compiled_model = new ov::CompiledModel(); + *compiled_model = core.compile_model(model, device_name, config); + duration_ms = double_to_string(get_total_ms_time(individual_start_time)); + individual_start_time = Time::now(); + slog::info << "Compile model ( " << model->get_friendly_name() << " ) took " << duration_ms << " ms" + << slog::endl; + return compiled_model; + }); + duration_ms = double_to_string(get_total_ms_time(start_time)); + slog::info << "Load network(s) took " << duration_ms << " ms" << slog::endl; + if (statistics) + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, + {{"load network time (ms)", duration_ms}}); +#endif + } else { + next_step(); + slog::info << "Skipping the step for compiled network" << slog::endl; + next_step(); + slog::info << "Skipping the step for compiled network" << slog::endl; + next_step(); + slog::info << "Skipping the step for compiled network" << slog::endl; + // ----------------- 7. Loading the model to the device -------------------------------------------------------- + next_step(); + auto compiled_graph_paths = split(FLAGS_cm, MULTIGRAPH_SEP); + compiled_models = vectorMapWithIndex<ov::CompiledModel*>( + split(FLAGS_cm, MULTIGRAPH_SEP), // get a list of compiled graphs + [&](const std::string& compiled_graph_path, size_t index) { + std::stringstream generated_name; + generated_name << "Graph_" << index; + slog::info << "Importing model from " << compiled_graph_paths[index] << " to " << device_name << " as " + << generated_name.str() << slog::endl; + auto start_time = Time::now(); + std::ifstream model_stream(compiled_graph_paths[index].c_str(), std::ios_base::in | std::ios_base::binary); + if (!model_stream.is_open()) { + throw std::runtime_error("Cannot open compiled model file: " + compiled_graph_paths[index]); + } + auto compiled_model = new ov::CompiledModel(); + core.set_property("FPGA", + {{DLIAPlugin::properties::streaming_input_pipe.name(), FLAGS_streaming_input_pipe}}); + // Import specific configs + ov::AnyMap config; + config.emplace(DLIAPlugin::properties::export_dir.name(), output_dir); + config.emplace(DLIAPlugin::properties::parameter_rom_export_dir.name(), parameter_rom_output_dir); + *compiled_model = core.import_model(model_stream, device_name, config); + topology_names.push_back(generated_name.str()); + model_stream.close(); + printInputAndOutputsInfoShort(*compiled_model); + auto duration_ms = double_to_string(get_total_ms_time(start_time)); + slog::info << "Import model took " << duration_ms << " ms" << slog::endl; + if (statistics) + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, + {{"import model time (ms)", duration_ms}}); + if (batch_size == 0) { + batch_size = 1; + } + const auto& inputs = compiled_model->inputs(); + for (const auto& item : inputs) { + const auto& shape = item.get_shape(); + if (shape[0] != batch_size) { + slog::err << "Batch size of the compiled model is " << shape[0] << " and batch size provided is " + << batch_size << slog::endl; + std::cout << "Set the same batch size = " << shape[0] << " when running the app" << std::endl; + std::cout << "Or recompile model with batch size = " << batch_size << std::endl; + exit(5); + } + } + bool reshape_required = false; + input_infos.push_back(GetInputsInfo(batch_size, + compiled_model->inputs(), + reshape_required, + FLAGS_bin_data, + FLAGS_mean_values, + FLAGS_scale_values)); + return compiled_model; + }); + } + // ----------------- 8. Setting optimal runtime parameters ----------------------------------------------------- + next_step(); + + // Number of requests + uint32_t nireq = FLAGS_nireq; +#if defined(__arm__) | defined(__aarch64__) + // In OpenVINO 2022.3 Arm plugin, when a AOT graph is compiled on CPU and dla_benchmark has -nireq > 1 + // the program will be killed. We force nireq = 1 for HETERO:CPU graph only. + // Note: -d CPU doesn't need to be checked for AOT because dlac does not support -fplugin CPU. + if (device_name == "HETERO:CPU" && nireq > 1) { + slog::warn << "-nireq > 1 is not supported for HETERO:CPU graph. Forcing -nireq = 1" << slog::endl; + nireq = 1; + } + +#endif + + if (nireq == 0) { + if (FLAGS_api == "sync") { + nireq = 1; + } else { + try { + nireq = 0; + for (auto& compiled_model : compiled_models) { + auto req = compiled_model->get_property(ov::optimal_number_of_infer_requests); + if (nireq == 0 || nireq > req) nireq = req; + } + } catch (const std::exception& ex) { + throw ov::Exception("Every device used with the dla_benchmark should support " + + std::string(ov::optimal_number_of_infer_requests.name()) + + " Failed to query the metric for the " + device_name + " with error: " + ex.what()); + } + } + } +#ifdef MAX_NUM_INFERENCE_REQUEST + if (nireq > MAX_NUM_INFERENCE_REQUEST) { + slog::warn << "-nireq > "<< MAX_NUM_INFERENCE_REQUEST << " is not supported for the underlying device. Forcing -nireq = 1" << slog::endl; + nireq = 1; + } +#endif + + // Iteration limit + uint32_t niter = FLAGS_niter; + if (niter > 0) { + // Round up niter to a multiple of nireq + niter = ((niter + nireq - 1) / nireq) * nireq; + // We previously checked that FLAGS_niter >= 0, so okay to cast to uint. + if (static_cast<uint32_t>(FLAGS_niter) != niter) { + slog::warn << "Number of iterations was aligned by request number from " << FLAGS_niter << " to " << niter + << " using number of requests " << nireq << slog::endl; + } + num_batches = niter; + } else if (niter > 0) { + num_batches = niter; + } + + // Graph-request limit on device + if (device_name.find("FPGA") != std::string::npos) { + int ip_num_instances = core.get_property("FPGA", "COREDLA_NUM_INSTANCES").as<int>(); + int numOutstandingInferRequest = nireq * net_size / ip_num_instances; + int maxOutstandingInferRequest = core.get_property("FPGA", "COREDLA_DMA_CSR_DESCRIPTOR_QUEUE_SIZE").as<int>(); + if (maxOutstandingInferRequest > 0 && numOutstandingInferRequest > maxOutstandingInferRequest) { + slog::err << "Possible number of outstanding inference requests per instance (" << numOutstandingInferRequest + << ") " + << "exceeds the CSR descriptor queue limit (" << maxOutstandingInferRequest << ")" << slog::endl; + return 1; + } + } + + if (statistics) { + for (auto& topology_name : topology_names) { + statistics->addParameters(StatisticsReport::Category::RUNTIME_CONFIG, + { + {"topology", topology_name}, + {"target device", device_name}, + {"API", FLAGS_api}, + {"precision", std::string(precision.get_type_name())}, + {"batch size", std::to_string(batch_size)}, + {"number of iterations", std::to_string(niter)}, + {"number of parallel infer requests", std::to_string(nireq)}, + }); + } + for (auto& nstreams : device_nstreams) { + std::stringstream ss; + ss << "number of " << nstreams.first << " streams"; + statistics->addParameters(StatisticsReport::Category::RUNTIME_CONFIG, + { + {ss.str(), std::to_string(nstreams.second)}, + }); + } + } + + // ----------------- 9. Creating infer requests and filling input blobs ---------------------------------------- + next_step(); + + // Data structure hierarchy + // Outermost vec: which model it corresponds to (multigraph) + // Map: input/output name and its corresponding TensorVector + // TensorVector: An alias for vector<ov::tensor> where each vector element correspond to the batch + std::vector<std::map<std::string, ov::TensorVector>> input_data_tensors; + std::vector<std::map<std::string, ov::TensorVector>> output_tensors(compiled_models.size()); + + std::vector<std::unique_ptr<InferRequestsQueue>> infer_request_queues; + const std::string resize_type = FLAGS_resize_type.empty() ? "resize" : FLAGS_resize_type; + for (size_t net_idx = 0; net_idx < compiled_models.size(); net_idx++) { + // Handle the case that use same inputs for all networks + const auto& inputFiles = + net_idx >= multi_input_files.size() ? multi_input_files.back() : multi_input_files[net_idx]; + input_data_tensors.push_back(GetStaticTensors(inputFiles.empty() ? std::vector<std::string>{} : inputFiles, + batch_size, + input_infos[net_idx], + num_batches, + resize_type, + FLAGS_bgr, + FLAGS_bin_data, + FLAGS_verbose)); + // Use unique_ptr to create InferRequestsQueue objects and avoid copying mutex and cv + infer_request_queues.push_back( + std::move(std::unique_ptr<InferRequestsQueue>(new InferRequestsQueue(*(compiled_models[net_idx]), nireq)))); + } + + // ----------------- 10. Measuring performance ------------------------------------------------------------------ + size_t progress_bar_total_count = progressBarDefaultTotalCount; + + std::stringstream ss; + ss << "Start inference " << FLAGS_api << "ronously"; + if (FLAGS_api == "async") { + if (!ss.str().empty()) { + ss << ", "; + } + ss << infer_request_queues.size() * infer_request_queues.at(0)->requests.size() << " inference requests"; + std::stringstream device_ss; + for (auto& nstreams : device_nstreams) { + if (!device_ss.str().empty()) { + device_ss << ", "; + } + device_ss << nstreams.second << " streams for " << nstreams.first; + } + if (!device_ss.str().empty()) { + ss << " using " << device_ss.str(); + } + } + ss << ", limits: " << niter << " iterations with each graph, " << compiled_models.size() << " graph(s)"; + progress_bar_total_count = niter; + next_step(ss.str()); + + /** Start inference & calculate performance **/ + /** to align number if iterations to guarantee that last infer requests are executed in the same conditions **/ + ProgressBar progress_bar(progress_bar_total_count, FLAGS_stream_output, FLAGS_progress); + std::vector<size_t> iterations(compiled_models.size(), 0); + try { + while ((niter != 0LL && iterations.back() < niter) || (FLAGS_api == "async" && iterations.back() % nireq != 0)) { + // set up all infer request and prep all i/o Blobs + for (size_t net_id = 0; net_id < compiled_models.size(); net_id++) { + for (size_t iireq = 0; iireq < nireq; iireq++) { + auto infer_request = infer_request_queues.at(net_id)->get_idle_request(); + if (!infer_request) { + THROW_IE_EXCEPTION << "No idle Infer Requests!"; + } + + if (niter != 0LL) { + const auto& outputs = compiled_models[net_id]->outputs(); + for (const auto& output : outputs) { + const std::string& name = output.get_any_name(); + output_tensors.at(net_id)[name].emplace_back(output.get_element_type(), output.get_shape()); + infer_request->set_tensor(output, output_tensors.at(net_id).at(name).at(iterations.at(net_id))); + } + const auto& inputs = compiled_models[net_id]->inputs(); + for (auto& input : inputs) { + const std::string& name = input.get_any_name(); + const auto& data = input_data_tensors.at(net_id).at(name)[iterations.at(net_id)]; + infer_request->set_tensor(input, data); + } + } + + // Execute one request/batch + if (FLAGS_api == "sync") { + infer_request->infer(); + } else { + // As the inference request is currently idle, the wait() adds no additional overhead (and should return + // immediately). The primary reason for calling the method is exception checking/re-throwing. Callback, + // that governs the actual execution can handle errors as well, but as it uses just error codes it has no + // details like ‘what()’ method of `std::exception` So, rechecking for any exceptions here. + infer_request->wait(); + infer_request->start_async(); + } + iterations.at(net_id)++; + if (net_id == compiled_models.size() - 1) { + progress_bar.addProgress(1); + } + } + } + } + + // wait the latest inference executions + for (auto& infer_request_queue : infer_request_queues) { + infer_request_queue->wait_all(); + } + } catch (const std::exception& ex) { + slog::err << "Inference failed:" << slog::endl; + slog::err << ex.what() << slog::endl; + ReadDebugNetworkInfo(core); + PrintLSUCounterInfo(core); + // Instead of setting return_code = 1 and continuing, exit immediately. + // High risk of segfaulting / weird behavior when inference fails. + return 1; + } + + size_t iteration = iterations.back(); + + std::vector<double> all_latencies; + auto start_time = infer_request_queues.at(0)->get_start_time(); + auto end_time = infer_request_queues.at(0)->get_end_time(); + for (auto& infer_request_queue : infer_request_queues) { + auto& latencies = infer_request_queue->get_latencies(); + all_latencies.insert(all_latencies.end(), latencies.begin(), latencies.end()); + start_time = std::min(start_time, infer_request_queue->get_start_time()); + end_time = std::max(end_time, infer_request_queue->get_end_time()); + } + double latency = GetMedianValue<double>(all_latencies); + double total_duration = std::chrono::duration_cast<ns>(end_time - start_time).count() * 0.000001; + double total_fps = (FLAGS_api == "sync") + ? compiled_models.size() * batch_size * 1000.0 / latency + : compiled_models.size() * batch_size * 1000.0 * iteration / total_duration; + + int ip_num_instances = 0; + double ip_duration = 0.0; + double ip_fps = 0.0; + double ip_fps_per_fmax = 0.0; + double estimated_ipFps = 0.0; + double estimated_ipFpsPerFmax = 0.0; + double fmax_core = -1.0; + double estimated_ipFps_assumed_fmax = 0.0; + if (device_name.find("FPGA") != std::string::npos) { + ip_num_instances = core.get_property("FPGA", "COREDLA_NUM_INSTANCES").as<int>(); + // even if hardware has 2 instances, only 1 instance actually gets used if only 1 inference is performed + size_t ip_num_instances_used = std::min((size_t)ip_num_instances, iteration); + ip_duration = core.get_property("FPGA", "IP_ACTIVE_TIME").as<double>(); + if (ip_duration) { + if (ip_duration != 0.0) { + ip_fps = (FLAGS_api == "sync") + ? compiled_models.size() * batch_size * 1000.0 / latency / ip_num_instances_used + : compiled_models.size() * batch_size * 1000.0 * iteration / ip_duration / ip_num_instances_used; + } + fmax_core = core.get_property("FPGA", "COREDLA_CLOCK_FREQUENCY").as<double>(); + if (fmax_core > 0.0) { + ip_fps_per_fmax = ip_fps / fmax_core; + } else { + slog::warn << "Warning: could not estimate clk_dla frequency on the FPGA" << slog::endl; + } + } + + if (FLAGS_perf_est && (device_name.find("FPGA") != std::string::npos)) { + if (is_model_compiled) { + // Ahead of Time Flow: getting the imported, precalculated performance estimate + estimated_ipFps = core.get_property("FPGA", "IMPORT_PERFORMANCE_EST").as<double>(); + if (estimated_ipFps < 0) + slog::warn << "Missing performance estimation from at least one of the compiled graphs" << slog::endl; + estimated_ipFps_assumed_fmax = core.get_property("FPGA", "IMPORT_PERFORMANCE_EST_ASSUMED_FMAX").as<double>(); + } else { +#ifndef DISABLE_JIT + // Just In Time Flow: running the performance estimate + if (fmax_core > 0.0) { +#if defined(_WIN32) || defined(_WIN64) + _putenv_s("PERF_EST_COREDLA_FMAX", double_to_string(fmax_core).c_str()); + _putenv_s("PERF_EST_PE_FMAX", double_to_string(fmax_core).c_str()); +#else + setenv("PERF_EST_COREDLA_FMAX", double_to_string(fmax_core).c_str(), true); + setenv("PERF_EST_PE_FMAX", double_to_string(fmax_core).c_str(), true); +#endif + estimated_ipFps_assumed_fmax = fmax_core; + } else { +// In case the fmax_core variable is not set, we use the estimated fmax values for AGX7 and A10. +// This if statement is just defensive programming for a condition that should not happen. +#ifdef DE10_AGILEX + estimated_ipFps_assumed_fmax = GetEnvOrDefault("PERF_EST_COREDLA_FMAX", 500); // AGX7 fMAX estimate +#else + estimated_ipFps_assumed_fmax = GetEnvOrDefault("PERF_EST_COREDLA_FMAX", 265); // A10 fMAX estimate +#endif + slog::warn + << "Warning: could not estimate clk_dla frequency on the FPGA, setting the fmax to default value." + << slog::endl; +#if defined(_WIN32) || defined(_WIN64) + _putenv_s("PERF_EST_COREDLA_FMAX", double_to_string(estimated_ipFps_assumed_fmax).c_str()); + _putenv_s("PERF_EST_PE_FMAX", double_to_string(estimated_ipFps_assumed_fmax).c_str()); +#else + setenv("PERF_EST_COREDLA_FMAX", double_to_string(estimated_ipFps_assumed_fmax).c_str(), true); + setenv("PERF_EST_PE_FMAX", double_to_string(estimated_ipFps_assumed_fmax).c_str(), true); +#endif + } + estimated_ipFps = core.get_property("FPGA", "PLUGIN_PERFORMANCE_EST").as<double>(); +#endif + } + estimated_ipFpsPerFmax = estimated_ipFps / estimated_ipFps_assumed_fmax; + } + } + + if (statistics) { + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, + { + {"total execution time (ms)", double_to_string(total_duration)}, + {"IP active time (ms)", double_to_string(ip_duration)}, + {"total number of iterations", std::to_string(iteration)}, + }); + if (device_name.find("MULTI") == std::string::npos) { + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, + { + {"latency (ms)", double_to_string(latency)}, + }); + } + statistics->addParameters( + StatisticsReport::Category::EXECUTION_RESULTS, + {{"throughput", double_to_string(total_fps)}, {"IP throughput", double_to_string(ip_fps)}}); + } + + progress_bar.finish(); + + // ----------------- 11. Dumping statistics report ------------------------------------------------------------- + next_step(); + + if (perf_count || !perf_count_sort.empty()) { + std::vector<std::vector<ov::ProfilingInfo>> perfCounts; + for (size_t ireq = 0; ireq < nireq; ireq++) { + auto reqPerfCounts = infer_request_queues.at(0)->requests[ireq]->get_performance_counts(); + perfCounts.push_back(reqPerfCounts); + } + if (statistics) { + if (perf_count_sort == "sort") { + statistics->printPerfCountersSort(perfCounts, "sort"); + } else if (perf_count_sort == "simple_sort") { + statistics->printPerfCountersSort(perfCounts, "simple_sort"); + } else { + statistics->printPerfCountersSort(perfCounts, "no_sort"); + } + } + } + + // dla_benchmark originally also implemented more detailed performance + // statistics via InferRequest's getPerformanceCounts function + // We did not support it, and removed it. If we want to re-implement it + // looking at the latest version of OpenVINO's benchmark_app or our git + // history would be a good starting point + if (statistics) { + statistics->dump(); + } + + std::cout << "count: " << iteration << " iterations" << std::endl; + std::cout << "system duration: " << double_to_string(total_duration) << " ms" << std::endl; + if (ip_duration != 0.0) std::cout << "IP duration: " << double_to_string(ip_duration) << " ms" << std::endl; + if (device_name.find("MULTI") == std::string::npos) + std::cout << "latency: " << double_to_string(latency) << " ms" << std::endl; + std::cout << "system throughput: " << double_to_string(total_fps) << " FPS" << std::endl; + if (ip_num_instances != 0) std::cout << "number of hardware instances: " << ip_num_instances << std::endl; + if (compiled_models.size() != 0) + std::cout << "number of network instances: " << compiled_models.size() << std::endl; + if (ip_fps != 0.0) std::cout << "IP throughput per instance: " << double_to_string(ip_fps) << " FPS" << std::endl; + if (ip_fps_per_fmax != 0.0) + std::cout << "IP throughput per fmax per instance: " << double_to_string(ip_fps_per_fmax) << " FPS/MHz" + << std::endl; + if (fmax_core > 0.0) std::cout << "IP clock frequency: " << double_to_string(fmax_core) << " MHz" << std::endl; + if (estimated_ipFps != 0.0) + std::cout << "estimated IP throughput per instance: " << double_to_string(estimated_ipFps) << " FPS (" + << (int)estimated_ipFps_assumed_fmax << " MHz assumed)" << std::endl; + if (estimated_ipFpsPerFmax != 0.0) + std::cout << "estimated IP throughput per fmax per instance: " << double_to_string(estimated_ipFpsPerFmax) + << " FPS/MHz" << std::endl; + + // ----------------- 12. Dumping output values ------------------------------------------------------------- + next_step(); + + if (FLAGS_dump_output) { + for (size_t i = 0; i < compiled_models.size(); i++) { + std::vector<ov::Output<const ov::Node>> output_info = compiled_models[i]->outputs(); + // For multi-outputs: Sort to ensure the order of each tensor dump aligns with the ground truth files + std::sort(output_info.begin(), output_info.end(), CompareOutputNodeNames); + const auto& output_tensors_map = output_tensors[i]; + // A flag regarding whether we can dump output tensor in a text file due to unsupported layout. + // This flag is set at first during dumping. + bool can_dump_txt = true; + bool can_dump_layout_info_in_txt = true; + // dump output tensor as bin, which can be loaded using Python Numpy + std::regex pattern("\\{batch\\}"); + std::string results_bin_file_name = output_dir + "result_{batch}.bin"; + // dump output tensor as text + // backward compatibility support for old regtests that used only one graph + std::string results_txt_file_name = output_dir + "result.txt"; + std::string results_boundaries_file_name = output_dir + "result_tensor_boundaries.txt"; + // dump inference arguments and metadata as JSON + std::string results_meta_file_name = output_dir + "result_meta.json"; + + if (compiled_models.size() > 1) { + results_bin_file_name = output_dir + topology_names[i] + "_result_{batch}.bin"; + results_txt_file_name = output_dir + topology_names[i] + "_result.txt"; + results_boundaries_file_name = output_dir + topology_names[i] + "_result_tensor_boundaries.txt"; + results_meta_file_name = output_dir + topology_names[i] + "_result_meta.json"; + } + + slog::info << "Dumping result of " << topology_names[i] + << " to " << results_txt_file_name << slog::endl; + slog::info << "Dumping per-batch result (raw output) of " << topology_names[i] + << " to " << results_bin_file_name << slog::endl; + slog::info << "Dumping inference meta data of " << topology_names[i] + << " to " << results_meta_file_name << slog::endl; + + std::ofstream result_txt_file(results_txt_file_name); + std::ofstream results_boundaries(results_boundaries_file_name); + std::ofstream result_meta_file(results_meta_file_name); + + dla_benchmark::InferenceMetaData result_metadata; + result_metadata.input_files = multi_input_files.at(i); // all input files in -i + result_metadata.groundtruth_loc = FLAGS_groundtruth_loc; + result_metadata.batch_size = FLAGS_batch_size; + result_metadata.niter = niter; + result_metadata.nireq = nireq; + result_metadata.model_input_info = input_infos[i]; + dla_benchmark::OutputsInfoVec model_output_info; + + uint32_t current_lines = 1; + size_t max_allowed_megabytes_to_dump = FLAGS_max_output_file_size; + + for (uint32_t batch = 0; batch < num_batches; batch++) { + std::string per_batch_results_bin_file_name = std::regex_replace(results_bin_file_name, + pattern, + std::to_string(batch)); + std::ofstream per_batch_results_bin_file(per_batch_results_bin_file_name, std::ios::binary); + + for (const auto& item : output_info) { + auto tensor = output_tensors_map.at(item.get_any_name()).at(batch); + unsigned int output_size = tensor.get_size() / batch_size; + + const ov::Layout& layout = ov::layout::get_layout(item); + const auto& shape = tensor.get_shape(); + const std::string& name = item.get_any_name(); + size_t total_bytes_to_dump = tensor.get_size() * niter * sizeof(float); + + if (can_dump_txt) { + // if we cannot dump as a text file, we set can_dump_txt flag to false and write the one-time message + if (total_bytes_to_dump > max_allowed_megabytes_to_dump * BYTE_TO_MEGABYTE) { + can_dump_txt = false; + std::string msg = "Output tensor (" + std::to_string(total_bytes_to_dump / BYTE_TO_MEGABYTE) + + " MB) " + "is too large to dump. Change environmental variable MAX_DUMP_OUTPUT_TXT (default " + + std::to_string(FLAGS_max_output_file_size) + " MB) to allow dumping larger tensors"; + slog::warn << msg << slog::endl; + result_txt_file << msg; + } else { + if (can_dump_layout_info_in_txt && shape.size() != 2 && shape.size() != 4 && shape.size() != 5) { + can_dump_layout_info_in_txt = false; + slog::warn << "Output data tensor of rank that is not 2, 4 or 5. layout info will not be dumped in " + << "result.txt." << slog::endl; + } + // Otherwise, dump text and write to the result_tensor_boundaries.txt with additional information + // about the result.txt file + results_boundaries << name << ": Line " << current_lines << " to " + << "line " << current_lines + output_size - 1 << std::endl; + results_boundaries << name << " output layout: " << layout.to_string() << std::endl; + results_boundaries << name << " output dimension:"; + for (unsigned int dim = 0; dim < shape.size(); dim++) { + results_boundaries << " " << shape[dim]; + } + results_boundaries << std::endl; + current_lines = current_lines + output_size; + DumpResultTxtFile(tensor, item, output_size, result_txt_file); + } + } + DumpResultBinFile(tensor, per_batch_results_bin_file); + + if (batch == 0) { + // all batches should have the same output info + dla_benchmark::OutputInfo output_info; + output_info.name = name; + output_info.shape = shape; + model_output_info.push_back(output_info); + } + } + per_batch_results_bin_file.close(); + } + + result_metadata.model_output_info = model_output_info; + DumpResultMetaJSONFile(result_metadata, result_meta_file); + result_txt_file.close(); + results_boundaries.close(); + result_meta_file.close(); + } + const std::string throughput_file_name = output_dir + "throughput_report.txt"; + std::ofstream throughput_file; + throughput_file.open(throughput_file_name); + throughput_file << "Throughput : " << total_fps << " fps" << std::endl; + throughput_file << "Batch Size : " << batch_size << std::endl; + throughput_file << "Graph number : " << compiled_models.size() << std::endl; + throughput_file << "Num Batches : " << num_batches << std::endl; + throughput_file.close(); + + // Append throughput to dataset + // Check both gz and non gz versions + std::string dataset_gz_file_name = "data.csv.gz"; + append_value_if_incomplete_to_csv(dataset_gz_file_name, ip_fps); + std::string dataset_file_name = "data.csv"; + append_value_if_incomplete_to_csv(dataset_file_name, ip_fps); + } + + // Calculate top 1, top 5 results + if (FLAGS_groundtruth_loc != "") { + auto groundtruth_files = split(FLAGS_groundtruth_loc, MULTIGRAPH_SEP); + for (size_t i = 0; i < compiled_models.size(); i++) { + // This flag `FLAGS_enable_object_detection_ap` enables accuracy checking subroutine that + // gives the mAP and COCO AP scores. These scores are two of the main detection evaluation + // metrics used in the Common Objects in Context contest, https://cocodataset.org/#detection-eval. + + std::vector<ov::Output<const ov::Node>> output_info = compiled_models[i]->outputs(); + // For multi-outputs: Sort to ensure the order of each tensor dump aligns with the ground truth files + std::sort(output_info.begin(), output_info.end(), CompareOutputNodeNames); + // Run the default top-1, top-5 evaluation routine if AP scores are not required. + if (!FLAGS_enable_object_detection_ap) { + if (groundtruth_files.size() <= i) { + slog::warn << "Missing ground truth file for " << topology_names[i] << "! SKIPPED" << slog::endl; + continue; // Print warnings for all missing ground truth graphs; + } + slog::info << "Comparing ground truth file " << groundtruth_files[i] << " with network " << topology_names[i] + << slog::endl; + // captures the results in higher precision for accuracy analysis + std::vector<float> results; + const auto& output_tensors_map = output_tensors[i]; + for (uint32_t batch = 0; batch < num_batches; batch++) { + for (unsigned int img = 0; img < batch_size; img++) { + for (const auto& item : output_info) { + auto tensor = output_tensors_map.at(item.get_any_name()).at(batch); + auto tensor_data = tensor.data<float>(); + unsigned int output_size = tensor.get_size() / batch_size; + size_t offset = img * output_size; + for (unsigned int j = 0; j < output_size; j++) { + results.push_back(tensor_data[j + offset]); + } + } + } + } + bool passed = TopResultsAnalyser::get_top_results(groundtruth_files[i], results, batch_size * num_batches); + if (passed) { + slog::info << "Get top results for \"" << topology_names[i] << "\" graph passed" << slog::endl; + } else { + // return 4 indicates that the accuracy of the result was below the threshold + return_code = 4; + } + } else { + // Runs the accuracy checking routine if AP scores are required. + set_runtime(FLAGS_yolo_version, FLAGS_niter, batch_size_flag, FLAGS_i, FLAGS_groundtruth_loc); + std::pair<double, double> res = + validate_yolo_wrapper(output_tensors[i], output_info, multi_input_files.at(0)); + std::cout << std::endl; + slog::info << "Batch metrics results:" << slog::endl; + std::cout << "Detection - mAP@0.5: " << std::setprecision(6) << res.first * 100 << "%" << std::endl; + std::cout << "Detection - mAP@0.5:0.95: " << std::setprecision(6) << res.second * 100 << "%" << std::endl; + } + } + } + // Output Debug Network Info if COREDLA_TEST_DEBUG_NETWORK is set + ReadDebugNetworkInfo(core); + if (FLAGS_report_lsu_counters) { + PrintLSUCounterInfo(core); + } + if (return_code) return return_code; + } catch (const std::exception& ex) { + slog::err << ex.what() << slog::endl; + + if (statistics) { + statistics->addParameters(StatisticsReport::Category::EXECUTION_RESULTS, + { + {"Error during dla_benchmark: ", ex.what()}, + }); + statistics->dump(); + } + + return 3; + } + + return 0; + // Bypass long function lint check + // NOLINTNEXTLINE(readability/fn_size) +} |