diff options
Diffstat (limited to 'python/openvino/runtime/common/demo_utils')
23 files changed, 2553 insertions, 0 deletions
diff --git a/python/openvino/runtime/common/demo_utils/CMakeLists.txt b/python/openvino/runtime/common/demo_utils/CMakeLists.txt new file mode 100644 index 0000000..b79d72a --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright (C) 2018-2020 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +file(GLOB_RECURSE HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/include/*") +file(GLOB_RECURSE SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/*") + +source_group("src" FILES ${SOURCES}) +source_group("include" FILES ${HEADERS}) + +add_library(utils STATIC ${HEADERS} ${SOURCES}) +target_include_directories(utils PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" + "$ENV{COREDLA_ROOT}/dla_plugin/inc/") +target_link_libraries(utils PRIVATE openvino::runtime opencv_core opencv_imgcodecs opencv_videoio ie_samples_utils) diff --git a/python/openvino/runtime/common/demo_utils/include/utils/args_helper.hpp b/python/openvino/runtime/common/demo_utils/include/utils/args_helper.hpp new file mode 100644 index 0000000..7a638cc --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/include/utils/args_helper.hpp @@ -0,0 +1,43 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief a header file with common samples functionality + * @file args_helper.hpp + */ + +#pragma once + +#include <map> +#include <set> +#include <string> +#include <vector> + +#include <opencv2/core/types.hpp> +#include <openvino/openvino.hpp> + +/** +* @brief This function checks input args and existence of specified files in a given folder +* @param arg path to a file to be checked for existence +* @return files updated vector of verified input files +*/ +void readInputFilesArguments(std::vector<std::string>& files, const std::string& arg); + +/** +* @brief This function finds -i/--i key in input args +* It's necessary to process multiple values for single key +* @return files updated vector of verified input files +*/ +void parseInputFilesArguments(std::vector<std::string>& files); + +std::vector<std::string> split(const std::string& s, char delim); + +std::vector<std::string> parseDevices(const std::string& device_string); + +std::map<std::string, int32_t> parseValuePerDevice(const std::set<std::string>& devices, + const std::string& values_string); + +cv::Size stringToSize(const std::string& str); + +std::map<std::string, ov::Layout> parseLayoutString(const std::string& layout_string); diff --git a/python/openvino/runtime/common/demo_utils/include/utils/common.hpp b/python/openvino/runtime/common/demo_utils/include/utils/common.hpp new file mode 100644 index 0000000..dbe7cf0 --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/include/utils/common.hpp @@ -0,0 +1,190 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief a header file with common samples functionality + * @file common.hpp + */ + +#pragma once + +#include <iostream> +#include <string> +#include <utility> +#include <vector> + +#include <openvino/openvino.hpp> +#include "utils/slog.hpp" +#include "utils/args_helper.hpp" + +#ifndef UNUSED +#ifdef _WIN32 +#define UNUSED +#else +#define UNUSED __attribute__((unused)) +#endif +#endif + +template <typename T, std::size_t N> +constexpr std::size_t arraySize(const T(&)[N]) noexcept { + return N; +} + +static inline void catcher() noexcept { + if (std::current_exception()) { + try { + std::rethrow_exception(std::current_exception()); + } catch (const std::exception& error) { + slog::err << error.what() << slog::endl; + } catch (...) { + slog::err << "Non-exception object thrown" << slog::endl; + } + std::exit(1); + } + std::abort(); +} + +template <typename T> +T clamp(T value, T low, T high) { + return value < low ? low : (value > high ? high : value); +} + +inline slog::LogStream& operator<<(slog::LogStream& os, const ov::Version& version) { + return os << "OpenVINO" << slog::endl + << "\tversion: " << OPENVINO_VERSION_MAJOR << "." << OPENVINO_VERSION_MINOR << "." << OPENVINO_VERSION_PATCH << slog::endl + << "\tbuild: " << version.buildNumber; +} + +/** + * @class Color + * @brief A Color class stores channels of a given color + */ +class Color { +private: + unsigned char _r; + unsigned char _g; + unsigned char _b; + +public: + /** + * A default constructor. + * @param r - value for red channel + * @param g - value for green channel + * @param b - value for blue channel + */ + Color(unsigned char r, + unsigned char g, + unsigned char b) : _r(r), _g(g), _b(b) {} + + inline unsigned char red() const { + return _r; + } + + inline unsigned char blue() const { + return _b; + } + + inline unsigned char green() const { + return _g; + } +}; + +// Known colors for training classes from the Cityscapes dataset +static UNUSED const Color CITYSCAPES_COLORS[] = { + { 128, 64, 128 }, + { 232, 35, 244 }, + { 70, 70, 70 }, + { 156, 102, 102 }, + { 153, 153, 190 }, + { 153, 153, 153 }, + { 30, 170, 250 }, + { 0, 220, 220 }, + { 35, 142, 107 }, + { 152, 251, 152 }, + { 180, 130, 70 }, + { 60, 20, 220 }, + { 0, 0, 255 }, + { 142, 0, 0 }, + { 70, 0, 0 }, + { 100, 60, 0 }, + { 90, 0, 0 }, + { 230, 0, 0 }, + { 32, 11, 119 }, + { 0, 74, 111 }, + { 81, 0, 81 } +}; + +inline void showAvailableDevices() { + ov::Core core; + std::vector<std::string> devices = core.get_available_devices(); + + std::cout << "Available devices:"; + for (const auto& device : devices) { + std::cout << ' ' << device; + } + std::cout << std::endl; +} + +inline std::string fileNameNoExt(const std::string& filepath) { + auto pos = filepath.rfind('.'); + if (pos == std::string::npos) return filepath; + return filepath.substr(0, pos); +} + +inline void logCompiledModelInfo( + const ov::CompiledModel& compiledModel, + const std::string& modelName, + const std::string& deviceName, + const std::string& modelType = "") { + slog::info << "The " << modelType << (modelType.empty() ? "" : " ") << "model " << modelName << " is loaded to " << deviceName << slog::endl; + std::set<std::string> devices; + for (const std::string& device : parseDevices(deviceName)) { + devices.insert(device); + } + + if (devices.find("AUTO") == devices.end()) { // do not print info for AUTO device + for (const auto& device : devices) { + try { + slog::info << "\tDevice: " << device << slog::endl; + int32_t nstreams = compiledModel.get_property(ov::streams::num); + slog::info << "\t\tNumber of streams: " << nstreams << slog::endl; + if (device == "CPU") { + int32_t nthreads = compiledModel.get_property(ov::inference_num_threads); + slog::info << "\t\tNumber of threads: " << (nthreads == 0 ? "AUTO" : std::to_string(nthreads)) << slog::endl; + } + } + catch (const ov::Exception&) {} + } + } +} + +inline void logBasicModelInfo(const std::shared_ptr<ov::Model>& model) { + slog::info << "Model name: " << model->get_friendly_name() << slog::endl; + + // Dump information about model inputs/outputs + ov::OutputVector inputs = model->inputs(); + ov::OutputVector outputs = model->outputs(); + + slog::info << "\tInputs: " << slog::endl; + for (const ov::Output<ov::Node>& input : inputs) { + const std::string name = input.get_any_name(); + const ov::element::Type type = input.get_element_type(); + const ov::PartialShape shape = input.get_partial_shape(); + const ov::Layout layout = ov::layout::get_layout(input); + + slog::info << "\t\t" << name << ", " << type << ", " << shape << ", " << layout.to_string() << slog::endl; + } + + slog::info << "\tOutputs: " << slog::endl; + for (const ov::Output<ov::Node>& output : outputs) { + const std::string name = output.get_any_name(); + const ov::element::Type type = output.get_element_type(); + const ov::PartialShape shape = output.get_partial_shape(); + const ov::Layout layout = ov::layout::get_layout(output); + + slog::info << "\t\t" << name << ", " << type << ", " << shape << ", " << layout.to_string() << slog::endl; + } + + return; +} diff --git a/python/openvino/runtime/common/demo_utils/include/utils/config_factory.h b/python/openvino/runtime/common/demo_utils/include/utils/config_factory.h new file mode 100644 index 0000000..c7440b5 --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/include/utils/config_factory.h @@ -0,0 +1,52 @@ +/* +// Copyright (C) 2020-2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#pragma once +#include <stdint.h> + +#include <map> +#include <set> +#include <string> + +#include <openvino/openvino.hpp> + +struct ModelConfig { + std::string deviceName; + std::string cpuExtensionsPath; + std::string clKernelsConfigPath; + std::string fpgaArchPath; + unsigned int maxAsyncRequests; + ov::AnyMap compiledModelConfig; + + std::set<std::string> getDevices(); + std::map<std::string, std::string> getLegacyConfig(); + +protected: + std::set<std::string> devices; +}; + +class ConfigFactory { +public: + static ModelConfig getUserConfig(const std::string& flags_d, + uint32_t flags_nireq, + const std::string& flags_nstreams, + uint32_t flags_nthreads, + const std::string &flags_arch); + static ModelConfig getMinLatencyConfig(const std::string& flags_d, uint32_t flags_nireq); + +protected: + static ModelConfig getCommonConfig(const std::string& flags_d, uint32_t flags_nireq); +}; diff --git a/python/openvino/runtime/common/demo_utils/include/utils/default_flags.hpp b/python/openvino/runtime/common/demo_utils/include/utils/default_flags.hpp new file mode 100644 index 0000000..83c32c2 --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/include/utils/default_flags.hpp @@ -0,0 +1,21 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include <gflags/gflags.h> + +#define DEFINE_INPUT_FLAGS \ +DEFINE_string(i, "", input_message); \ +DEFINE_bool(loop, false, loop_message); + +#define DEFINE_OUTPUT_FLAGS \ +DEFINE_string(o, "", output_message); \ +DEFINE_int32(limit, 1000, limit_message); + +static const char input_message[] = "Required. An input to process. The input must be a single image, a folder of " + "images, video file or camera id."; +static const char loop_message[] = "Optional. Enable reading the input in a loop."; +static const char output_message[] = "Optional. Name of the output file(s) to save."; +static const char limit_message[] = "Optional. Number of frames to store in output. If 0 is set, all frames are stored."; diff --git a/python/openvino/runtime/common/demo_utils/include/utils/grid_mat.hpp b/python/openvino/runtime/common/demo_utils/include/utils/grid_mat.hpp new file mode 100644 index 0000000..7d46d2b --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/include/utils/grid_mat.hpp @@ -0,0 +1,127 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include <algorithm> +#include <set> +#include <string> +#include <vector> + +#include <opencv2/core/core.hpp> + +class GridMat { +public: + cv::Mat outimg; + + explicit GridMat(const std::vector<cv::Size>& sizes, const cv::Size maxDisp = cv::Size{1920, 1080}) { + size_t maxWidth = 0; + size_t maxHeight = 0; + for (size_t i = 0; i < sizes.size(); i++) { + maxWidth = std::max(maxWidth, static_cast<size_t>(sizes[i].width)); + maxHeight = std::max(maxHeight, static_cast<size_t>(sizes[i].height)); + } + if (0 == maxWidth || 0 == maxHeight) { + throw std::invalid_argument("Input resolution must not be zero."); + } + + size_t nGridCols = static_cast<size_t>(ceil(sqrt(static_cast<float>(sizes.size())))); + size_t nGridRows = (sizes.size() - 1) / nGridCols + 1; + size_t gridMaxWidth = static_cast<size_t>(maxDisp.width/nGridCols); + size_t gridMaxHeight = static_cast<size_t>(maxDisp.height/nGridRows); + + float scaleWidth = static_cast<float>(gridMaxWidth) / maxWidth; + float scaleHeight = static_cast<float>(gridMaxHeight) / maxHeight; + float scaleFactor = std::min(1.f, std::min(scaleWidth, scaleHeight)); + + cellSize.width = static_cast<int>(maxWidth * scaleFactor); + cellSize.height = static_cast<int>(maxHeight * scaleFactor); + + for (size_t i = 0; i < sizes.size(); i++) { + cv::Point p; + p.x = cellSize.width * (i % nGridCols); + p.y = cellSize.height * (i / nGridCols); + points.push_back(p); + } + + outimg.create(cellSize.height * nGridRows, cellSize.width * nGridCols, CV_8UC3); + outimg.setTo(0); + clear(); + } + + cv::Size getCellSize() { + return cellSize; + } + + void fill(std::vector<cv::Mat>& frames) { + if (frames.size() > points.size()) { + throw std::logic_error("Cannot display " + std::to_string(frames.size()) + " channels in a grid with " + std::to_string(points.size()) + " cells"); + } + + for (size_t i = 0; i < frames.size(); i++) { + cv::Mat cell = outimg(cv::Rect(points[i].x, points[i].y, cellSize.width, cellSize.height)); + + if ((cellSize.width == frames[i].cols) && (cellSize.height == frames[i].rows)) { + frames[i].copyTo(cell); + } else if ((cellSize.width > frames[i].cols) && (cellSize.height > frames[i].rows)) { + frames[i].copyTo(cell(cv::Rect(0, 0, frames[i].cols, frames[i].rows))); + } else { + cv::resize(frames[i], cell, cellSize); + } + } + unupdatedSourceIDs.clear(); + } + + void update(const cv::Mat& frame, const size_t sourceID) { + const cv::Mat& cell = outimg(cv::Rect(points[sourceID], cellSize)); + + if ((cellSize.width == frame.cols) && (cellSize.height == frame.rows)) { + frame.copyTo(cell); + } else if ((cellSize.width > frame.cols) && (cellSize.height > frame.rows)) { + frame.copyTo(cell(cv::Rect(0, 0, frame.cols, frame.rows))); + } else { + cv::resize(frame, cell, cellSize); + } + unupdatedSourceIDs.erase(unupdatedSourceIDs.find(sourceID)); + } + + bool isFilled() const noexcept { + return unupdatedSourceIDs.empty(); + } + void clear() { + size_t counter = 0; + std::generate_n(std::inserter(unupdatedSourceIDs, unupdatedSourceIDs.end()), points.size(), [&counter]{return counter++;}); + } + std::set<size_t> getUnupdatedSourceIDs() const noexcept { + return unupdatedSourceIDs; + } + cv::Mat getMat() const noexcept { + return outimg; + } + +private: + cv::Size cellSize; + std::set<size_t> unupdatedSourceIDs; + std::vector<cv::Point> points; +}; + +void fillROIColor(cv::Mat& displayImage, cv::Rect roi, cv::Scalar color, double opacity) { + if (opacity > 0) { + roi = roi & cv::Rect(0, 0, displayImage.cols, displayImage.rows); + cv::Mat textROI = displayImage(roi); + cv::addWeighted(color, opacity, textROI, 1.0 - opacity , 0.0, textROI); + } +} + +void putTextOnImage(cv::Mat& displayImage, std::string str, cv::Point p, + cv::HersheyFonts font, double fontScale, cv::Scalar color, + int thickness = 1, cv::Scalar bgcolor = cv::Scalar(), + double opacity = 0) { + int baseline = 0; + cv::Size textSize = cv::getTextSize(str, font, 0.5, 1, &baseline); + fillROIColor(displayImage, cv::Rect(cv::Point(p.x, p.y + baseline), + cv::Point(p.x + textSize.width, p.y - textSize.height)), + bgcolor, opacity); + cv::putText(displayImage, str, p, font, fontScale, color, thickness); +} diff --git a/python/openvino/runtime/common/demo_utils/include/utils/image_utils.h b/python/openvino/runtime/common/demo_utils/include/utils/image_utils.h new file mode 100644 index 0000000..2731a9a --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/include/utils/image_utils.h @@ -0,0 +1,29 @@ +/* +// Copyright (C) 2021-2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#pragma once + +#include <opencv2/opencv.hpp> + +enum RESIZE_MODE { + RESIZE_FILL, + RESIZE_KEEP_ASPECT, + RESIZE_KEEP_ASPECT_LETTERBOX +}; + +cv::Mat resizeImageExt(const cv::Mat& mat, int width, int height, RESIZE_MODE resizeMode = RESIZE_FILL, + cv::InterpolationFlags interpolationMode = cv::INTER_LINEAR, cv::Rect* roi = nullptr, + cv::Scalar BorderConstant = cv::Scalar(0, 0, 0)); diff --git a/python/openvino/runtime/common/demo_utils/include/utils/images_capture.h b/python/openvino/runtime/common/demo_utils/include/utils/images_capture.h new file mode 100644 index 0000000..f2afdfc --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/include/utils/images_capture.h @@ -0,0 +1,53 @@ +// Copyright (C) 2020-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once +#include <stddef.h> + +#include <limits> +#include <memory> +#include <string> + +#include <opencv2/core.hpp> + +#include "utils/performance_metrics.hpp" + +enum class read_type { efficient, safe }; + +class ImagesCapture { +public: + const bool loop; + + ImagesCapture(bool loop) : loop{loop} {} + virtual double fps() const = 0; + virtual cv::Mat read() = 0; + virtual std::string getType() const = 0; + const PerformanceMetrics& getMetrics() { + return readerMetrics; + } + virtual ~ImagesCapture() = default; + +protected: + PerformanceMetrics readerMetrics; +}; + +// An advanced version of +// try { +// return cv::VideoCapture(std::stoi(input)); +// } catch (const std::invalid_argument&) { +// return cv::VideoCapture(input); +// } catch (const std::out_of_range&) { +// return cv::VideoCapture(input); +// } +// Some VideoCapture backends continue owning the video buffer under cv::Mat. safe_copy forses to return a copy from +// read() +// https://github.com/opencv/opencv/blob/46e1560678dba83d25d309d8fbce01c40f21b7be/modules/gapi/include/opencv2/gapi/streaming/cap.hpp#L72-L76 +std::unique_ptr<ImagesCapture> openImagesCapture( + const std::string& input, + bool loop, + read_type type = read_type::efficient, + size_t initialImageId = 0, + size_t readLengthLimit = std::numeric_limits<size_t>::max(), // General option + cv::Size cameraResolution = {1280, 720} + ); diff --git a/python/openvino/runtime/common/demo_utils/include/utils/input_wrappers.hpp b/python/openvino/runtime/common/demo_utils/include/utils/input_wrappers.hpp new file mode 100644 index 0000000..eff38a7 --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/include/utils/input_wrappers.hpp @@ -0,0 +1,149 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include <list> +#include <memory> +#include <set> +#include <thread> +#include <vector> +#include <queue> + +#include <opencv2/opencv.hpp> + +class InputChannel; + +class IInputSource { +public: + virtual bool read(cv::Mat& mat, const std::shared_ptr<InputChannel>& caller) = 0; + virtual void addSubscriber(const std::weak_ptr<InputChannel>& inputChannel) = 0; + virtual cv::Size getSize() = 0; + virtual void lock() { + sourceLock.lock(); + } + virtual void unlock() { + sourceLock.unlock(); + } + virtual ~IInputSource() = default; +private: + std::mutex sourceLock; +}; + +class InputChannel: public std::enable_shared_from_this<InputChannel> { // note: public inheritance +public: + InputChannel(const InputChannel&) = delete; + InputChannel& operator=(const InputChannel&) = delete; + static std::shared_ptr<InputChannel> create(const std::shared_ptr<IInputSource>& source) { + auto tmp = std::shared_ptr<InputChannel>(new InputChannel(source)); + source->addSubscriber(tmp); + return tmp; + } + bool read(cv::Mat& mat) { + readQueueMutex.lock(); + if (readQueue.empty()) { + readQueueMutex.unlock(); + source->lock(); + readQueueMutex.lock(); + if (readQueue.empty()) { + bool res = source->read(mat, shared_from_this()); + readQueueMutex.unlock(); + source->unlock(); + return res; + } else { + source->unlock(); + } + } + mat = readQueue.front().clone(); + readQueue.pop(); + readQueueMutex.unlock(); + return true; + } + void push(const cv::Mat& mat) { + readQueueMutex.lock(); + readQueue.push(mat); + readQueueMutex.unlock(); + } + cv::Size getSize() { + return source->getSize(); + } + +private: + explicit InputChannel(const std::shared_ptr<IInputSource>& source): source{source} {} + std::shared_ptr<IInputSource> source; + std::queue<cv::Mat, std::list<cv::Mat>> readQueue; + std::mutex readQueueMutex; +}; + +class VideoCaptureSource: public IInputSource { +public: + VideoCaptureSource(const cv::VideoCapture& videoCapture, bool loop): videoCapture{videoCapture}, loop{loop}, + imSize{static_cast<int>(videoCapture.get(cv::CAP_PROP_FRAME_WIDTH)), static_cast<int>(videoCapture.get(cv::CAP_PROP_FRAME_HEIGHT))} {} + bool read(cv::Mat& mat, const std::shared_ptr<InputChannel>& caller) override { + if (!videoCapture.read(mat)) { + if (loop) { + videoCapture.set(cv::CAP_PROP_POS_FRAMES, 0); + videoCapture.read(mat); + } else { + return false; + } + } + if (1 != subscribedInputChannels.size()) { + cv::Mat shared = mat.clone(); + for (const std::weak_ptr<InputChannel>& weakInputChannel : subscribedInputChannels) { + try { + std::shared_ptr<InputChannel> sharedInputChannel = std::shared_ptr<InputChannel>(weakInputChannel); + if (caller != sharedInputChannel) { + sharedInputChannel->push(shared); + } + } catch (const std::bad_weak_ptr&) {} + } + } + return true; + } + void addSubscriber(const std::weak_ptr<InputChannel>& inputChannel) override { + subscribedInputChannels.push_back(inputChannel); + } + cv::Size getSize() override { + return imSize; + } + +private: + std::vector<std::weak_ptr<InputChannel>> subscribedInputChannels; + cv::VideoCapture videoCapture; + bool loop; + cv::Size imSize; +}; + +class ImageSource: public IInputSource { +public: + ImageSource(const cv::Mat& im, bool loop): im{im.clone()}, loop{loop} {} // clone to avoid image changing + bool read(cv::Mat& mat, const std::shared_ptr<InputChannel>& caller) override { + if (!loop) { + auto subscribedInputChannelsIt = subscribedInputChannels.find(caller); + if (subscribedInputChannels.end() == subscribedInputChannelsIt) { + return false; + } else { + subscribedInputChannels.erase(subscribedInputChannelsIt); + mat = im; + return true; + } + } else { + mat = im; + return true; + } + } + void addSubscriber(const std::weak_ptr<InputChannel>& inputChannel) override { + if (false == subscribedInputChannels.insert(inputChannel).second) + throw std::invalid_argument("The insertion did not take place"); + } + cv::Size getSize() override { + return im.size(); + } + +private: + std::set<std::weak_ptr<InputChannel>, std::owner_less<std::weak_ptr<InputChannel>>> subscribedInputChannels; + cv::Mat im; + bool loop; +}; diff --git a/python/openvino/runtime/common/demo_utils/include/utils/kuhn_munkres.hpp b/python/openvino/runtime/common/demo_utils/include/utils/kuhn_munkres.hpp new file mode 100644 index 0000000..6e6ac51 --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/include/utils/kuhn_munkres.hpp @@ -0,0 +1,57 @@ +// Copyright (C) 2018-2019 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include "opencv2/core.hpp" + +#include <memory> +#include <vector> + + +/// +/// \brief The KuhnMunkres class +/// +/// Solves the assignment problem. +/// +class KuhnMunkres { +public: + /// + /// \brief Initializes the class for assignment problem solving. + /// \param[in] greedy If a faster greedy matching algorithm should be used. + explicit KuhnMunkres(bool greedy = false); + + /// + /// \brief Solves the assignment problem for given dissimilarity matrix. + /// It returns a vector that where each element is a column index for + /// corresponding row (e.g. result[0] stores optimal column index for very + /// first row in the dissimilarity matrix). + /// \param dissimilarity_matrix CV_32F dissimilarity matrix. + /// \return Optimal column index for each row. -1 means that there is no + /// column for row. + /// + std::vector<size_t> Solve(const cv::Mat &dissimilarity_matrix); + +private: + static constexpr int kStar = 1; + static constexpr int kPrime = 2; + + cv::Mat dm_; + cv::Mat marked_; + std::vector<cv::Point> points_; + + std::vector<int> is_row_visited_; + std::vector<int> is_col_visited_; + + int n_; + bool greedy_; + + void TrySimpleCase(); + bool CheckIfOptimumIsFound(); + cv::Point FindUncoveredMinValPos(); + void UpdateDissimilarityMatrix(float val); + int FindInRow(int row, int what); + int FindInCol(int col, int what); + void Run(); +}; diff --git a/python/openvino/runtime/common/demo_utils/include/utils/nms.hpp b/python/openvino/runtime/common/demo_utils/include/utils/nms.hpp new file mode 100644 index 0000000..1fd475f --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/include/utils/nms.hpp @@ -0,0 +1,81 @@ +/* +// Copyright (C) 2021-2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#pragma once + +#include "opencv2/core.hpp" +#include <numeric> +#include <vector> + +struct Anchor { + float left; + float top; + float right; + float bottom; + + float getWidth() const { + return (right - left) + 1.0f; + } + float getHeight() const { + return (bottom - top) + 1.0f; + } + float getXCenter() const { + return left + (getWidth() - 1.0f) / 2.0f; + } + float getYCenter() const { + return top + (getHeight() - 1.0f) / 2.0f; + } +}; + +template <typename Anchor> +std::vector<int> nms(const std::vector<Anchor>& boxes, const std::vector<float>& scores, + const float thresh, bool includeBoundaries=false) { + std::vector<float> areas(boxes.size()); + for (size_t i = 0; i < boxes.size(); ++i) { + areas[i] = (boxes[i].right - boxes[i].left + includeBoundaries) * (boxes[i].bottom - boxes[i].top + includeBoundaries); + } + std::vector<int> order(scores.size()); + std::iota(order.begin(), order.end(), 0); + std::sort(order.begin(), order.end(), [&scores](int o1, int o2) { return scores[o1] > scores[o2]; }); + + size_t ordersNum = 0; + for (; ordersNum < order.size() && scores[order[ordersNum]] >= 0; ordersNum++); + + std::vector<int> keep; + bool shouldContinue = true; + for (size_t i = 0; shouldContinue && i < ordersNum; ++i) { + auto idx1 = order[i]; + if (idx1 >= 0) { + keep.push_back(idx1); + shouldContinue = false; + for (size_t j = i + 1; j < ordersNum; ++j) { + auto idx2 = order[j]; + if (idx2 >= 0) { + shouldContinue = true; + auto overlappingWidth = std::fminf(boxes[idx1].right, boxes[idx2].right) - std::fmaxf(boxes[idx1].left, boxes[idx2].left); + auto overlappingHeight = std::fminf(boxes[idx1].bottom, boxes[idx2].bottom) - std::fmaxf(boxes[idx1].top, boxes[idx2].top); + auto intersection = overlappingWidth > 0 && overlappingHeight > 0 ? overlappingWidth * overlappingHeight : 0; + auto overlap = intersection / (areas[idx1] + areas[idx2] - intersection); + + if (overlap >= thresh) { + order[j] = -1; + } + } + } + } + } + return keep; +} diff --git a/python/openvino/runtime/common/demo_utils/include/utils/ocv_common.hpp b/python/openvino/runtime/common/demo_utils/include/utils/ocv_common.hpp new file mode 100644 index 0000000..ebb5e14 --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/include/utils/ocv_common.hpp @@ -0,0 +1,289 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief a header file with common samples functionality using OpenCV + * @file ocv_common.hpp + */ + +#pragma once + +#include <opencv2/opencv.hpp> +#include <openvino/openvino.hpp> + +#include "utils/common.hpp" +#include "utils/shared_tensor_allocator.hpp" + +/** +* @brief Get cv::Mat value in the correct format. +*/ +template <typename T> +const T getMatValue(const cv::Mat& mat, size_t h, size_t w, size_t c) { + switch (mat.type()) { + case CV_8UC1: return (T)mat.at<uchar>(h, w); + case CV_8UC3: return (T)mat.at<cv::Vec3b>(h, w)[c]; + case CV_32FC1: return (T)mat.at<float>(h, w); + case CV_32FC3: return (T)mat.at<cv::Vec3f>(h, w)[c]; + } + throw std::runtime_error("cv::Mat type is not recognized"); +}; + +/** +* @brief Resize and copy image data from cv::Mat object to a given Tensor object. +* @param mat - given cv::Mat object with an image data. +* @param tensor - Tensor object which to be filled by an image data. +* @param batchIndex - batch index of an image inside of the blob. +*/ +static UNUSED void matToTensor(const cv::Mat& mat, const ov::Tensor& tensor, int batchIndex = 0) { + ov::Shape tensorShape = tensor.get_shape(); + static const ov::Layout layout("NCHW"); + const size_t width = tensorShape[ov::layout::width_idx(layout)]; + const size_t height = tensorShape[ov::layout::height_idx(layout)]; + const size_t channels = tensorShape[ov::layout::channels_idx(layout)]; + if (static_cast<size_t>(mat.channels()) != channels) { + throw std::runtime_error("The number of channels for model input and image must match"); + } + if (channels != 1 && channels != 3) { + throw std::runtime_error("Unsupported number of channels"); + } + int batchOffset = batchIndex * width * height * channels; + + cv::Mat resizedMat; + if (static_cast<int>(width) != mat.size().width || static_cast<int>(height) != mat.size().height) { + cv::resize(mat, resizedMat, cv::Size(width, height)); + } else { + resizedMat = mat; + } + + if (tensor.get_element_type() == ov::element::f32) { + float_t* tensorData = tensor.data<float_t>(); + for (size_t c = 0; c < channels; c++) + for (size_t h = 0; h < height; h++) + for (size_t w = 0; w < width; w++) + tensorData[batchOffset + c * width * height + h * width + w] = + getMatValue<float_t>(resizedMat, h, w, c); + } else { + uint8_t* tensorData = tensor.data<uint8_t>(); + if (resizedMat.depth() == CV_32F) { + throw std::runtime_error("Conversion of cv::Mat from float_t to uint8_t is forbidden"); + } + for (size_t c = 0; c < channels; c++) + for (size_t h = 0; h < height; h++) + for (size_t w = 0; w < width; w++) + tensorData[batchOffset + c * width * height + h * width + w] = + getMatValue<uint8_t>(resizedMat, h, w, c); + } +} + +static UNUSED ov::Tensor wrapMat2Tensor(const cv::Mat& mat) { + auto matType = mat.type() & CV_MAT_DEPTH_MASK; + if (matType != CV_8U && matType != CV_32F) { + throw std::runtime_error("Unsupported mat type for wrapping"); + } + bool isMatFloat = matType == CV_32F; + + const size_t channels = mat.channels(); + const size_t height = mat.rows; + const size_t width = mat.cols; + + const size_t strideH = mat.step.buf[0]; + const size_t strideW = mat.step.buf[1]; + + const bool isDense = !isMatFloat ? (strideW == channels && strideH == channels * width) : + (strideW == channels * sizeof(float) && strideH == channels * width * sizeof(float)); + if (!isDense) { + throw std::runtime_error("Doesn't support conversion from not dense cv::Mat"); + } + auto precision = isMatFloat ? ov::element::f32 : ov::element::u8; + auto allocator = std::make_shared<SharedTensorAllocator>(mat); + return ov::Tensor(precision, ov::Shape{ 1, height, width, channels }, ov::Allocator(allocator)); +} + +static inline void resize2tensor(const cv::Mat& mat, const ov::Tensor& tensor) { + static const ov::Layout layout{"NHWC"}; + const ov::Shape& shape = tensor.get_shape(); + cv::Size size{int(shape[ov::layout::width_idx(layout)]), int(shape[ov::layout::height_idx(layout)])}; + assert(tensor.get_element_type() == ov::element::u8); + assert(shape.size() == 4); + assert(shape[ov::layout::batch_idx(layout)] == 1); + assert(shape[ov::layout::channels_idx(layout)] == 3); + cv::resize(mat, cv::Mat{size, CV_8UC3, tensor.data()}, size); +} + +static inline ov::Layout getLayoutFromShape(const ov::Shape& shape) { + if (shape.size() == 2) { + return "NC"; + } + else if (shape.size() == 3) { + return (shape[0] >= 1 && shape[0] <= 4) ? "CHW" : + "HWC"; + } + else if (shape.size() == 4) { + return (shape[1] >= 1 && shape[1] <= 4) ? "NCHW" : + "NHWC"; + } + else { + throw std::runtime_error("Usupported " + std::to_string(shape.size()) + "D shape"); + } +} + +/** + * @brief Puts text message on the frame, highlights the text with a white border to make it distinguishable from + * the background. + * @param frame - frame to put the text on. + * @param message - text of the message. + * @param position - bottom-left corner of the text string in the image. + * @param fontFace - font type. + * @param fontScale - font scale factor that is multiplied by the font-specific base size. + * @param color - text color. + * @param thickness - thickness of the lines used to draw a text. + */ +inline void putHighlightedText(const cv::Mat& frame, + const std::string& message, + cv::Point position, + int fontFace, + double fontScale, + cv::Scalar color, + int thickness) { + cv::putText(frame, message, position, fontFace, fontScale, cv::Scalar(255, 255, 255), thickness + 1); + cv::putText(frame, message, position, fontFace, fontScale, color, thickness); +} + +// TODO: replace with Size::empty() after OpenCV3 is dropped +static inline bool isSizeEmpty(const cv::Size& size) { + return size.width <= 0 || size.height <= 0; +} + +// TODO: replace with Rect::empty() after OpenCV3 is dropped +static inline bool isRectEmpty(const cv::Rect& rect) { + return rect.width <= 0 || rect.height <= 0; +} + +class OutputTransform { +public: + OutputTransform() : doResize(false), scaleFactor(1) {} + + OutputTransform(cv::Size inputSize, cv::Size outputResolution) : + doResize(true), scaleFactor(1), inputSize(inputSize), outputResolution(outputResolution) {} + + cv::Size computeResolution() { + float inputWidth = static_cast<float>(inputSize.width); + float inputHeight = static_cast<float>(inputSize.height); + scaleFactor = std::min(outputResolution.height / inputHeight, outputResolution.width / inputWidth); + newResolution = cv::Size{static_cast<int>(inputWidth * scaleFactor), static_cast<int>(inputHeight * scaleFactor)}; + return newResolution; + } + + void resize(cv::Mat& image) { + if (!doResize) { return; } + cv::Size currSize = image.size(); + if (currSize != inputSize) { + inputSize = currSize; + computeResolution(); + } + if (scaleFactor == 1) { return; } + cv::resize(image, image, newResolution); + } + + template<typename T> + void scaleCoord(T& coord) { + if (!doResize || scaleFactor == 1) { return; } + coord.x = std::floor(coord.x * scaleFactor); + coord.y = std::floor(coord.y * scaleFactor); + } + + template<typename T> + void scaleRect(T& rect) { + if (!doResize || scaleFactor == 1) { return; } + scaleCoord(rect); + rect.width = std::floor(rect.width * scaleFactor); + rect.height = std::floor(rect.height * scaleFactor); + } + + bool doResize; + +private: + float scaleFactor; + cv::Size inputSize; + cv::Size outputResolution; + cv::Size newResolution; +}; + +class InputTransform { +public: + InputTransform() : reverseInputChannels(false), isTrivial(true) {} + + InputTransform(bool reverseInputChannels, const std::string& meanValues, const std::string& scaleValues) : + reverseInputChannels(reverseInputChannels), + isTrivial(!reverseInputChannels && meanValues.empty() && scaleValues.empty()), + means(meanValues.empty() ? cv::Scalar(0.0, 0.0, 0.0) : string2Vec(meanValues)), + stdScales(scaleValues.empty() ? cv::Scalar(1.0, 1.0, 1.0) : string2Vec(scaleValues)) { + } + + cv::Scalar string2Vec(const std::string& string) { + const auto& strValues = split(string, ' '); + std::vector<float> values; + try { + for (auto& str : strValues) + values.push_back(std::stof(str)); + } + catch (const std::invalid_argument&) { + throw std::runtime_error("Invalid parameter --mean_values or --scale_values is provided."); + } + if (values.size() != 3) { + throw std::runtime_error("InputTransform expects 3 values per channel, but get \"" + string + "\"."); + } + return cv::Scalar(values[0], values[1], values[2]); + } + + void setPrecision(ov::preprocess::PrePostProcessor& ppp, const std::string& tensorName) { + const auto precision = isTrivial ? ov::element::u8 : ov::element::f32; + ppp.input(tensorName).tensor(). + set_element_type(precision); + } + + cv::Mat operator()(const cv::Mat& inputs) { + if (isTrivial) { return inputs; } + cv::Mat result; + inputs.convertTo(result, CV_32F); + if (reverseInputChannels) { + cv::cvtColor(result, result, cv::COLOR_BGR2RGB); + } + // TODO: merge the two following lines after OpenCV3 is droppped + result -= means; + result /= cv::Mat{stdScales}; + return result; + } + +private: + bool reverseInputChannels; + bool isTrivial; + cv::Scalar means; + cv::Scalar stdScales; +}; + +class LazyVideoWriter { + cv::VideoWriter writer; + unsigned nwritten; +public: + const std::string filenames; + const double fps; + const unsigned lim; + + LazyVideoWriter(const std::string& filenames, double fps, unsigned lim) : + nwritten{1}, filenames{filenames}, fps{fps}, lim{lim} {} + void write(const cv::Mat& im) { + if (writer.isOpened() && (nwritten < lim || 0 == lim)) { + writer.write(im); + ++nwritten; + return; + } + if (!writer.isOpened() && !filenames.empty()) { + if (!writer.open(filenames, cv::VideoWriter::fourcc('M', 'J', 'P', 'G'), fps, im.size())) { + throw std::runtime_error("Can't open video writer"); + } + writer.write(im); + } + } +}; diff --git a/python/openvino/runtime/common/demo_utils/include/utils/performance_metrics.hpp b/python/openvino/runtime/common/demo_utils/include/utils/performance_metrics.hpp new file mode 100644 index 0000000..6c728b0 --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/include/utils/performance_metrics.hpp @@ -0,0 +1,92 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief a header file for performance metrics calculation class + * @file performance_metrics.hpp + */ + +#pragma once + +#include <chrono> +#include <iomanip> +#include <iostream> +#include <sstream> + +#include "utils/ocv_common.hpp" + +class PerformanceMetrics { +public: + using Clock = std::chrono::steady_clock; + using TimePoint = std::chrono::time_point<Clock>; + using Duration = Clock::duration; + using Ms = std::chrono::duration<double, std::ratio<1, 1000>>; + using Sec = std::chrono::duration<double, std::ratio<1, 1>>; + + struct Metrics { + double latency; + double fps; + }; + + enum MetricTypes { + ALL, + FPS, + LATENCY + }; + + PerformanceMetrics(Duration timeWindow = std::chrono::seconds(1)); + void update(TimePoint lastRequestStartTime, + const cv::Mat& frame, + cv::Point position = {15, 30}, + int fontFace = cv::FONT_HERSHEY_COMPLEX, + double fontScale = 0.75, + cv::Scalar color = {200, 10, 10}, + int thickness = 2, MetricTypes metricType = ALL); + void update(TimePoint lastRequestStartTime); + + /// Paints metrics over provided mat + /// @param frame frame to paint over + /// @param position left top corner of text block + /// @param fontScale font scale + /// @param color font color + /// @param thickness font thickness + void paintMetrics(const cv::Mat& frame, + cv::Point position = { 15, 30 }, + int fontFace = cv::FONT_HERSHEY_COMPLEX, + double fontScale = 0.75, + cv::Scalar color = { 200, 10, 10 }, + int thickness = 2, MetricTypes metricType = ALL) const; + + Metrics getLast() const; + Metrics getTotal() const; + void logTotal() const; + +private: + struct Statistic { + Duration latency; + Duration period; + int frameCount; + + Statistic() { + latency = Duration::zero(); + period = Duration::zero(); + frameCount = 0; + } + + void combine(const Statistic& other) { + latency += other.latency; + period += other.period; + frameCount += other.frameCount; + } + }; + + Duration timeWindowSize; + Statistic lastMovingStatistic; + Statistic currentMovingStatistic; + Statistic totalStatistic; + TimePoint lastUpdateTime; + bool firstFrameProcessed; +}; + +void logLatencyPerStage(double readLat, double preprocLat, double inferLat, double postprocLat, double renderLat); diff --git a/python/openvino/runtime/common/demo_utils/include/utils/shared_tensor_allocator.hpp b/python/openvino/runtime/common/demo_utils/include/utils/shared_tensor_allocator.hpp new file mode 100644 index 0000000..f74e8d0 --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/include/utils/shared_tensor_allocator.hpp @@ -0,0 +1,47 @@ +/* +// Copyright (C) 2021-2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#pragma once + +#include <opencv2/core.hpp> +#include <openvino/runtime/allocator.hpp> + +// To prevent false-positive clang compiler warning +// (https://github.com/openvinotoolkit/openvino/pull/11092#issuecomment-1073846256): +// warning: destructor called on non-final 'SharedTensorAllocator' that has virtual functions +// but non-virtual destructor [-Wdelete-non-abstract-non-virtual-dtor] +// SharedTensorAllocator class declared as final + +class SharedTensorAllocator final : public ov::AllocatorImpl { +public: + SharedTensorAllocator(const cv::Mat& img) : img(img) {} + + ~SharedTensorAllocator() = default; + + void* allocate(const size_t bytes, const size_t) override { + return bytes <= img.rows * img.step[0] ? img.data : nullptr; + } + + void deallocate(void* handle, const size_t bytes, const size_t) override {} + + bool is_equal(const AllocatorImpl& other) const override { + auto other_tensor_allocator = dynamic_cast<const SharedTensorAllocator*>(&other); + return other_tensor_allocator != nullptr && other_tensor_allocator == this; + } + +private: + const cv::Mat img; +}; diff --git a/python/openvino/runtime/common/demo_utils/include/utils/slog.hpp b/python/openvino/runtime/common/demo_utils/include/utils/slog.hpp new file mode 100644 index 0000000..316b98d --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/include/utils/slog.hpp @@ -0,0 +1,99 @@ +// Copyright (C) 2018-2019 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +/** + * @brief a header file with logging facility for common samples + * @file log.hpp + */ + +#pragma once + +#include <iostream> +#include <string> + +namespace slog { + +/** + * @class LogStreamEndLine + * @brief The LogStreamEndLine class implements an end line marker for a log stream + */ +class LogStreamEndLine { }; + +static constexpr LogStreamEndLine endl; + + +/** + * @class LogStreamBoolAlpha + * @brief The LogStreamBoolAlpha class implements bool printing for a log stream + */ +class LogStreamBoolAlpha { }; + +static constexpr LogStreamBoolAlpha boolalpha; + + +/** + * @class LogStream + * @brief The LogStream class implements a stream for sample logging + */ +class LogStream { + std::string _prefix; + std::ostream* _log_stream; + bool _new_line; + +public: + /** + * @brief A constructor. Creates a LogStream object + * @param prefix The prefix to print + */ + LogStream(const std::string &prefix, std::ostream& log_stream) + : _prefix(prefix), _new_line(true) { + _log_stream = &log_stream; + } + + /** + * @brief A stream output operator to be used within the logger + * @param arg Object for serialization in the logger message + */ + template<class T> + LogStream &operator<<(const T &arg) { + if (_new_line) { + (*_log_stream) << "[ " << _prefix << " ] "; + _new_line = false; + } + + (*_log_stream) << arg; + return *this; + } + + // Specializing for LogStreamEndLine to support slog::endl + LogStream& operator<< (const LogStreamEndLine &/*arg*/) { + _new_line = true; + + (*_log_stream) << std::endl; + return *this; + } + + // Specializing for LogStreamBoolAlpha to support slog::boolalpha + LogStream& operator<< (const LogStreamBoolAlpha &/*arg*/) { + (*_log_stream) << std::boolalpha; + return *this; + } + + // Specializing for std::vector and std::list + template<template<class, class> class Container, class T> + LogStream& operator<< (const Container<T, std::allocator<T>>& container) { + for (const auto& el : container) { + *this << el << slog::endl; + } + return *this; + } +}; + + +static LogStream info("INFO", std::cout); +static LogStream debug("DEBUG", std::cout); +static LogStream warn("WARNING", std::cout); +static LogStream err("ERROR", std::cerr); + +} // namespace slog diff --git a/python/openvino/runtime/common/demo_utils/include/utils/threads_common.hpp b/python/openvino/runtime/common/demo_utils/include/utils/threads_common.hpp new file mode 100644 index 0000000..f0e5cbf --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/include/utils/threads_common.hpp @@ -0,0 +1,165 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#include <algorithm> +#include <atomic> +#include <condition_variable> +#include <memory> +#include <mutex> +#include <utility> +#include <set> +#include <string> +#include <thread> +#include <vector> + +#include <opencv2/core/core.hpp> +#include "utils/performance_metrics.hpp" + +// VideoFrame can represent not a single image but the whole grid +class VideoFrame { +public: + typedef std::shared_ptr<VideoFrame> Ptr; + + VideoFrame(unsigned sourceID, int64_t frameId, const cv::Mat& frame = cv::Mat()) : + sourceID{sourceID}, frameId{frameId}, frame{frame} {} + virtual ~VideoFrame() = default; // A user has to define how it is reconstructed + + const unsigned sourceID; + const int64_t frameId; + cv::Mat frame; + + PerformanceMetrics::TimePoint timestamp; +}; + +class Worker; + +class Task { +public: + explicit Task(VideoFrame::Ptr sharedVideoFrame, float priority = 0): + sharedVideoFrame{sharedVideoFrame}, priority{priority} {} + virtual bool isReady() = 0; + virtual void process() = 0; + virtual ~Task() = default; + + std::string name; + VideoFrame::Ptr sharedVideoFrame; // it is possible that two tasks try to draw on the same cvMat + const float priority; +}; + +struct HigherPriority { + bool operator()(const std::shared_ptr<Task>& lhs, const std::shared_ptr<Task>& rhs) const { + return lhs->priority > rhs->priority + || (lhs->priority == rhs->priority && lhs->sharedVideoFrame->frameId < rhs->sharedVideoFrame->frameId) + || (lhs->priority == rhs->priority && lhs->sharedVideoFrame->frameId == rhs->sharedVideoFrame->frameId && lhs < rhs); + } +}; + +class Worker { +public: + explicit Worker(unsigned threadNum): + threadPool(threadNum), running{false} {} + ~Worker() { + stop(); + } + void runThreads() { + running = true; + for (std::thread& t : threadPool) { + t = std::thread(&Worker::threadFunc, this); + } + } + void push(std::shared_ptr<Task> task) { + tasksMutex.lock(); + tasks.insert(task); + tasksMutex.unlock(); + tasksCondVar.notify_one(); + } + void threadFunc() { + while (running) { + std::unique_lock<std::mutex> lk(tasksMutex); + while (running && tasks.empty()) { + tasksCondVar.wait(lk); + } + try { + auto it = std::find_if(tasks.begin(), tasks.end(), [](const std::shared_ptr<Task>& task){return task->isReady();}); + if (tasks.end() != it) { + const std::shared_ptr<Task> task = std::move(*it); + tasks.erase(it); + lk.unlock(); + task->process(); + } + } catch (...) { + std::lock_guard<std::mutex> lock{exceptionMutex}; + if (nullptr == currentException) { + currentException = std::current_exception(); + stop(); + } + } + } + } + void stop() { + running = false; + tasksCondVar.notify_all(); + } + void join() { + for (auto& t : threadPool) { + t.join(); + } + if (nullptr != currentException) { + std::rethrow_exception(currentException); + } + } + +private: + std::condition_variable tasksCondVar; + std::set<std::shared_ptr<Task>, HigherPriority> tasks; + std::mutex tasksMutex; + std::vector<std::thread> threadPool; + std::atomic<bool> running; + std::exception_ptr currentException; + std::mutex exceptionMutex; +}; + +void tryPush(const std::weak_ptr<Worker>& worker, std::shared_ptr<Task>&& task) { + try { + std::shared_ptr<Worker>(worker)->push(task); + } catch (const std::bad_weak_ptr&) {} +} + +template <class C> class ConcurrentContainer { +public: + C container; + mutable std::mutex mutex; + + bool lockedEmpty() const noexcept { + std::lock_guard<std::mutex> lock{mutex}; + return container.empty(); + } + typename C::size_type lockedSize() const noexcept { + std::lock_guard<std::mutex> lock{mutex}; + return container.size(); + } + void lockedPushBack(const typename C::value_type& value) { + std::lock_guard<std::mutex> lock{mutex}; + container.push_back(value); + } + bool lockedTryPop(typename C::value_type& value) { + mutex.lock(); + if (!container.empty()) { + value = container.back(); + container.pop_back(); + mutex.unlock(); + return true; + } else { + mutex.unlock(); + return false; + } + } + + operator C() const { + std::lock_guard<std::mutex> lock{mutex}; + return container; + } +}; diff --git a/python/openvino/runtime/common/demo_utils/src/args_helper.cpp b/python/openvino/runtime/common/demo_utils/src/args_helper.cpp new file mode 100644 index 0000000..8f4bc35 --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/src/args_helper.cpp @@ -0,0 +1,155 @@ +// Copyright (C) 2018-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "utils/args_helper.hpp" +#include "utils/slog.hpp" + +#ifdef _WIN32 +#include "w_dirent.hpp" +#else +#include <dirent.h> +#endif + +#include <gflags/gflags.h> + +#include <sys/stat.h> +#include <map> + +#include <algorithm> +#include <cctype> +#include <sstream> + +void readInputFilesArguments(std::vector<std::string>& files, const std::string& arg) { + struct stat sb; + if (stat(arg.c_str(), &sb) != 0) { + if (arg.compare(0, 5, "rtsp:") != 0) { + slog::warn << "File " << arg << " cannot be opened!" << slog::endl; + return; + } + } + if (S_ISDIR(sb.st_mode)) { + DIR *dp; + dp = opendir(arg.c_str()); + if (dp == nullptr) { + slog::warn << "Directory " << arg << " cannot be opened!" << slog::endl; + return; + } + + struct dirent *ep; + while (nullptr != (ep = readdir(dp))) { + std::string fileName = ep->d_name; + if (fileName == "." || fileName == "..") continue; + files.push_back(arg + "/" + ep->d_name); + } + closedir(dp); + } else { + files.push_back(arg); + } +} + +void parseInputFilesArguments(std::vector<std::string>& files) { + std::vector<std::string> args = gflags::GetArgvs(); + bool readArguments = false; + for (size_t i = 0; i < args.size(); i++) { + if (args.at(i) == "-i" || args.at(i) == "--i") { + readArguments = true; + continue; + } + if (!readArguments) { + continue; + } + if (args.at(i).c_str()[0] == '-') { + break; + } + readInputFilesArguments(files, args.at(i)); + } +} + +std::vector<std::string> split(const std::string& s, char delim) { + std::vector<std::string> result; + std::stringstream ss(s); + std::string item; + + while (getline(ss, item, delim)) { + result.push_back(item); + } + return result; +} + +std::vector<std::string> parseDevices(const std::string& device_string) { + const std::string::size_type colon_position = device_string.find(":"); + if (colon_position != std::string::npos) { + std::string device_type = device_string.substr(0, colon_position); + if (device_type == "HETERO" || device_type == "MULTI") { + std::string comma_separated_devices = device_string.substr(colon_position + 1); + std::vector<std::string> devices = split(comma_separated_devices, ','); + for (auto& device : devices) + device = device.substr(0, device.find("(")); + return devices; + } + } + return {device_string}; +} + +// Format: <device1>:<value1>,<device2>:<value2> or just <value> +std::map<std::string, int32_t> parseValuePerDevice(const std::set<std::string>& devices, + const std::string& values_string) { + auto values_string_upper = values_string; + std::transform(values_string_upper.begin(), + values_string_upper.end(), + values_string_upper.begin(), + [](unsigned char c){ return std::toupper(c); }); + std::map<std::string, int32_t> result; + auto device_value_strings = split(values_string_upper, ','); + for (auto& device_value_string : device_value_strings) { + auto device_value_vec = split(device_value_string, ':'); + if (device_value_vec.size() == 2) { + auto it = std::find(devices.begin(), devices.end(), device_value_vec.at(0)); + if (it != devices.end()) { + result[device_value_vec.at(0)] = std::stoi(device_value_vec.at(1)); + } + } else if (device_value_vec.size() == 1) { + uint32_t value = std::stoi(device_value_vec.at(0)); + for (const auto& device : devices) { + result[device] = value; + } + } else if (device_value_vec.size() != 0) { + throw std::runtime_error("Unknown string format: " + values_string); + } + } + return result; +} + +cv::Size stringToSize(const std::string& str) { + std::vector<std::string> strings = split(str, 'x'); + if (strings.size() != 2) { + throw std::invalid_argument("Can't convert std::string to cv::Size. The string must contain exactly one x"); + } + return {std::stoi(strings[0]), std::stoi(strings[1])}; +} + +std::map<std::string, ov::Layout> parseLayoutString(const std::string& layout_string) { + // Parse parameter string like "input0:NCHW,input1:NC" or "NCHW" (applied to all + // inputs) + std::map<std::string, ov::Layout> layouts; + std::string searchStr = (layout_string.find_last_of(':') == std::string::npos && !layout_string.empty() ? + ":" : "") + layout_string; + auto colonPos = searchStr.find_last_of(':'); + while (colonPos != std::string::npos) { + auto startPos = searchStr.find_last_of(','); + auto inputName = searchStr.substr(startPos + 1, colonPos - startPos - 1); + auto inputLayout = searchStr.substr(colonPos + 1); + layouts[inputName] = ov::Layout(inputLayout); + searchStr = searchStr.substr(0, startPos + 1); + if (searchStr.empty() || searchStr.back() != ',') { + break; + } + searchStr.pop_back(); + colonPos = searchStr.find_last_of(':'); + } + if (!searchStr.empty()) { + throw std::invalid_argument("Can't parse input layout string: " + layout_string); + } + return layouts; +} diff --git a/python/openvino/runtime/common/demo_utils/src/config_factory.cpp b/python/openvino/runtime/common/demo_utils/src/config_factory.cpp new file mode 100644 index 0000000..2e9a442 --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/src/config_factory.cpp @@ -0,0 +1,111 @@ +/* +// Copyright (C) 2020-2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include "utils/config_factory.h" + +#include <set> +#include <string> +#include <utility> +#include <vector> + +#include <openvino/runtime/intel_gpu/properties.hpp> +#include "dla_plugin_config.hpp" +#include "utils/args_helper.hpp" +#include <sys/stat.h> + +std::set<std::string> ModelConfig::getDevices() { + if (devices.empty()) { + for (const std::string& device : parseDevices(deviceName)) { + devices.insert(device); + } + } + + return devices; +} + +ModelConfig ConfigFactory::getUserConfig(const std::string& flags_d, + uint32_t flags_nireq, + const std::string& flags_nstreams, + uint32_t flags_nthreads, + const std::string &flags_arch) { + auto config = getCommonConfig(flags_d, flags_nireq); + + std::map<std::string, int> deviceNstreams = parseValuePerDevice(config.getDevices(), flags_nstreams); + for (const auto& device : config.getDevices()) { + if (flags_arch != "" && device == "FPGA") { + struct stat buffer; + if (stat(flags_arch.c_str(), &buffer) != 0) { + std::cout << "Error: architecture file: " << flags_arch << " doesn't exist. Please provide a valid path." << std::endl; + throw std::logic_error("architecture file path does not exist."); + } + config.compiledModelConfig.emplace(DLIAPlugin::properties::arch_path.name(), flags_arch); + } else if (device == "CPU") { // CPU supports a few special performance-oriented keys + // limit threading for CPU portion of inference + if (flags_nthreads != 0) + config.compiledModelConfig.emplace(ov::inference_num_threads.name(), flags_nthreads); + + config.compiledModelConfig.emplace(ov::affinity.name(), ov::Affinity::NONE); + + ov::streams::Num nstreams = + deviceNstreams.count(device) > 0 ? ov::streams::Num(deviceNstreams[device]) : ov::streams::AUTO; + config.compiledModelConfig.emplace(ov::streams::num.name(), nstreams); + } else if (device == "GPU") { + ov::streams::Num nstreams = + deviceNstreams.count(device) > 0 ? ov::streams::Num(deviceNstreams[device]) : ov::streams::AUTO; + config.compiledModelConfig.emplace(ov::streams::num.name(), nstreams); + if (flags_d.find("MULTI") != std::string::npos && + config.getDevices().find("CPU") != config.getDevices().end()) { + // multi-device execution with the CPU + GPU performs best with GPU throttling hint, + // which releases another CPU thread (that is otherwise used by the GPU driver for active polling) + config.compiledModelConfig.emplace(ov::intel_gpu::hint::queue_throttle.name(), + ov::intel_gpu::hint::ThrottleLevel(1)); + } + } + } + return config; +} + +ModelConfig ConfigFactory::getMinLatencyConfig(const std::string& flags_d, uint32_t flags_nireq) { + auto config = getCommonConfig(flags_d, flags_nireq); + for (const auto& device : config.getDevices()) { + if (device == "CPU") { // CPU supports a few special performance-oriented keys + config.compiledModelConfig.emplace(ov::streams::num.name(), 1); + } else if (device == "GPU") { + config.compiledModelConfig.emplace(ov::streams::num.name(), 1); + } + } + return config; +} + +ModelConfig ConfigFactory::getCommonConfig(const std::string& flags_d, uint32_t flags_nireq) { + ModelConfig config; + + if (!flags_d.empty()) { + config.deviceName = flags_d; + } + + config.maxAsyncRequests = flags_nireq; + + return config; +} + +std::map<std::string, std::string> ModelConfig::getLegacyConfig() { + std::map<std::string, std::string> config; + for (const auto& item : compiledModelConfig) { + config[item.first] = item.second.as<std::string>(); + } + return config; +} diff --git a/python/openvino/runtime/common/demo_utils/src/image_utils.cpp b/python/openvino/runtime/common/demo_utils/src/image_utils.cpp new file mode 100644 index 0000000..039dd66 --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/src/image_utils.cpp @@ -0,0 +1,55 @@ +/* +// Copyright (C) 2021-2022 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +*/ + +#include "utils/image_utils.h" + +cv::Mat resizeImageExt(const cv::Mat& mat, int width, int height, RESIZE_MODE resizeMode, + cv::InterpolationFlags interpolationMode, cv::Rect* roi, cv::Scalar BorderConstant) { + if (width == mat.cols && height == mat.rows) { + return mat; + } + + cv::Mat dst; + + switch (resizeMode) { + case RESIZE_FILL: + { + cv::resize(mat, dst, cv::Size(width, height), interpolationMode); + if (roi) { + *roi = cv::Rect(0, 0, width, height); + } + break; + } + case RESIZE_KEEP_ASPECT: + case RESIZE_KEEP_ASPECT_LETTERBOX: + { + double scale = std::min(static_cast<double>(width) / mat.cols, static_cast<double>(height) / mat.rows); + cv::Mat resizedImage; + cv::resize(mat, resizedImage, cv::Size(0, 0), scale, scale, interpolationMode); + + int dx = resizeMode == RESIZE_KEEP_ASPECT ? 0 : (width - resizedImage.cols) / 2; + int dy = resizeMode == RESIZE_KEEP_ASPECT ? 0 : (height - resizedImage.rows) / 2; + + cv::copyMakeBorder(resizedImage, dst, dy, height - resizedImage.rows - dy, + dx, width - resizedImage.cols - dx, cv::BORDER_CONSTANT, BorderConstant); + if (roi) { + *roi = cv::Rect(dx, dy, resizedImage.cols, resizedImage.rows); + } + break; + } + } + return dst; +} diff --git a/python/openvino/runtime/common/demo_utils/src/images_capture.cpp b/python/openvino/runtime/common/demo_utils/src/images_capture.cpp new file mode 100644 index 0000000..febcdd7 --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/src/images_capture.cpp @@ -0,0 +1,327 @@ +// Copyright (C) 2020-2022 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "utils/images_capture.h" + +#include <string.h> + +#ifdef _WIN32 +# include "w_dirent.hpp" +#else +# include <dirent.h> // for closedir, dirent, opendir, readdir, DIR +#endif + +#include <algorithm> +#include <chrono> +#include <fstream> +#include <memory> +#include <stdexcept> +#include <string> +#include <vector> + +#include <opencv2/imgcodecs.hpp> +#include <opencv2/videoio.hpp> + +class InvalidInput : public std::runtime_error { +public: + explicit InvalidInput(const std::string& message) noexcept : std::runtime_error(message) {} +}; + +class OpenError : public std::runtime_error { +public: + explicit OpenError(const std::string& message) noexcept : std::runtime_error(message) {} +}; + +class ImreadWrapper : public ImagesCapture { + cv::Mat img; + bool canRead; + +public: + ImreadWrapper(const std::string& input, bool loop) : ImagesCapture{loop}, canRead{true} { + auto startTime = std::chrono::steady_clock::now(); + + std::ifstream file(input.c_str()); + if (!file.good()) + throw InvalidInput("Can't find the image by " + input); + + img = cv::imread(input); + if (!img.data) + throw OpenError("Can't open the image from " + input); + else + readerMetrics.update(startTime); + } + + double fps() const override { + return 1.0; + } + + std::string getType() const override { + return "IMAGE"; + } + + cv::Mat read() override { + if (loop) + return img.clone(); + if (canRead) { + canRead = false; + return img.clone(); + } + return cv::Mat{}; + } +}; + +class DirReader : public ImagesCapture { + std::vector<std::string> names; + size_t fileId; + size_t nextImgId; + const size_t initialImageId; + const size_t readLengthLimit; + const std::string input; + +public: + DirReader(const std::string& input, bool loop, size_t initialImageId, size_t readLengthLimit) + : ImagesCapture{loop}, + fileId{0}, + nextImgId{0}, + initialImageId{initialImageId}, + readLengthLimit{readLengthLimit}, + input{input} { + DIR* dir = opendir(input.c_str()); + if (!dir) + throw InvalidInput("Can't find the dir by " + input); + while (struct dirent* ent = readdir(dir)) + if (strcmp(ent->d_name, ".") && strcmp(ent->d_name, "..")) + names.emplace_back(ent->d_name); + closedir(dir); + if (names.empty()) + throw OpenError("The dir " + input + " is empty"); + sort(names.begin(), names.end()); + size_t readImgs = 0; + while (fileId < names.size()) { + cv::Mat img = cv::imread(input + '/' + names[fileId]); + if (img.data) { + ++readImgs; + if (readImgs - 1 >= initialImageId) + return; + } + ++fileId; + } + throw OpenError("Can't read the first image from " + input); + } + + double fps() const override { + return 1.0; + } + + std::string getType() const override { + return "DIR"; + } + + cv::Mat read() override { + auto startTime = std::chrono::steady_clock::now(); + + while (fileId < names.size() && nextImgId < readLengthLimit) { + cv::Mat img = cv::imread(input + '/' + names[fileId]); + ++fileId; + if (img.data) { + ++nextImgId; + readerMetrics.update(startTime); + return img; + } + } + + if (loop) { + fileId = 0; + size_t readImgs = 0; + while (fileId < names.size()) { + cv::Mat img = cv::imread(input + '/' + names[fileId]); + ++fileId; + if (img.data) { + ++readImgs; + if (readImgs - 1 >= initialImageId) { + nextImgId = 1; + readerMetrics.update(startTime); + return img; + } + } + } + } + return cv::Mat{}; + } +}; + +class VideoCapWrapper : public ImagesCapture { + cv::VideoCapture cap; + bool first_read; + const read_type type; + size_t nextImgId; + const double initialImageId; + size_t readLengthLimit; + +public: + VideoCapWrapper(const std::string& input, bool loop, read_type type, size_t initialImageId, size_t readLengthLimit) + : ImagesCapture{loop}, + first_read{true}, + type{type}, + nextImgId{0}, + initialImageId{static_cast<double>(initialImageId)} { + if (0 == readLengthLimit) { + throw std::runtime_error("readLengthLimit must be positive"); + } + if (cap.open(input)) { + this->readLengthLimit = readLengthLimit; + if (!cap.set(cv::CAP_PROP_POS_FRAMES, this->initialImageId)) + throw OpenError("Can't set the frame to begin with"); + return; + } + throw InvalidInput("Can't open the video from " + input); + } + + double fps() const override { + return cap.get(cv::CAP_PROP_FPS); + } + + std::string getType() const override { + return "VIDEO"; + } + + cv::Mat read() override { + auto startTime = std::chrono::steady_clock::now(); + + if (nextImgId >= readLengthLimit) { + if (loop && cap.set(cv::CAP_PROP_POS_FRAMES, initialImageId)) { + nextImgId = 1; + cv::Mat img; + cap.read(img); + if (type == read_type::safe) { + img = img.clone(); + } + readerMetrics.update(startTime); + return img; + } + return cv::Mat{}; + } + cv::Mat img; + bool success = cap.read(img); + if (!success && first_read) { + throw std::runtime_error("The first image can't be read"); + } + first_read = false; + if (!success && loop && cap.set(cv::CAP_PROP_POS_FRAMES, initialImageId)) { + nextImgId = 1; + cap.read(img); + } else { + ++nextImgId; + } + if (type == read_type::safe) { + img = img.clone(); + } + readerMetrics.update(startTime); + return img; + } +}; + +class CameraCapWrapper : public ImagesCapture { + cv::VideoCapture cap; + const read_type type; + size_t nextImgId; + size_t readLengthLimit; + +public: + CameraCapWrapper(const std::string& input, + bool loop, + read_type type, + size_t readLengthLimit, + cv::Size cameraResolution) + : ImagesCapture{loop}, + type{type}, + nextImgId{0} { + if (0 == readLengthLimit) { + throw std::runtime_error("readLengthLimit must be positive"); + } + try { + if (cap.open(std::stoi(input))) { + this->readLengthLimit = loop ? std::numeric_limits<size_t>::max() : readLengthLimit; + cap.set(cv::CAP_PROP_BUFFERSIZE, 1); + cap.set(cv::CAP_PROP_FRAME_WIDTH, cameraResolution.width); + cap.set(cv::CAP_PROP_FRAME_HEIGHT, cameraResolution.height); + cap.set(cv::CAP_PROP_AUTOFOCUS, true); + cap.set(cv::CAP_PROP_FOURCC, cv::VideoWriter::fourcc('M', 'J', 'P', 'G')); + return; + } + throw OpenError("Can't open the camera from " + input); + } catch (const std::invalid_argument&) { + throw InvalidInput("Can't find the camera " + input); + } catch (const std::out_of_range&) { throw InvalidInput("Can't find the camera " + input); } + } + + double fps() const override { + return cap.get(cv::CAP_PROP_FPS) > 0 ? cap.get(cv::CAP_PROP_FPS) : 30; + } + + std::string getType() const override { + return "CAMERA"; + } + + cv::Mat read() override { + auto startTime = std::chrono::steady_clock::now(); + + if (nextImgId >= readLengthLimit) { + return cv::Mat{}; + } + cv::Mat img; + if (!cap.read(img)) { + throw std::runtime_error("The image can't be captured from the camera"); + } + if (type == read_type::safe) { + img = img.clone(); + } + ++nextImgId; + + readerMetrics.update(startTime); + return img; + } +}; + +std::unique_ptr<ImagesCapture> openImagesCapture(const std::string& input, + bool loop, + read_type type, + size_t initialImageId, + size_t readLengthLimit, + cv::Size cameraResolution + ) { + if (readLengthLimit == 0) + throw std::runtime_error{"Read length limit must be positive"}; + std::vector<std::string> invalidInputs, openErrors; + try { + return std::unique_ptr<ImagesCapture>(new ImreadWrapper{input, loop}); + } catch (const InvalidInput& e) { invalidInputs.push_back(e.what()); } catch (const OpenError& e) { + openErrors.push_back(e.what()); + } + + try { + return std::unique_ptr<ImagesCapture>(new DirReader{input, loop, initialImageId, readLengthLimit}); + } catch (const InvalidInput& e) { invalidInputs.push_back(e.what()); } catch (const OpenError& e) { + openErrors.push_back(e.what()); + } + + try { + return std::unique_ptr<ImagesCapture>(new VideoCapWrapper{input, loop, type, initialImageId, readLengthLimit}); + } catch (const InvalidInput& e) { invalidInputs.push_back(e.what()); } catch (const OpenError& e) { + openErrors.push_back(e.what()); + } + + try { + return std::unique_ptr<ImagesCapture>( + new CameraCapWrapper{input, loop, type, readLengthLimit, cameraResolution}); + } catch (const InvalidInput& e) { invalidInputs.push_back(e.what()); } catch (const OpenError& e) { + openErrors.push_back(e.what()); + } + + std::vector<std::string> errorMessages = openErrors.empty() ? invalidInputs : openErrors; + std::string errorsInfo; + for (const auto& message : errorMessages) { + errorsInfo.append(message + "\n"); + } + throw std::runtime_error(errorsInfo); +} diff --git a/python/openvino/runtime/common/demo_utils/src/kuhn_munkres.cpp b/python/openvino/runtime/common/demo_utils/src/kuhn_munkres.cpp new file mode 100644 index 0000000..7d612c1 --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/src/kuhn_munkres.cpp @@ -0,0 +1,169 @@ +// Copyright (C) 2018-2019 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include <algorithm> +#include <limits> +#include <vector> + +#include <utils/kuhn_munkres.hpp> + +KuhnMunkres::KuhnMunkres(bool greedy) : n_(), greedy_(greedy) {} + +std::vector<size_t> KuhnMunkres::Solve(const cv::Mat& dissimilarity_matrix) { + CV_Assert(dissimilarity_matrix.type() == CV_32F); + double min_val; + cv::minMaxLoc(dissimilarity_matrix, &min_val); + + n_ = std::max(dissimilarity_matrix.rows, dissimilarity_matrix.cols); + dm_ = cv::Mat(n_, n_, CV_32F, cv::Scalar(0)); + marked_ = cv::Mat(n_, n_, CV_8S, cv::Scalar(0)); + points_ = std::vector<cv::Point>(n_ * 2); + + dissimilarity_matrix.copyTo(dm_( + cv::Rect(0, 0, dissimilarity_matrix.cols, dissimilarity_matrix.rows))); + + is_row_visited_ = std::vector<int>(n_, 0); + is_col_visited_ = std::vector<int>(n_, 0); + + Run(); + + std::vector<size_t> results(dissimilarity_matrix.rows, -1); + for (int i = 0; i < dissimilarity_matrix.rows; i++) { + const auto ptr = marked_.ptr<char>(i); + for (int j = 0; j < dissimilarity_matrix.cols; j++) { + if (ptr[j] == kStar) { + results[i] = (size_t)j; + } + } + } + return results; +} + +void KuhnMunkres::TrySimpleCase() { + auto is_row_visited = std::vector<int>(n_, 0); + auto is_col_visited = std::vector<int>(n_, 0); + + for (int row = 0; row < n_; row++) { + auto ptr = dm_.ptr<float>(row); + auto marked_ptr = marked_.ptr<char>(row); + auto min_val = *std::min_element(ptr, ptr + n_); + for (int col = 0; col < n_; col++) { + ptr[col] -= min_val; + if (ptr[col] == 0 && !is_col_visited[col] && !is_row_visited[row]) { + marked_ptr[col] = kStar; + is_col_visited[col] = 1; + is_row_visited[row] = 1; + } + } + } +} + +bool KuhnMunkres::CheckIfOptimumIsFound() { + int count = 0; + for (int i = 0; i < n_; i++) { + const auto marked_ptr = marked_.ptr<char>(i); + for (int j = 0; j < n_; j++) { + if (marked_ptr[j] == kStar) { + is_col_visited_[j] = 1; + count++; + } + } + } + + return count >= n_; +} + +cv::Point KuhnMunkres::FindUncoveredMinValPos() { + auto min_val = std::numeric_limits<float>::max(); + cv::Point min_val_pos(-1, -1); + for (int i = 0; i < n_; i++) { + if (!is_row_visited_[i]) { + auto dm_ptr = dm_.ptr<float>(i); + for (int j = 0; j < n_; j++) { + if (!is_col_visited_[j] && dm_ptr[j] < min_val) { + min_val = dm_ptr[j]; + min_val_pos = cv::Point(j, i); + } + } + } + } + return min_val_pos; +} + +void KuhnMunkres::UpdateDissimilarityMatrix(float val) { + for (int i = 0; i < n_; i++) { + auto dm_ptr = dm_.ptr<float>(i); + for (int j = 0; j < n_; j++) { + if (is_row_visited_[i]) dm_ptr[j] += val; + if (!is_col_visited_[j]) dm_ptr[j] -= val; + } + } +} + +int KuhnMunkres::FindInRow(int row, int what) { + for (int j = 0; j < n_; j++) { + if (marked_.at<char>(row, j) == what) { + return j; + } + } + return -1; +} + +int KuhnMunkres::FindInCol(int col, int what) { + for (int i = 0; i < n_; i++) { + if (marked_.at<char>(i, col) == what) { + return i; + } + } + return -1; +} + +void KuhnMunkres::Run() { + TrySimpleCase(); + if (greedy_) + return; + while (!CheckIfOptimumIsFound()) { + while (true) { + auto point = FindUncoveredMinValPos(); + auto min_val = dm_.at<float>(point.y, point.x); + if (min_val > 0) { + UpdateDissimilarityMatrix(min_val); + } else { + marked_.at<char>(point.y, point.x) = kPrime; + int col = FindInRow(point.y, kStar); + if (col >= 0) { + is_row_visited_[point.y] = 1; + is_col_visited_[col] = 0; + } else { + int count = 0; + points_[count] = point; + + while (true) { + int row = FindInCol(points_[count].x, kStar); + if (row >= 0) { + count++; + points_[count] = cv::Point(points_[count - 1].x, row); + int col = FindInRow(points_[count].y, kPrime); + count++; + points_[count] = cv::Point(col, points_[count - 1].y); + } else { + break; + } + } + + for (int i = 0; i < count + 1; i++) { + auto& mark = marked_.at<char>(points_[i].y, points_[i].x); + mark = mark == kStar ? 0 : kStar; + } + + is_row_visited_ = std::vector<int>(n_, 0); + is_col_visited_ = std::vector<int>(n_, 0); + + marked_.setTo(0, marked_ == kPrime); + break; + } + } + } + } +} diff --git a/python/openvino/runtime/common/demo_utils/src/performance_metrics.cpp b/python/openvino/runtime/common/demo_utils/src/performance_metrics.cpp new file mode 100644 index 0000000..d1e494e --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/src/performance_metrics.cpp @@ -0,0 +1,114 @@ +// Copyright (C) 2020 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include <limits> +#include "utils/performance_metrics.hpp" +#include "utils/slog.hpp" + +// timeWindow defines the length of the timespan over which the 'current fps' value is calculated +PerformanceMetrics::PerformanceMetrics(Duration timeWindow) + : timeWindowSize(timeWindow) + , firstFrameProcessed(false) +{} + +void PerformanceMetrics::update(TimePoint lastRequestStartTime, + const cv::Mat& frame, + cv::Point position, + int fontFace, + double fontScale, + cv::Scalar color, + int thickness, + MetricTypes metricType) { + update(lastRequestStartTime); + paintMetrics(frame, position, fontFace, fontScale, color, thickness, metricType); +} + +void PerformanceMetrics::update(TimePoint lastRequestStartTime) { + TimePoint currentTime = Clock::now(); + + if (!firstFrameProcessed) { + lastUpdateTime = lastRequestStartTime; + firstFrameProcessed = true; + } + + currentMovingStatistic.latency += currentTime - lastRequestStartTime; + currentMovingStatistic.period = currentTime - lastUpdateTime; + currentMovingStatistic.frameCount++; + + if (currentTime - lastUpdateTime > timeWindowSize) { + lastMovingStatistic = currentMovingStatistic; + totalStatistic.combine(lastMovingStatistic); + currentMovingStatistic = Statistic(); + + lastUpdateTime = currentTime; + } +} + +void PerformanceMetrics::paintMetrics(const cv::Mat& frame, cv::Point position, int fontFace, + double fontScale, cv::Scalar color, int thickness, MetricTypes metricType) const { + // Draw performance stats over frame + Metrics metrics = getLast(); + + std::ostringstream out; + if (!std::isnan(metrics.latency) && + (metricType == PerformanceMetrics::MetricTypes::LATENCY || metricType == PerformanceMetrics::MetricTypes::ALL)) { + out << "Latency: " << std::fixed << std::setprecision(1) << metrics.latency << " ms"; + putHighlightedText(frame, out.str(), position, fontFace, fontScale, color, thickness); + } + if (!std::isnan(metrics.fps) && + (metricType == PerformanceMetrics::MetricTypes::FPS || metricType == PerformanceMetrics::MetricTypes::ALL)) { + out.str(""); + out << "FPS: " << std::fixed << std::setprecision(1) << metrics.fps; + int offset = metricType == PerformanceMetrics::MetricTypes::ALL ? 30 : 0; + putHighlightedText(frame, out.str(), {position.x, position.y + offset}, fontFace, fontScale, color, thickness); + } +} + +PerformanceMetrics::Metrics PerformanceMetrics::getLast() const { + Metrics metrics; + + metrics.latency = lastMovingStatistic.frameCount != 0 + ? std::chrono::duration_cast<Ms>(lastMovingStatistic.latency).count() + / lastMovingStatistic.frameCount + : std::numeric_limits<double>::signaling_NaN(); + metrics.fps = lastMovingStatistic.period != Duration::zero() + ? lastMovingStatistic.frameCount + / std::chrono::duration_cast<Sec>(lastMovingStatistic.period).count() + : std::numeric_limits<double>::signaling_NaN(); + + return metrics; +} + +PerformanceMetrics::Metrics PerformanceMetrics::getTotal() const { + Metrics metrics; + + int frameCount = totalStatistic.frameCount + currentMovingStatistic.frameCount; + if (frameCount != 0) { + metrics.latency = std::chrono::duration_cast<Ms>( + totalStatistic.latency + currentMovingStatistic.latency).count() / frameCount; + metrics.fps = frameCount / std::chrono::duration_cast<Sec>( + totalStatistic.period + currentMovingStatistic.period).count(); + } else { + metrics.latency = std::numeric_limits<double>::signaling_NaN(); + metrics.fps = std::numeric_limits<double>::signaling_NaN(); + } + + return metrics; +} + +void PerformanceMetrics::logTotal() const { + Metrics metrics = getTotal(); + + slog::info << "\tLatency: " << std::fixed << std::setprecision(1) << metrics.latency << " ms" << slog::endl; + slog::info << "\tFPS: " << metrics.fps << slog::endl; +} + +void logLatencyPerStage(double readLat, double preprocLat, double inferLat, double postprocLat, double renderLat) { + slog::info << "\tDecoding:\t" << std::fixed << std::setprecision(1) << + readLat << " ms" << slog::endl; + slog::info << "\tPreprocessing:\t" << preprocLat << " ms" << slog::endl; + slog::info << "\tInference:\t" << inferLat << " ms" << slog::endl; + slog::info << "\tPostprocessing:\t" << postprocLat << " ms" << slog::endl; + slog::info << "\tRendering:\t" << renderLat << " ms" << slog::endl; +} diff --git a/python/openvino/runtime/common/demo_utils/src/w_dirent.hpp b/python/openvino/runtime/common/demo_utils/src/w_dirent.hpp new file mode 100644 index 0000000..0df8636 --- /dev/null +++ b/python/openvino/runtime/common/demo_utils/src/w_dirent.hpp @@ -0,0 +1,114 @@ +// Copyright (C) 2018-2019 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#pragma once + +#if defined(_WIN32) + +#ifndef NOMINMAX +# define NOMINMAX +#endif + +#include <WinSock2.h> +#include <Windows.h> +#include <stdlib.h> + +#else + +#include <unistd.h> +#include <cstdlib> +#include <string.h> + +#endif + +#include <string> + +#include <sys/stat.h> + +#if defined(WIN32) + // Copied from linux libc sys/stat.h: + #define S_ISREG(m) (((m) & S_IFMT) == S_IFREG) + #define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) +#endif + +struct dirent { + char *d_name; + + explicit dirent(const wchar_t *wsFilePath) { + size_t i; + auto slen = wcslen(wsFilePath); + d_name = static_cast<char*>(malloc(slen + 1)); + wcstombs_s(&i, d_name, slen + 1, wsFilePath, slen); + } + + ~dirent() { + free(d_name); + } +}; + +class DIR { + WIN32_FIND_DATAA FindFileData; + HANDLE hFind; + dirent *next; + + static inline bool endsWith(const std::string &src, const char *with) { + int wl = static_cast<int>(strlen(with)); + int so = static_cast<int>(src.length()) - wl; + if (so < 0) return false; + return 0 == strncmp(with, &src[so], wl); + } + +public: + explicit DIR(const char *dirPath) : next(nullptr) { + std::string ws = dirPath; + if (endsWith(ws, "\\")) + ws += "*"; + else + ws += "\\*"; + hFind = FindFirstFileA(ws.c_str(), &FindFileData); + FindFileData.dwReserved0 = hFind != INVALID_HANDLE_VALUE; + } + + ~DIR() { + if (!next) delete next; + FindClose(hFind); + } + + bool isValid() const { + return (hFind != INVALID_HANDLE_VALUE && FindFileData.dwReserved0); + } + + dirent* nextEnt() { + if (next != nullptr) delete next; + next = nullptr; + + if (!FindFileData.dwReserved0) return nullptr; + + wchar_t wbuf[4096]; + + size_t outSize; + mbstowcs_s(&outSize, wbuf, 4094, FindFileData.cFileName, 4094); + next = new dirent(wbuf); + FindFileData.dwReserved0 = FindNextFileA(hFind, &FindFileData); + return next; + } +}; + + +static DIR *opendir(const char* dirPath) { + auto dp = new DIR(dirPath); + if (!dp->isValid()) { + delete dp; + return nullptr; + } + return dp; +} + +static struct dirent *readdir(DIR *dp) { + return dp->nextEnt(); +} + +static void closedir(DIR *dp) { + delete dp; +} |