diff options
Diffstat (limited to 'python/openvino/runtime/streaming/image_streaming_app')
18 files changed, 1790 insertions, 0 deletions
diff --git a/python/openvino/runtime/streaming/image_streaming_app/CMakeLists.txt b/python/openvino/runtime/streaming/image_streaming_app/CMakeLists.txt new file mode 100644 index 0000000..4ac6627 --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/CMakeLists.txt @@ -0,0 +1,36 @@ +# Copyright 2023 Intel Corporation + +# This software and the related documents are Intel copyrighted materials, +# and your use of them is governed by the express license under which they +# were provided to you ("License"). Unless the License provides otherwise, +# you may not use, modify, copy, publish, distribute, disclose or transmit +# this software or the related documents without Intel's prior written +# permission. + +# This software and the related documents are provided as is, with no express +# or implied warranties, other than those that are expressly stated in the +# License. + +project(image_streaming_app) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +set(all_files ${header_files} + command_line.cpp + command_line.h + image_streaming_app.cpp + image_streaming_app.h + raw_image.cpp + raw_image.h + bmp_file.cpp + bmp_file.h + float16.h) + +# Targets +add_executable(${PROJECT_NAME} ${all_files}) + +add_subdirectory(uio) +add_subdirectory(layout_transform) + +target_link_libraries(${PROJECT_NAME} layout_transform) diff --git a/python/openvino/runtime/streaming/image_streaming_app/bmp_file.cpp b/python/openvino/runtime/streaming/image_streaming_app/bmp_file.cpp new file mode 100644 index 0000000..6502897 --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/bmp_file.cpp @@ -0,0 +1,277 @@ +// Copyright 2023 Intel Corporation. +// +// This software and the related documents are Intel copyrighted materials, +// and your use of them is governed by the express license under which they +// were provided to you ("License"). Unless the License provides otherwise, +// you may not use, modify, copy, publish, distribute, disclose or transmit +// this software or the related documents without Intel's prior written +// permission. +// +// This software and the related documents are provided as is, with no express +// or implied warranties, other than those that are expressly stated in the +// License. + +#include "bmp_file.h" +#include <cstdlib> +#include <cstring> +#include <fstream> + +BmpFile::BmpFile(const std::string& filename, bool planarBGR) { + bool loaded = LoadFile(filename, planarBGR); + (void)loaded; +} + +bool BmpFile::LoadFile(const std::string& filename, bool planarBGR) { + std::ifstream inputFile(filename, std::fstream::binary); + if (inputFile.bad()) { + return false; + } + + // Read signature + uint16_t fileSignature = 0; + if (!inputFile.read((char*)&fileSignature, sizeof(fileSignature))) { + return false; + } + + if (fileSignature != 0x4d42) { + return false; + } + + // Read file size + uint32_t fileSize = 0; + if (!inputFile.read((char*)&fileSize, sizeof(fileSize))) { + return false; + } + + if (fileSize > (8192 * 4320 * 3)) { // Check excessive file size + return false; + } + + // Reserved + uint32_t unused = 0; + if (!inputFile.read((char*)&unused, sizeof(unused))) { + return false; + } + + // Read data offset + uint32_t dataOffset = 0; + if (!inputFile.read((char*)&dataOffset, sizeof(dataOffset))) { + return false; + } + + if ((dataOffset >= fileSize) or (dataOffset == 0)) { + return false; + } + + // Read bitmap header + BitmapHeader infoHeader{}; + if (!inputFile.read((char*)&infoHeader, sizeof(infoHeader))) { + return false; + } + + uint32_t headerSize = sizeof(infoHeader); + uint32_t header4Size = 108; // sizeof(BITMAPV4HEADER); + uint32_t header5Size = 124; // sizeof(BITMAPV5HEADER); + if ((infoHeader._size != headerSize) and (infoHeader._size != header4Size) and (infoHeader._size != header5Size)) { + return false; + } + + int palletteSize = infoHeader._colorUsed; + std::vector<uint32_t> pallette; + if ((infoHeader._bitCount < 16) and (infoHeader._colorUsed == 0) and (infoHeader._bitCount != 1)) { + palletteSize = 1 << infoHeader._bitCount; + } + + if (palletteSize > 0) { + // V3 Pallette follows 4 bytes per entry + pallette.resize(palletteSize); + if (!inputFile.read((char*)pallette.data(), pallette.size())) { + return false; + } + } + + inputFile.seekg(dataOffset); + + uint32_t height = static_cast<uint32_t>(std::abs(infoHeader._height)); + size_t dataSize = static_cast<size_t>(infoHeader._sizeImage); + uint32_t nPixels = height * static_cast<uint32_t>(infoHeader._width); + + if (infoHeader._bitCount == 32) { + dataSize = height * infoHeader._width * 4; + } else if (infoHeader._bitCount == 16) { + dataSize = height * infoHeader._width * 2; + } else if (infoHeader._bitCount == 8) { + if (dataSize == 0) dataSize = height * infoHeader._width; // 8 bit data - through pallette + } else { + uint32_t line_length = infoHeader._width; + if ((infoHeader._bitCount == 24) and ((infoHeader._width % 4) != 0)) { + line_length = (infoHeader._width + 4) & ~3; + } + dataSize = height * line_length * 3; + } + + std::vector<uint8_t> _temporaryBuffer; + bool useTemporaryBuffer = (infoHeader._bitCount == 16) or (infoHeader._bitCount == 1) or (palletteSize > 0); + + if (useTemporaryBuffer) { + _temporaryBuffer.resize(dataSize); + if (!inputFile.read((char*)_temporaryBuffer.data(), dataSize)) return false; + } else { + _data.resize(dataSize); + if (!inputFile.read((char*)_data.data(), dataSize)) return false; + } + + if (infoHeader._bitCount == 16) { + int inputStride = infoHeader._sizeImage / height; + + dataSize = nPixels * 4; + _data.resize(dataSize); + uint32_t* pOutputScan = reinterpret_cast<uint32_t*>(_data.data()); + + for (uint32_t y = 0; y < height; y++) { + uint8_t* pInputLineStart = _temporaryBuffer.data() + (y * inputStride); + uint16_t* pInputScan = (uint16_t*)pInputLineStart; + + for (int x = 0; x < infoHeader._width; x++) { + uint16_t inputValue = *pInputScan++; + uint32_t r = ((inputValue & 0x7C00) >> 10) * 8; + uint32_t g = ((inputValue & 0x3E0) >> 5) * 8; + uint32_t b = ((inputValue & 0x1f) * 8); + + *pOutputScan++ = 0xff000000 | r << 16 | g << 8 | b; + } + } + + infoHeader._bitCount = 32; + } else if (infoHeader._bitCount == 1) { + int inputStride = infoHeader._sizeImage / height; + + dataSize = nPixels * 4; + _data.resize(dataSize); + uint32_t* pOutputScan = reinterpret_cast<uint32_t*>(_data.data()); + + for (uint32_t y = 0; y < height; y++) { + uint8_t* pInputLineStart = _temporaryBuffer.data() + (y * inputStride); + uint8_t* pInputScan = pInputLineStart; + + uint16_t inputValue = *pInputScan++; + for (int x = 0; x < infoHeader._width; x++) { + int bit = x % 8; + if (bit == 0) { + inputValue = *pInputScan++; + } + + int bit_mask = 1 << (7 - bit); + + if ((inputValue & bit_mask) == 0) + *pOutputScan++ = 0xff000000; + else + *pOutputScan++ = 0xffffffff; + } + } + + infoHeader._bitCount = 32; + } + + if (palletteSize > 0) { + // we're using a pallette - convert _buffer using pallette + _data.resize(dataSize * sizeof(uint32_t)); + uint32_t* pOutputScan = reinterpret_cast<uint32_t*>(_data.data()); + infoHeader._bitCount = 32; // pretend were now 32 bits as that is format of Pallette + for (size_t i = 0; i < dataSize; i++) { + *pOutputScan++ = pallette[_temporaryBuffer[i]]; + } + } + + _height = height; + _width = infoHeader._width; + _bitsPerPixel = infoHeader._bitCount; + + uint32_t lineLengthBytes = (_width * _bitsPerPixel) / 8; + + if ((_bitsPerPixel == 24) and ((lineLengthBytes % 4) != 0)) { + _stride = (lineLengthBytes + 4) & ~3; + } else { + _stride = lineLengthBytes; + } + + _upsideDown = (infoHeader._height > 0); + + // BMP channel order is BGR, as required by ResNet + if (_upsideDown) { + std::vector<uint8_t> flippedData(_data.size()); + for (uint32_t y = 0; y < _height; y++) { + uint8_t* pDestinationLine = flippedData.data() + (y * _stride); + uint8_t* pSourceLine = _data.data() + ((_height - y - 1) * _stride); + + std::memcpy(pDestinationLine, pSourceLine, _stride); + } + + _data = flippedData; + } + + if (planarBGR) { + uint32_t channelSize = _width * _height; + std::vector<uint8_t> planarData(_data.size()); + uint8_t* pBPlane = planarData.data(); + uint8_t* pGPlane = pBPlane + channelSize; + uint8_t* pRPlane = pGPlane + channelSize; + uint8_t* pInputBGR = _data.data(); + + for (uint32_t i = 0; i < channelSize; i++) { + *pBPlane++ = *pInputBGR++; + *pGPlane++ = *pInputBGR++; + *pRPlane++ = *pInputBGR++; + + // Skip alpha channel + if (infoHeader._bitCount == 32) { + pInputBGR++; + } + } + + _data = planarData; + } else { + uint32_t channelSize = _width * _height; + + // Must be 32bpp + if (infoHeader._bitCount == 32) { + // Swap endianness + uint8_t* pInputBGR = _data.data(); + + for (uint32_t i = 0; i < channelSize; i++) { + uint8_t b = pInputBGR[0]; + uint8_t g = pInputBGR[1]; + uint8_t r = pInputBGR[2]; + uint8_t a = pInputBGR[3]; + + pInputBGR[0] = a; + pInputBGR[1] = r; + pInputBGR[2] = g; + pInputBGR[3] = b; + + pInputBGR += 4; + } + } else { + std::vector<uint8_t> newData(channelSize * 4); + uint8_t* pInputBGR = _data.data(); + uint8_t* pOutputBGRA = newData.data(); + for (uint32_t i = 0; i < channelSize; i++) { + uint8_t b = pInputBGR[0]; + uint8_t g = pInputBGR[1]; + uint8_t r = pInputBGR[2]; + + pOutputBGRA[0] = 0; + pOutputBGRA[1] = r; + pOutputBGRA[2] = g; + pOutputBGRA[3] = b; + + pInputBGR += 3; + pOutputBGRA += 4; + } + + _data = newData; + } + } + + return true; +} diff --git a/python/openvino/runtime/streaming/image_streaming_app/bmp_file.h b/python/openvino/runtime/streaming/image_streaming_app/bmp_file.h new file mode 100644 index 0000000..95ab306 --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/bmp_file.h @@ -0,0 +1,47 @@ +// Copyright 2023 Intel Corporation. +// +// This software and the related documents are Intel copyrighted materials, +// and your use of them is governed by the express license under which they +// were provided to you ("License"). Unless the License provides otherwise, +// you may not use, modify, copy, publish, distribute, disclose or transmit +// this software or the related documents without Intel's prior written +// permission. +// +// This software and the related documents are provided as is, with no express +// or implied warranties, other than those that are expressly stated in the +// License. + +#pragma once +#include <cstdint> +#include <string> +#include <vector> + +struct BitmapHeader { + uint32_t _size; + int32_t _width; + int32_t _height; + uint16_t _planes; + uint16_t _bitCount; + uint32_t _compression; + uint32_t _sizeImage; + int32_t _xPixelsPerMeter; + int32_t _yPixelsPerMeter; + uint32_t _colorUsed; + uint32_t _colorImportant; +}; + +class BmpFile { + public: + BmpFile(const std::string& filename, bool planarBGR); + std::vector<uint8_t>& GetData() { return _data; } + uint32_t GetNumPixels() { return (_width * _height); } + + private: + bool LoadFile(const std::string& filename, bool planarBGR); + std::vector<uint8_t> _data; + uint32_t _width = 0; + uint32_t _height = 0; + uint32_t _bitsPerPixel = 0; + uint32_t _stride = 0; + bool _upsideDown = false; +}; diff --git a/python/openvino/runtime/streaming/image_streaming_app/command_line.cpp b/python/openvino/runtime/streaming/image_streaming_app/command_line.cpp new file mode 100644 index 0000000..794310b --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/command_line.cpp @@ -0,0 +1,72 @@ +// Copyright 2021-2023 Intel Corporation. +// +// This software and the related documents are Intel copyrighted materials, +// and your use of them is governed by the express license under which they +// were provided to you ("License"). Unless the License provides otherwise, +// you may not use, modify, copy, publish, distribute, disclose or transmit +// this software or the related documents without Intel's prior written +// permission. +// +// This software and the related documents are provided as is, with no express +// or implied warranties, other than those that are expressly stated in the +// License. + +#include "command_line.h" +#include <algorithm> + +static void TrimString(std::string& trimString) { + trimString.erase(0, trimString.find_first_not_of(" \n\r\t")); + trimString.erase(trimString.find_last_not_of(" \n\r\t") + 1); +} + +static void MakeLower(std::string& stringValue) { + std::transform(stringValue.begin(), stringValue.end(), stringValue.begin(), ::tolower); +} + +// Program -option=value +CommandLine::CommandLine(int argumentCount, char* argumentValues[]) { + if (argumentCount > 0) _executableName = argumentValues[0]; + + for (int i = 1; i < argumentCount; i++) { + std::string inputString(argumentValues[i]); + std::string nextChar = inputString.substr(0, 1); + if ((nextChar == "-") or (nextChar == "/")) { + inputString = inputString.substr(1); + size_t equals = inputString.find("="); + std::string option; + std::string value; + + if (equals == std::string::npos) { + option = inputString; + } else { + option = inputString.substr(0, equals); + value = inputString.substr(equals + 1); + } + + TrimString(option); + TrimString(value); + MakeLower(option); + _optionMap[option] = value; + } + } +} + +std::string CommandLine::GetOptionValue(const char* optionName) { + auto i = _optionMap.find(optionName); + if (i != _optionMap.end()) + return i->second; + else + return ""; +} + +bool CommandLine::HaveOption(const char* optionName) { return (_optionMap.find(optionName) != _optionMap.end()); } + +bool CommandLine::GetOption(const char* optionName, std::string& optionValue) { + auto i = _optionMap.find(optionName); + if (i == _optionMap.end()) return false; + + optionValue = i->second; + return true; +} + +size_t CommandLine::NumOptions() { return _optionMap.size(); } diff --git a/python/openvino/runtime/streaming/image_streaming_app/command_line.h b/python/openvino/runtime/streaming/image_streaming_app/command_line.h new file mode 100644 index 0000000..41b12f0 --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/command_line.h @@ -0,0 +1,31 @@ +// Copyright 2021-2023 Intel Corporation. +// +// This software and the related documents are Intel copyrighted materials, +// and your use of them is governed by the express license under which they +// were provided to you ("License"). Unless the License provides otherwise, +// you may not use, modify, copy, publish, distribute, disclose or transmit +// this software or the related documents without Intel's prior written +// permission. +// +// This software and the related documents are provided as is, with no express +// or implied warranties, other than those that are expressly stated in the +// License. + +#pragma once +#include <string> +#include <unordered_map> + +class CommandLine { + public: + CommandLine(int argumentCount, char* argumentValues[]); + + std::string GetOptionValue(const char* optionName); + bool GetOption(const char* optionName, std::string& optionValue); + bool HaveOption(const char* optionName); + std::string GetExecutableName() { return _executableName; } + size_t NumOptions(); + + private: + std::string _executableName; + std::unordered_map<std::string, std::string> _optionMap; +}; diff --git a/python/openvino/runtime/streaming/image_streaming_app/float16.h b/python/openvino/runtime/streaming/image_streaming_app/float16.h new file mode 100644 index 0000000..07b8ece --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/float16.h @@ -0,0 +1,204 @@ +// Copyright 2023 Intel Corporation. +// +// This software and the related documents are Intel copyrighted materials, +// and your use of them is governed by the express license under which they +// were provided to you ("License"). Unless the License provides otherwise, +// you may not use, modify, copy, publish, distribute, disclose or transmit +// this software or the related documents without Intel's prior written +// permission. +// +// This software and the related documents are provided as is, with no express +// or implied warranties, other than those that are expressly stated in the +// License. + +/* +Copyright (C) 2009 Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +/* + * This file helps with conversion to/from fp16. The contents of this file are duplicated + * in at least four places, just to ensure they are not accidentally lost: + * + * 1) runtime/plugin/io_transformations/dlia_fp.hpp. + * 2) hps/apps/coredla-test-harness/float16.h (Intel internal) + * 3) runtime/streaming/image_streaming_app/float16.h + * 4) util/inc/dla_element.h (internal) + * + * The algorithm for conversion to fp16 is described in this 2008 paper by Jeroen van der Zijp + * http://www.fox-toolkit.org/ftp/fasthalffloatconversion.pdf + * + * We received this code, I am told, by way of OpenVINO. As best as I can infer from + * the copyright header and available bits of discussion on the web, OpenVINO seems to have + * copied this from Webkit. + */ + +#pragma once + +#include <cstdint> +#include <cstring> + +class Float16 { + public: + Float16(float value) { + // This method is used by the CoreDLA plugin, it is not binary equivalent + // to f32to16 (below) used by OpenVino + union FloatAndUint32 { + float _floatValue; + uint32_t _uint32Value; + }; + FloatAndUint32 convertType; + convertType._floatValue = value; + + uint32_t floatBits = convertType._uint32Value; + uint32_t shiftIndex = (floatBits >> 23) & 0x1ff; + _uintValue = base[shiftIndex] + ((floatBits & 0x007fffff) >> shift[shiftIndex]); + } + + Float16() {} + operator uint16_t() { return _uintValue; } + uint16_t _uintValue; + + const uint16_t base[512] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, 2, 4, 8, 16, 32, 64, 128, 256, + 512, 1024, 2048, 3072, 4096, 5120, 6144, 7168, 8192, 9216, 10240, 11264, 12288, 13312, 14336, 15360, + 16384, 17408, 18432, 19456, 20480, 21504, 22528, 23552, 24576, 25600, 26624, 27648, 28672, 29696, 30720, 31744, + 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, + 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, + 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, + 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, + 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, + 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, + 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, 31744, + 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, + 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, + 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, + 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, + 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, + 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768, + 32768, 32768, 32768, 32768, 32768, 32768, 32768, 32769, 32770, 32772, 32776, 32784, 32800, 32832, 32896, 33024, + 33280, 33792, 34816, 35840, 36864, 37888, 38912, 39936, 40960, 41984, 43008, 44032, 45056, 46080, 47104, 48128, + 49152, 50176, 51200, 52224, 53248, 54272, 55296, 56320, 57344, 58368, 59392, 60416, 61440, 62464, 63488, 64512, + 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, + 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, + 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, + 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, + 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, + 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, + 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512, 64512}; + + const uint8_t shift[512] = { + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, + 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13}; + +// Function to convert F32 into F16 +// F32: exp_bias:127 SEEEEEEE EMMMMMMM MMMMMMMM MMMMMMMM. +// F16: exp_bias:15 SEEEEEMM MMMMMMMM +#define EXP_MASK_F32 0x7F800000U +#define EXP_MASK_F16 0x7C00U + + // small helper function to represent uint32_t value as float32 + inline float asfloat(uint32_t v) { + // Both type-punning casts and unions are UB per C++ spec + // But compilers usually only break code with casts + union { + float f; + uint32_t i; + }; + i = v; + return f; + } + + uint16_t f32tof16_OpenVino(float x) { + // create minimal positive normal f16 value in f32 format + // exp:-14,mantissa:0 -> 2^-14 * 1.0 + static float min16 = asfloat((127 - 14) << 23); + + // create maximal positive normal f16 value in f32 and f16 formats + // exp:15,mantissa:11111 -> 2^15 * 1.(11111) + static float max16 = asfloat(((127 + 15) << 23) | 0x007FE000); + static uint32_t max16f16 = ((15 + 15) << 10) | 0x3FF; + + // define and declare variable for intermediate and output result + // the union is used to simplify representation changing + union { + float f; + uint32_t u; + } v; + v.f = x; + + // get sign in 16bit format + uint32_t s = (v.u >> 16) & 0x8000; // sign 16: 00000000 00000000 10000000 00000000 + + // make it abs + v.u &= 0x7FFFFFFF; // abs mask: 01111111 11111111 11111111 11111111 + + // check NAN and INF + if ((v.u & EXP_MASK_F32) == EXP_MASK_F32) { + if (v.u & 0x007FFFFF) { + return s | (v.u >> (23 - 10)) | 0x0200; // return NAN f16 + } else { + return s | EXP_MASK_F16; // return INF f16 + } + } + + // to make f32 round to nearest f16 + // create halfULP for f16 and add it to origin value + float halfULP = asfloat(v.u & EXP_MASK_F32) * asfloat((127 - 11) << 23); + v.f += halfULP; + + // if input value is not fit normalized f16 then return 0 + // denormals are not covered by this code and just converted to 0 + if (v.f < min16 * 0.5F) { + return s; + } + + // if input value between min16/2 and min16 then return min16 + if (v.f < min16) { + return s | (1 << 10); + } + + // if input value more than maximal allowed value for f16 + // then return this maximal value + if (v.f >= max16) { + return max16f16 | s; + } + + // change exp bias from 127 to 15 + v.u -= ((127 - 15) << 23); + + // round to f16 + v.u >>= (23 - 10); + + return v.u | s; + } +}; diff --git a/python/openvino/runtime/streaming/image_streaming_app/image_streaming_app.cpp b/python/openvino/runtime/streaming/image_streaming_app/image_streaming_app.cpp new file mode 100644 index 0000000..bf4cb2b --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/image_streaming_app.cpp @@ -0,0 +1,306 @@ +// Copyright 2023 Intel Corporation. +// +// This software and the related documents are Intel copyrighted materials, +// and your use of them is governed by the express license under which they +// were provided to you ("License"). Unless the License provides otherwise, +// you may not use, modify, copy, publish, distribute, disclose or transmit +// this software or the related documents without Intel's prior written +// permission. +// +// This software and the related documents are provided as is, with no express +// or implied warranties, other than those that are expressly stated in the +// License. + +#include "image_streaming_app.h" +#include <signal.h> +#include <algorithm> +#include <filesystem> +#include <fstream> +#include <iostream> +#include <string> +#include <thread> +#include <fcntl.h> +#include "raw_image.h" + +int main(int numParams, char* paramValues[]) { + ImageStreamingApp imageStreamingApp(numParams, paramValues); + imageStreamingApp.Run(); + return 0; +} + +volatile bool ImageStreamingApp::_shutdownEvent; + +ImageStreamingApp::ImageStreamingApp(int numParams, char* paramValues[]) : _commandLine(numParams, paramValues) { + std::string imagesFolder; + if (_commandLine.GetOption("images_folder", imagesFolder)) + _imageFilesFolder = imagesFolder; + else + _imageFilesFolder = "./"; + + std::string imageFile; + if (_commandLine.GetOption("image", imageFile)) { + _numToSend = 1; + _imageFile = imageFile; + } + + std::string nSendStr; + if (_commandLine.GetOption("send", nSendStr)) _numToSend = std::strtoul(nSendStr.c_str(), 0, 0); + + std::string rateStr; + if (_commandLine.GetOption("rate", rateStr)) _sendRate = std::strtoul(rateStr.c_str(), 0, 0); + + _dumpTransformedImages = _commandLine.HaveOption("dump"); + _runLayoutTransform = _commandLine.HaveOption("layout_transform"); + + _ltConfiguration._width = GetUintOption("width", 224); + _ltConfiguration._height = GetUintOption("height", 224); + _ltConfiguration._cVector = GetUintOption("c_vector", 32); + _ltConfiguration._blueVariance = GetFloatOption("blue_variance", 1.0f); + _ltConfiguration._greenVariance = GetFloatOption("green_variance", 1.0f); + _ltConfiguration._redVariance = GetFloatOption("red_variance", 1.0f); + _ltConfiguration._blueShift = GetFloatOption("blue_shift", -103.94f); + _ltConfiguration._greenShift = GetFloatOption("green_shift", -116.78f); + _ltConfiguration._redShift = GetFloatOption("red_shift", -123.68f); + + signal(SIGINT, SigIntHandler); +} + +void ImageStreamingApp::Run() { + if (_commandLine.HaveOption("-help")) { + std::cout << "Usage:\n"; + std::cout << " image_streaming_app [Options]\n"; + std::cout << "\nOptions:\n"; + std::cout << "-images_folder=folder Location of bitmap files. Defaults to working folder.\n"; + std::cout << "-image=path Location of a single bitmap file for single inference.\n"; + std::cout << "-send=n Number of images to stream. Default is 1 if -image is set, otherwise infinite.\n"; + std::cout << "-rate=n Rate to stream images, in Hz. n is an integer. Default is 30.\n"; + std::cout << "-width=n Image width in pixels, default = 224\n"; + std::cout << "-height=n Image height in pixels, default = 224\n"; + std::cout << "-c_vector=n C vector size, default = 32\n"; + std::cout << "-blue_variance=n Blue variance, default = 1.0\n"; + std::cout << "-green_variance=n Green variance, default = 1.0\n"; + std::cout << "-red_variance=n Red variance, default = 1.0\n"; + std::cout << "-blue_shift=n Blue shift, default = -103.94\n"; + std::cout << "-green_shift=n Green shift, default -116.78\n"; + std::cout << "-red_shift=n Red shift, default = -123.68\n"; + return; + } + + if (not ProgramLayoutTransform()) { + return; + } + + if (not LoadImageFiles(_dumpTransformedImages)) { + return; + } + + if (_dumpTransformedImages) { + return; + } + + if (not WaitForInferenceApp()) + return; + + // Start event signal thread + auto sendImageEventThreadCB = [this]() { RunSendImageSignalThread(); }; + std::thread sendImageEventThread(sendImageEventThreadCB); + uint32_t sentCount = 0; + + while (not _shutdownEvent) { + // Wait for the send image event + _sendNextImageEvent.Wait(); + + if (not SendNextImage()) { + _shutdownEvent = true; + break; + } + sentCount++; + + if ((_numToSend > 0) and (sentCount >= _numToSend)) { + _shutdownEvent = true; + break; + } + } + + // Wait for signalling thread to finish + sendImageEventThread.join(); +} + +bool ImageStreamingApp::LoadImageFiles(bool dumpLayoutTransform) { + if (not _imageFile.empty()) { + std::filesystem::path filePath(_imageFile); + std::string extension = filePath.extension(); + std::transform(extension.begin(), extension.end(), extension.begin(), ::tolower); + if ((extension == ".bmp") or (extension == ".raw") or (extension == ".lt")) { + auto spRawImage = std::make_shared<RawImage>(filePath, _runLayoutTransform, _ltConfiguration); + if (spRawImage->IsValid()) { + _images.push_back(spRawImage); + + if (dumpLayoutTransform and _runLayoutTransform) { + spRawImage->DumpLayoutTransform(); + } + } else { + std::cout << "Unsupported image: " << filePath << '\n'; + } + } + } else { + for (const auto& entry : std::filesystem::directory_iterator(_imageFilesFolder)) { + std::string filename = entry.path(); + std::filesystem::path filePath(filename); + std::string extension = filePath.extension(); + std::transform(extension.begin(), extension.end(), extension.begin(), ::tolower); + if ((extension == ".bmp") or (extension == ".raw") or (extension == ".lt")) { + auto spRawImage = std::make_shared<RawImage>(filePath, _runLayoutTransform, _ltConfiguration); + _images.push_back(spRawImage); + + if (dumpLayoutTransform and _runLayoutTransform) { + spRawImage->DumpLayoutTransform(); + } + + // Don't load any more than we need to send + if (_images.size() == _numToSend) { + break; + } + } + } + } + + std::cout << "Loaded " << _images.size() << " image"; + if (_images.size() > 1) { + std::cout << "s"; + } + std::cout << '\n'; + return not _images.empty(); +} + +bool ImageStreamingApp::OpenMsgDmaStream() { + if (_msgDmaStream) { + return true; + } + + constexpr const char* msgdmaFilename = "/dev/msgdma_stream0"; + _msgDmaStream = ::fopen(msgdmaFilename, "w+"); + if (_msgDmaStream == NULL) { + std::cout << "Failed to open" << '\n'; + return false; + } + + // Turn off output buffering + setvbuf(_msgDmaStream, NULL, _IONBF, 0); + + return true; +} + +void ImageStreamingApp::CloseMsgDmaStream() { + if (_msgDmaStream) { + ::fclose(_msgDmaStream); + _msgDmaStream = nullptr; + } +} + +bool ImageStreamingApp::SendNextImage() { + size_t nImages = _images.size(); + if (nImages == 0) { + return false; + } + + if (not _msgDmaStream) { + if (not OpenMsgDmaStream()) { + return false; + } + } + + std::shared_ptr<RawImage> uploadImage = _images[_nextImageIndex]; + + // Move to next index for next time + _nextImageIndex = (_nextImageIndex + 1) % nImages; + _sentCount++; + + char* pBuffer = reinterpret_cast<char*>(uploadImage->GetData()); + size_t bufferSize = uploadImage->GetSize(); + + std::cout << _sentCount << " Send image " << uploadImage->Filename() << " size = " << bufferSize; + + size_t nWritten = ::fwrite(pBuffer, 1, bufferSize, _msgDmaStream); + bool ok = (nWritten == bufferSize); + if (ok) { + std::cout << '\n'; + } else { + std::cout << " failed\n"; + } + + return ok; +} + +void ImageStreamingApp::RunSendImageSignalThread() { + int64_t microSeconds = 1000000 / _sendRate; + if (_sendRate == 59) { + microSeconds = 16683; // 59.94 Hz + } + + while (not _shutdownEvent) { + std::this_thread::sleep_for(std::chrono::microseconds(microSeconds)); + _sendNextImageEvent.Set(); + } +} + +bool ImageStreamingApp::ProgramLayoutTransform() { + auto spLayoutTransform = ILayoutTransform::Create(); + spLayoutTransform->SetConfiguration(_ltConfiguration); + return true; +} + +uint32_t ImageStreamingApp::GetUintOption(const char* optionName, uint32_t defaultValue) { + std::string optionValue; + if (_commandLine.GetOption(optionName, optionValue)) { + return std::strtoul(optionValue.c_str(), nullptr, 0); + } else { + return defaultValue; + } +} + +float ImageStreamingApp::GetFloatOption(const char* optionName, float defaultValue) { + std::string optionValue; + if (_commandLine.GetOption(optionName, optionValue)) { + return std::strtof(optionValue.c_str(), nullptr); + } else { + return defaultValue; + } +} + +void ImageStreamingApp::SigIntHandler(int) { + std::cout << "\nShutting down application\n"; + _shutdownEvent = true; +} + +bool ImageStreamingApp::WaitForInferenceApp() { + bool isReady = false; + bool firstTime = true; + sem_t* pSemaphore = ::sem_open("/CoreDLA_ready_for_streaming", O_CREAT, 0644, 0); + if (!pSemaphore) { + return isReady; + } + + while (not _shutdownEvent) { + // Don't use a wait timeout because we need to break + // if the user presses Ctrl+C + timespec waitTimeout = {}; + int r = ::sem_timedwait(pSemaphore, &waitTimeout); + if (r == 0) { + isReady = true; + ::sem_post(pSemaphore); + break; + } + + if (firstTime) { + firstTime = false; + std::cout << "Waiting for streaming_inference_app to become ready." << std::endl; + } + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + + ::sem_close(pSemaphore); + + return isReady; +} diff --git a/python/openvino/runtime/streaming/image_streaming_app/image_streaming_app.h b/python/openvino/runtime/streaming/image_streaming_app/image_streaming_app.h new file mode 100644 index 0000000..0693ef8 --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/image_streaming_app.h @@ -0,0 +1,79 @@ +// Copyright 2023 Intel Corporation. +// +// This software and the related documents are Intel copyrighted materials, +// and your use of them is governed by the express license under which they +// were provided to you ("License"). Unless the License provides otherwise, +// you may not use, modify, copy, publish, distribute, disclose or transmit +// this software or the related documents without Intel's prior written +// permission. +// +// This software and the related documents are provided as is, with no express +// or implied warranties, other than those that are expressly stated in the +// License. + +#pragma once +#include <condition_variable> +#include <filesystem> +#include <vector> +#include <semaphore.h> +#include "ILayoutTransform.h" +#include "command_line.h" + +using namespace std::chrono_literals; + +class RawImage; + +class Event { + public: + void Wait() { + std::unique_lock<std::mutex> lock(_signalMutex); + _conditionVariable.wait(lock); + } + + void Set() { + std::lock_guard<std::mutex> lock(_signalMutex); + _conditionVariable.notify_all(); + } + + bool IsSignalled() { + std::unique_lock<std::mutex> lock(_signalMutex); + return (_conditionVariable.wait_for(lock, 0ms) != std::cv_status::timeout); + } + + private: + std::mutex _signalMutex; + std::condition_variable _conditionVariable; +}; + +class ImageStreamingApp { + public: + ImageStreamingApp(int numParams, char* paramValues[]); + void Run(); + + private: + bool ProgramLayoutTransform(); + bool SendNextImage(); + bool LoadImageFiles(bool dumpLayoutTransform); + void RunSendImageSignalThread(); + static void SigIntHandler(int); + uint32_t GetUintOption(const char* optionName, uint32_t defaultValue); + float GetFloatOption(const char* optionName, float defaultValue); + bool OpenMsgDmaStream(); + void CloseMsgDmaStream(); + bool WaitForInferenceApp(); + + CommandLine _commandLine; + std::filesystem::path _imageFilesFolder; + std::string _imageFile; + std::vector<std::shared_ptr<RawImage>> _images; + Event _sendNextImageEvent; + static volatile bool _shutdownEvent; + size_t _nextImageIndex = 0; + uint32_t _numToSend = 0; + uint32_t _sendRate = 30; + uint32_t _sentCount = 0; + bool _dumpTransformedImages = false; + bool _runLayoutTransform = true; + FILE* _msgDmaStream = nullptr; + ILayoutTransform::Configuration _ltConfiguration = {}; +}; diff --git a/python/openvino/runtime/streaming/image_streaming_app/layout_transform/CMakeLists.txt b/python/openvino/runtime/streaming/image_streaming_app/layout_transform/CMakeLists.txt new file mode 100644 index 0000000..ecbffca --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/layout_transform/CMakeLists.txt @@ -0,0 +1,35 @@ +# Copyright 2023 Intel Corporation +# +# This software and the related documents are Intel copyrighted materials, +# and your use of them is governed by the express license under which they +# were provided to you ("License"). Unless the License provides otherwise, +# you may not use, modify, copy, publish, distribute, disclose or transmit +# this software or the related documents without Intel's prior written +# permission. +# +# This software and the related documents are provided as is, with no express +# or implied warranties, other than those that are expressly stated in the +# License. + +project(layout_transform) + +set(header_files "") +set(source_files "") + +# specify the C++ standard +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +# Header files +set(header_files ${header_files} "include/ILayoutTransform.h") +set(header_files ${header_files} "source/LayoutTransform.h") + +# Source files +set(source_files ${source_files} "source/LayoutTransform.cpp") + +set(all_files ${header_files} ${source_files}) + +add_library(${PROJECT_NAME} STATIC ${all_files}) +target_include_directories(${PROJECT_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") +target_link_libraries(${PROJECT_NAME} uio) + diff --git a/python/openvino/runtime/streaming/image_streaming_app/layout_transform/include/ILayoutTransform.h b/python/openvino/runtime/streaming/image_streaming_app/layout_transform/include/ILayoutTransform.h new file mode 100644 index 0000000..f54f9d5 --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/layout_transform/include/ILayoutTransform.h @@ -0,0 +1,38 @@ +// Copyright 2023 Intel Corporation. +// +// This software and the related documents are Intel copyrighted materials, +// and your use of them is governed by the express license under which they +// were provided to you ("License"). Unless the License provides otherwise, +// you may not use, modify, copy, publish, distribute, disclose or transmit +// this software or the related documents without Intel's prior written +// permission. +// +// This software and the related documents are provided as is, with no express +// or implied warranties, other than those that are expressly stated in the +// License. + +#pragma once + +#include <cstdint> +#include <memory> + +class ILayoutTransform { + public: + class Configuration { + public: + uint32_t _width; + uint32_t _height; + uint32_t _cVector; + float _blueVariance; + float _greenVariance; + float _redVariance; + float _blueShift; + float _greenShift; + float _redShift; + }; + + virtual ~ILayoutTransform() {} + virtual void SetConfiguration(Configuration& configuration) = 0; + + static std::shared_ptr<ILayoutTransform> Create(); +}; diff --git a/python/openvino/runtime/streaming/image_streaming_app/layout_transform/source/LayoutTransform.cpp b/python/openvino/runtime/streaming/image_streaming_app/layout_transform/source/LayoutTransform.cpp new file mode 100644 index 0000000..a045b6a --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/layout_transform/source/LayoutTransform.cpp @@ -0,0 +1,51 @@ +// Copyright 2023 Intel Corporation. +// +// This software and the related documents are Intel copyrighted materials, +// and your use of them is governed by the express license under which they +// were provided to you ("License"). Unless the License provides otherwise, +// you may not use, modify, copy, publish, distribute, disclose or transmit +// this software or the related documents without Intel's prior written +// permission. +// +// This software and the related documents are provided as is, with no express +// or implied warranties, other than those that are expressly stated in the +// License. + +#include "LayoutTransform.h" +#include <thread> + +std::shared_ptr<ILayoutTransform> ILayoutTransform::Create() { return std::make_shared<LayoutTransform>(); } + +LayoutTransform::LayoutTransform() { _spUioDevice = UIO::IDevice::Load("layout_transform"); } + +static uint32_t FloatToUint32(float value) { + union FloatAndUint32 { + float _floatValue; + uint32_t _uint32Value; + }; + FloatAndUint32 convertType; + convertType._floatValue = value; + return convertType._uint32Value; +} + +void LayoutTransform::SetConfiguration(Configuration& configuration) { + _configuration = configuration; + + if (_spUioDevice) { + _spUioDevice->Write((uint32_t)RegisterMap::RESET, 1u); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + _spUioDevice->Write((uint32_t)RegisterMap::RESET, 0u); + + _spUioDevice->Write((uint32_t)RegisterMap::C_VECT, _configuration._cVector); + _spUioDevice->Write((uint32_t)RegisterMap::WIDTH, _configuration._width); + _spUioDevice->Write((uint32_t)RegisterMap::HEIGHT, _configuration._height); + + _spUioDevice->Write((uint32_t)RegisterMap::VARIANCES + 0, FloatToUint32(_configuration._blueVariance)); + _spUioDevice->Write((uint32_t)RegisterMap::VARIANCES + 1, FloatToUint32(_configuration._greenVariance)); + _spUioDevice->Write((uint32_t)RegisterMap::VARIANCES + 2, FloatToUint32(_configuration._redVariance)); + + _spUioDevice->Write((uint32_t)RegisterMap::SHIFTS + 0, FloatToUint32(_configuration._blueShift)); + _spUioDevice->Write((uint32_t)RegisterMap::SHIFTS + 1, FloatToUint32(_configuration._greenShift)); + _spUioDevice->Write((uint32_t)RegisterMap::SHIFTS + 2, FloatToUint32(_configuration._redShift)); + } +} diff --git a/python/openvino/runtime/streaming/image_streaming_app/layout_transform/source/LayoutTransform.h b/python/openvino/runtime/streaming/image_streaming_app/layout_transform/source/LayoutTransform.h new file mode 100644 index 0000000..3d67a5d --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/layout_transform/source/LayoutTransform.h @@ -0,0 +1,38 @@ +// Copyright 2023 Intel Corporation. +// +// This software and the related documents are Intel copyrighted materials, +// and your use of them is governed by the express license under which they +// were provided to you ("License"). Unless the License provides otherwise, +// you may not use, modify, copy, publish, distribute, disclose or transmit +// this software or the related documents without Intel's prior written +// permission. +// +// This software and the related documents are provided as is, with no express +// or implied warranties, other than those that are expressly stated in the +// License. + +#pragma once +#include <memory> +#include "ILayoutTransform.h" +#include "IUioDevice.h" + +enum class RegisterMap : uint32_t { + RESET = 0, + C_VECT, + WIDTH, + HEIGHT, + VARIANCES = 0x10, // to 0x1f + SHIFTS = 0x20, // to 0x2f +}; + +class LayoutTransform : public ILayoutTransform { + public: + LayoutTransform(); + + // ILayoutTransform interface + void SetConfiguration(Configuration& configuration) override; + + private: + ILayoutTransform::Configuration _configuration = {}; + std::shared_ptr<UIO::IDevice> _spUioDevice; +}; diff --git a/python/openvino/runtime/streaming/image_streaming_app/raw_image.cpp b/python/openvino/runtime/streaming/image_streaming_app/raw_image.cpp new file mode 100644 index 0000000..bd5658b --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/raw_image.cpp @@ -0,0 +1,225 @@ +// Copyright 2023 Intel Corporation. +// +// This software and the related documents are Intel copyrighted materials, +// and your use of them is governed by the express license under which they +// were provided to you ("License"). Unless the License provides otherwise, +// you may not use, modify, copy, publish, distribute, disclose or transmit +// this software or the related documents without Intel's prior written +// permission. +// +// This software and the related documents are provided as is, with no express +// or implied warranties, other than those that are expressly stated in the +// License. + +#include "raw_image.h" +#include <algorithm> +#include <filesystem> +#include <fstream> + +std::vector<int32_t> RawImage::_indexes; + +RawImage::RawImage(std::filesystem::path filePath, + bool runLayoutTransform, + const ILayoutTransform::Configuration& ltConfiguration) + : _filePath(filePath), _ltConfiguration(ltConfiguration) { + std::string extension = filePath.extension(); + std::transform(extension.begin(), extension.end(), extension.begin(), ::tolower); + if (extension == ".lt") { + uintmax_t fileSize = std::filesystem::file_size(filePath); + std::ifstream layoutFile(filePath, std::fstream::binary); + _layoutTransformData.resize(fileSize / sizeof(uint16_t)); + layoutFile.read((char*)_layoutTransformData.data(), fileSize); + } else { + bool planar = runLayoutTransform; + _spBmpFile = std::make_shared<BmpFile>(filePath, planar); + if (_runLayoutTransform) LayoutTransform(_ltConfiguration); + } +} + +uint8_t* RawImage::GetData() { + if (_runLayoutTransform) + return reinterpret_cast<uint8_t*>(_layoutTransformData.data()); + else + return _spBmpFile->GetData().data(); +} + +size_t RawImage::GetSize() { + if (_runLayoutTransform) + return _layoutTransformData.size() * sizeof(uint16_t); + else + return _spBmpFile->GetData().size(); +} + +bool RawImage::IsValid() +{ + constexpr size_t dlaImageSize = 224 * 224 * 4; + return (GetSize() == dlaImageSize); +} + +std::vector<uint16_t> RawImage::LayoutTransform(uint32_t width, + uint32_t height, + const std::vector<uint8_t>& sourceData, + const ILayoutTransform::Configuration& ltConfiguration) { + uint32_t numPixels = width * height; + std::vector<uint16_t> layoutTransformData = LayoutTransform(sourceData, numPixels, ltConfiguration); + return layoutTransformData; +} + +void RawImage::LayoutTransform(const ILayoutTransform::Configuration& ltConfiguration) { + const std::vector<uint8_t>& sourceData = _spBmpFile->GetData(); + uint32_t numPixels = _spBmpFile->GetNumPixels(); + _layoutTransformData = LayoutTransform(sourceData, numPixels, ltConfiguration); +} + +std::vector<uint16_t> RawImage::LayoutTransform(const std::vector<uint8_t>& sourceData, + uint32_t numPixels, + const ILayoutTransform::Configuration& ltConfiguration) { + if (_indexes.empty()) GenerateLayoutIndexes(ltConfiguration); + + uint32_t numChannels = 3; + uint32_t numSamples = numPixels * numChannels; + + std::vector<uint16_t> meanAdjustedData(numSamples); + const uint8_t* pSourceData = sourceData.data(); + + const uint8_t* pBlueSourcePlane = pSourceData; + const uint8_t* pGreenSourcePlane = pBlueSourcePlane + numPixels; + const uint8_t* pRedSourcePlane = pGreenSourcePlane + numPixels; + + // First adjust by subtracting the means values + std::vector<float> meanAdjustedFloat32(numSamples); + float* pBlueFloat32 = &meanAdjustedFloat32[0]; + float* pGreenFloat32 = pBlueFloat32 + numPixels; + float* pRedFloat32 = pGreenFloat32 + numPixels; + + for (uint32_t i = 0; i < numPixels; i++) { + *pBlueFloat32++ = static_cast<float>(*pBlueSourcePlane++) + ltConfiguration._blueShift; + *pGreenFloat32++ = static_cast<float>(*pGreenSourcePlane++) + ltConfiguration._greenShift; + *pRedFloat32++ = static_cast<float>(*pRedSourcePlane++) + ltConfiguration._redShift; + } + + pBlueFloat32 = &meanAdjustedFloat32[0]; + pGreenFloat32 = pBlueFloat32 + numPixels; + pRedFloat32 = pGreenFloat32 + numPixels; + uint16_t* pBlueDestinationPlane = &meanAdjustedData[0]; + uint16_t* pGreenDestinationPlane = pBlueDestinationPlane + numPixels; + uint16_t* pRedDestinationPlane = pGreenDestinationPlane + numPixels; + + for (uint32_t i = 0; i < numPixels; i++) { + *pBlueDestinationPlane++ = Float16(*pBlueFloat32++); + *pGreenDestinationPlane++ = Float16(*pGreenFloat32++); + *pRedDestinationPlane++ = Float16(*pRedFloat32++); + } + + // Now map the data to the layout expected by the DLA + size_t nLayoutEntries = _indexes.size(); + std::vector<uint16_t> layoutTransformData(nLayoutEntries); + + for (size_t outputIndex = 0; outputIndex < nLayoutEntries; outputIndex++) { + int32_t inputIndex = _indexes[outputIndex]; + if (inputIndex >= 0) + layoutTransformData[outputIndex] = meanAdjustedData[inputIndex]; + else + layoutTransformData[outputIndex] = 0; + } + + return layoutTransformData; +} + +bool RawImage::DumpLayoutTransform() { + if (!_spBmpFile) return false; + + std::filesystem::path filePath(_filePath); + filePath.replace_extension("raw"); + std::ofstream rawRgbaFile(filePath, std::fstream::binary); + if (rawRgbaFile.bad()) return false; + + uint32_t numPixels = _spBmpFile->GetNumPixels(); + uint32_t numChannels = 4; + uint32_t numSamples = numPixels * numChannels; + std::vector<uint8_t> buffer(numSamples); + uint8_t* pSourceData = _spBmpFile->GetData().data(); + + uint8_t* pBlueSourcePlane = pSourceData; + uint8_t* pGreenSourcePlane = pBlueSourcePlane + numPixels; + uint8_t* pRedSourcePlane = pGreenSourcePlane + numPixels; + uint8_t* pDestination = buffer.data(); + + for (uint32_t i = 0; i < numPixels; i++) { + *pDestination++ = *pBlueSourcePlane++; + *pDestination++ = *pGreenSourcePlane++; + *pDestination++ = *pRedSourcePlane++; + *pDestination++ = 0; + } + + rawRgbaFile.write((char*)buffer.data(), buffer.size()); + + filePath.replace_extension("lt"); + std::ofstream transformFile(filePath, std::fstream::binary); + if (transformFile.bad()) return false; + + transformFile.write((char*)GetData(), GetSize()); + + return true; +} + +// Convert from RGBA to planar BGR +std::vector<uint8_t> RawImage::MakePlanar(uint32_t width, uint32_t height, const std::vector<uint8_t>& data) { + uint32_t channelSize = width * height; + std::vector<uint8_t> planarData(channelSize * 3); + uint8_t* pBPlane = planarData.data(); + uint8_t* pGPlane = pBPlane + channelSize; + uint8_t* pRPlane = pGPlane + channelSize; + const uint8_t* pInputRGBA = data.data(); + + for (uint32_t i = 0; i < channelSize; i++) { + *pRPlane++ = *pInputRGBA++; + *pGPlane++ = *pInputRGBA++; + *pBPlane++ = *pInputRGBA++; + + // Skip alpha channel + uint8_t alpha = *pInputRGBA++; + alpha = alpha; + } + + return planarData; +} + +void RawImage::GenerateLayoutIndexes(const ILayoutTransform::Configuration& ltConfiguration) { + size_t nEntries = ltConfiguration._width * ltConfiguration._height * ltConfiguration._cVector; + + uint32_t c_vector = ltConfiguration._cVector; + uint32_t width_stride = 1; + uint32_t height_stride = 1; + uint32_t input_width = ltConfiguration._width; + uint32_t input_height = ltConfiguration._height; + uint32_t input_channels = 3; + uint32_t output_width = ltConfiguration._width; + uint32_t output_width_banked = ltConfiguration._width; + uint32_t output_height = ltConfiguration._height; + uint32_t pad_left = 0; + uint32_t pad_top = 0; + + _indexes.resize(nEntries, -1); + + for (uint32_t c = 0; c < input_channels; c++) { + for (uint32_t h = 0; h < input_height; h++) { + for (uint32_t w = 0; w < input_width; w++) { + uint32_t output_w = (w + pad_left) / width_stride; + uint32_t output_h = (h + pad_top) / height_stride; + uint32_t output_d = c * height_stride * width_stride + ((h + pad_top) % height_stride) * width_stride + + (w + pad_left) % width_stride; + uint32_t output_d_c_vector = output_d / c_vector; + uint32_t cvec = output_d % c_vector; + uint32_t inIndex = c * input_height * input_width + h * input_width + w; + + uint32_t outIndex = (output_d_c_vector * output_height * output_width_banked * c_vector) + + (output_h * output_width_banked * c_vector) + (output_w * c_vector) + cvec; + + if ((output_h < output_height) && (output_w < output_width)) { + _indexes[outIndex] = static_cast<int32_t>(inIndex); + } + } + } + } +} diff --git a/python/openvino/runtime/streaming/image_streaming_app/raw_image.h b/python/openvino/runtime/streaming/image_streaming_app/raw_image.h new file mode 100644 index 0000000..9cb08b4 --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/raw_image.h @@ -0,0 +1,52 @@ +// Copyright 2023 Intel Corporation. +// +// This software and the related documents are Intel copyrighted materials, +// and your use of them is governed by the express license under which they +// were provided to you ("License"). Unless the License provides otherwise, +// you may not use, modify, copy, publish, distribute, disclose or transmit +// this software or the related documents without Intel's prior written +// permission. +// +// This software and the related documents are provided as is, with no express +// or implied warranties, other than those that are expressly stated in the +// License. + +#pragma once +#include <cstdint> +#include <filesystem> +#include <memory> +#include <vector> +#include "ILayoutTransform.h" +#include "bmp_file.h" +#include "float16.h" + +class RawImage { + public: + RawImage(std::filesystem::path filePath, + bool runLayoutTransform, + const ILayoutTransform::Configuration& ltConfiguration); + uint8_t* GetData(); + size_t GetSize(); + std::string Filename() { return _filePath; } + bool DumpLayoutTransform(); + static std::vector<uint16_t> LayoutTransform(uint32_t width, + uint32_t height, + const std::vector<uint8_t>& data, + const ILayoutTransform::Configuration& ltConfiguration); + static std::vector<uint8_t> MakePlanar(uint32_t width, uint32_t height, const std::vector<uint8_t>& data); + bool IsValid(); + + private: + static void GenerateLayoutIndexes(const ILayoutTransform::Configuration& ltConfiguration); + void LayoutTransform(const ILayoutTransform::Configuration& ltConfiguration); + static std::vector<uint16_t> LayoutTransform(const std::vector<uint8_t>& sourceData, + uint32_t numPixels, + const ILayoutTransform::Configuration& ltConfiguration); + + std::filesystem::path _filePath; + std::shared_ptr<BmpFile> _spBmpFile; + std::vector<uint16_t> _layoutTransformData; + static std::vector<int32_t> _indexes; + bool _runLayoutTransform = false; + ILayoutTransform::Configuration _ltConfiguration; +}; diff --git a/python/openvino/runtime/streaming/image_streaming_app/uio/CMakeLists.txt b/python/openvino/runtime/streaming/image_streaming_app/uio/CMakeLists.txt new file mode 100644 index 0000000..4c445a2 --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/uio/CMakeLists.txt @@ -0,0 +1,35 @@ +# Copyright 2023 Intel Corporation +# +# This software and the related documents are Intel copyrighted materials, +# and your use of them is governed by the express license under which they +# were provided to you ("License"). Unless the License provides otherwise, +# you may not use, modify, copy, publish, distribute, disclose or transmit +# this software or the related documents without Intel's prior written +# permission. +# +# This software and the related documents are provided as is, with no express +# or implied warranties, other than those that are expressly stated in the +# License. + +project(uio) + +set(header_files "") +set(source_files "") + +# specify the C++ standard +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +# Header files +set(header_files ${header_files} "include/IUioDevice.h") +set(header_files ${header_files} "source/UioDevice.h") + +# Source files +set(source_files ${source_files} "source/UioDevice.cpp") + +set(all_files ${header_files} ${source_files}) + +add_library(${PROJECT_NAME} STATIC ${all_files}) + +# Include directories +target_include_directories(${PROJECT_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") diff --git a/python/openvino/runtime/streaming/image_streaming_app/uio/include/IUioDevice.h b/python/openvino/runtime/streaming/image_streaming_app/uio/include/IUioDevice.h new file mode 100644 index 0000000..9b964a7 --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/uio/include/IUioDevice.h @@ -0,0 +1,40 @@ +// Copyright 2023 Intel Corporation. +// +// This software and the related documents are Intel copyrighted materials, +// and your use of them is governed by the express license under which they +// were provided to you ("License"). Unless the License provides otherwise, +// you may not use, modify, copy, publish, distribute, disclose or transmit +// this software or the related documents without Intel's prior written +// permission. +// +// This software and the related documents are provided as is, with no express +// or implied warranties, other than those that are expressly stated in the +// License. + +#pragma once +#include <filesystem> +#include <memory> +#include <string> +#include <vector> + +namespace UIO { +class DeviceItem { + public: + std::string _name; + uint32_t _index; + std::string _indexedName; + std::filesystem::path _rootPath; +}; + +class IDevice { + public: + static std::shared_ptr<IDevice> Load(const std::string& deviceName, uint32_t index = 0); + static std::vector<DeviceItem> GetDevices(); + + virtual uint32_t Read(uint32_t registerIndex) = 0; + virtual void Write(uint32_t registerIndex, uint32_t value) = 0; + virtual void ReadBlock(void* host_addr, size_t offset, size_t size) = 0; + virtual void WriteBlock(const void* host_addr, size_t offset, size_t size) = 0; +}; + +} // namespace UIO diff --git a/python/openvino/runtime/streaming/image_streaming_app/uio/source/UioDevice.cpp b/python/openvino/runtime/streaming/image_streaming_app/uio/source/UioDevice.cpp new file mode 100644 index 0000000..d1b5d17 --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/uio/source/UioDevice.cpp @@ -0,0 +1,168 @@ +// Copyright 2023 Intel Corporation. +// +// This software and the related documents are Intel copyrighted materials, +// and your use of them is governed by the express license under which they +// were provided to you ("License"). Unless the License provides otherwise, +// you may not use, modify, copy, publish, distribute, disclose or transmit +// this software or the related documents without Intel's prior written +// permission. +// +// This software and the related documents are provided as is, with no express +// or implied warranties, other than those that are expressly stated in the +// License. + +#include "UioDevice.h" +#include <fcntl.h> +#include <string.h> +#include <sys/mman.h> +#include <unistd.h> +#include <filesystem> +#include <fstream> + +namespace UIO { +static const std::string uioDriverFolder = "/sys/class/uio"; + +std::shared_ptr<IDevice> IDevice::Load(const std::string& deviceName, uint32_t index) { + std::vector<DeviceItem> deviceItems = GetDevices(); + std::string indexedDeviceName = deviceName + std::to_string(index); + + for (auto& deviceItem : deviceItems) { + if (deviceItem._indexedName == indexedDeviceName) { + auto spUioDevice = std::make_shared<Device>(deviceItem); + return spUioDevice->IsValid() ? spUioDevice : nullptr; + } + } + + return nullptr; +} + +std::vector<DeviceItem> IDevice::GetDevices() { + std::vector<DeviceItem> deviceItems; + + for (const auto& entry : std::filesystem::directory_iterator(uioDriverFolder)) { + // Filter out uio* + if (entry.is_directory()) { + std::filesystem::path filePath = entry.path(); + std::string stem = filePath.filename(); + if (stem.substr(0, 3) == "uio") { + std::string indexedDeviceName = Device::ReadStringFromFile(filePath / "name"); + if (not indexedDeviceName.empty()) { + std::string deviceName; + uint32_t index = 0; + Device::SplitIndexedDeviceName(indexedDeviceName, deviceName, index); + deviceItems.push_back({deviceName, index, indexedDeviceName, filePath}); + } + } + } + } + + return deviceItems; +} + +/////////////////////////////////////////////////////////////////////////// + +Device::Device(const DeviceItem& deviceItem) : _deviceItem(deviceItem) { MapRegion(); } + +Device::~Device() { UnmapRegion(); } + +bool Device::IsValid() { return (_fd >= 0); } + +uint32_t Device::Read(uint32_t registerIndex) { + if (registerIndex >= _maximumRegisterIndex) return 0; + + uint32_t* pRegister = (uint32_t*)_pPtr; + + uint32_t value = pRegister[registerIndex]; + return value; +} + +void Device::Write(uint32_t registerIndex, uint32_t value) { + if (registerIndex < _maximumRegisterIndex) { + uint32_t* pRegister = (uint32_t*)_pPtr; + pRegister[registerIndex] = value; + } +} + +void Device::ReadBlock(void* pHostDestination, size_t offset, size_t size) { + if ((offset + size) < _size) { + uint8_t* pDeviceMem = (uint8_t*)_pPtr + offset; + ::memcpy(pHostDestination, pDeviceMem, size); + } +} + +void Device::WriteBlock(const void* pHostSourceAddress, size_t offset, size_t size) { + if ((offset + size) < _size) { + uint8_t* pDeviceMem = (uint8_t*)_pPtr + offset; + ::memcpy(pDeviceMem, pHostSourceAddress, size); + } +} + +uint64_t Device::ReadValueFromFile(const std::filesystem::path& path) { + std::string line = ReadStringFromFile(path); + int base = (line.substr(0, 2) == "0x") ? 16 : 10; + return std::stoull(line, nullptr, base); +} + +std::string Device::ReadStringFromFile(const std::filesystem::path& path) { + std::ifstream inputStream(path); + if (inputStream.good()) { + std::string line; + std::getline(inputStream, line); + return line; + } + return ""; +} + +bool Device::MapRegion() { + _size = ReadValueFromFile(_deviceItem._rootPath / "maps/map0/size"); + _offset = ReadValueFromFile(_deviceItem._rootPath / "maps/map0/offset"); + _physicalAddress = ReadValueFromFile(_deviceItem._rootPath / "maps/map0/addr"); + _maximumRegisterIndex = _size / sizeof(uint32_t); + + std::filesystem::path uioDevicePath = "/dev"; + std::filesystem::path uioDeviceId = _deviceItem._rootPath.stem(); + uioDevicePath /= uioDeviceId; + + _fd = ::open(uioDevicePath.c_str(), O_RDWR); + if (_fd < 0) { + return false; + } + + // Map the region into userspace + _pBase = (uint8_t*)::mmap(NULL, (size_t)_size, PROT_READ | PROT_WRITE, MAP_SHARED, _fd, 0); + if (_pBase == MAP_FAILED) { + return false; + } + + // CST base address is at _pBase + _offset + _pPtr = (uint32_t*)(_pBase + _offset); + + return true; +}; + +void Device::UnmapRegion() { + int r = 0; + if (_pBase) { + r = ::munmap(_pBase, _size); + _pBase = nullptr; + } + + if (_fd >= 0) { + r = ::close(_fd); + _fd = -1; + } + (void)r; +} + +void Device::SplitIndexedDeviceName(const std::string& indexedDeviceName, std::string& deviceName, uint32_t& index) { + int32_t len = static_cast<int32_t>(indexedDeviceName.length()); + int32_t nDecimals = 0; + for (int32_t i = (len - 1); i >= 0; i--) { + if (::isdigit(indexedDeviceName[i])) nDecimals++; + } + + deviceName = indexedDeviceName.substr(0, len - nDecimals); + index = std::stoul(indexedDeviceName.substr(len - nDecimals)); +} + +} // namespace UIO diff --git a/python/openvino/runtime/streaming/image_streaming_app/uio/source/UioDevice.h b/python/openvino/runtime/streaming/image_streaming_app/uio/source/UioDevice.h new file mode 100644 index 0000000..49c6f51 --- /dev/null +++ b/python/openvino/runtime/streaming/image_streaming_app/uio/source/UioDevice.h @@ -0,0 +1,56 @@ +// Copyright 2023 Intel Corporation. +// +// This software and the related documents are Intel copyrighted materials, +// and your use of them is governed by the express license under which they +// were provided to you ("License"). Unless the License provides otherwise, +// you may not use, modify, copy, publish, distribute, disclose or transmit +// this software or the related documents without Intel's prior written +// permission. +// +// This software and the related documents are provided as is, with no express +// or implied warranties, other than those that are expressly stated in the +// License. + +#pragma once + +#include <filesystem> +#include <memory> +#include <string> +#include <vector> +#include "IUioDevice.h" + +namespace UIO { +class Device : public IDevice { + public: + Device(const DeviceItem& deviceItem); + ~Device(); + + // IDevice interface + uint32_t Read(uint32_t registerIndex) override; + void Write(uint32_t registerIndex, uint32_t value) override; + void ReadBlock(void* host_addr, size_t offset, size_t size) override; + void WriteBlock(const void* host_addr, size_t offset, size_t size) override; + + bool IsValid(); + static uint64_t ReadValueFromFile(const std::filesystem::path& path); + static std::string ReadStringFromFile(const std::filesystem::path& path); + static void SplitIndexedDeviceName(const std::string& indexedDeviceName, std::string& deviceName, uint32_t& index); + + private: + Device() = delete; + Device(Device const&) = delete; + void operator=(Device const&) = delete; + + bool MapRegion(); + void UnmapRegion(); + + DeviceItem _deviceItem; + uint32_t _maximumRegisterIndex = 0; + int _fd = -1; // File pointer to UIO - Used to indicate the the Device is valid + uint64_t _physicalAddress = 0; + uint64_t _size = 0; // Size of the mmapped region + uint64_t _offset = 0; // Offset of the first register + uint8_t* _pBase = nullptr; // Base of the mmapped region + uint32_t* _pPtr = nullptr; // The first register +}; +} // namespace UIO |