diff --git a/mediapipe/BUILD b/mediapipe/BUILD index 0810f23582c..51c91987c58 100644 --- a/mediapipe/BUILD +++ b/mediapipe/BUILD @@ -215,8 +215,6 @@ filegroup( srcs = [ "//mediapipe/calculators/geti/serialization:calculators", "//mediapipe/calculators/geti/inference:geti_calculator_base", - "//mediapipe/calculators/geti/inference:grpc_predict_v2_cc_proto", - "//mediapipe/calculators/geti/inference:grpc_predict_v2_proto", "//mediapipe/calculators/geti/inference:inference_calculators", "//mediapipe/calculators/geti/inference:inference_utils", "//mediapipe/calculators/geti/inference:openvino_inference_adapter_calculator_cc_proto", diff --git a/mediapipe/calculators/geti/inference/BUILD b/mediapipe/calculators/geti/inference/BUILD index c7bada606fe..1bf2c10cd44 100644 --- a/mediapipe/calculators/geti/inference/BUILD +++ b/mediapipe/calculators/geti/inference/BUILD @@ -1,18 +1,7 @@ load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") -load("@rules_cc//cc:defs.bzl", "cc_proto_library") package(default_visibility = ["//visibility:public"]) -proto_library( - name = "grpc_predict_v2_proto", - srcs = ["grpc_predict_v2.proto"], -) - -cc_proto_library( - name = "grpc_predict_v2_cc_proto", - deps = [":grpc_predict_v2_proto"], -) - proto_library( name = "openvino_inference_adapter_calculator_proto", srcs = ["openvino_inference_adapter_calculator.proto"], @@ -63,7 +52,7 @@ cc_library( ], deps = [ ":openvino_inference_adapter_calculator_cc_proto", - ":grpc_predict_v2_cc_proto", + "@ovms//src/kfserving_api:kfserving_api_cpp", "//mediapipe/framework:calculator_framework", "//mediapipe/framework/formats:image_frame_opencv", "//mediapipe/framework/port:opencv_core", diff --git a/mediapipe/calculators/geti/inference/kserve.h b/mediapipe/calculators/geti/inference/kserve.h index f5db488d68c..811912ce69c 100644 --- a/mediapipe/calculators/geti/inference/kserve.h +++ b/mediapipe/calculators/geti/inference/kserve.h @@ -16,7 +16,7 @@ #ifndef KSERVE_H #define KSERVE_H -#include "mediapipe/calculators/geti/inference/grpc_predict_v2.pb.h" +#include "src/kfserving_api/grpc_predict_v2.pb.h" using KFSRequest = inference::ModelInferRequest; using KFSResponse = inference::ModelInferResponse; diff --git a/mediapipe/calculators/ovms/BUILD b/mediapipe/calculators/ovms/BUILD index b32f05e5e1e..5efa8e8a6fa 100644 --- a/mediapipe/calculators/ovms/BUILD +++ b/mediapipe/calculators/ovms/BUILD @@ -38,6 +38,26 @@ cc_library( alwayslink = 1, ) +cc_library( + name = "ovms_calculator_runtime", + srcs = [ + ], + deps = [ + "//mediapipe/calculators/openvino:openvino_tensors_to_classification_calculator_cc_proto", + "//mediapipe/calculators/openvino:openvino_tensors_to_detections_calculator_cc_proto", + "//mediapipe/calculators/openvino:openvino_converter_calculator_cc_proto", + "//mediapipe/calculators/openvino:openvino_converter_calculator", + "//mediapipe/calculators/openvino:openvino_tensors_to_classification_calculator", + "//mediapipe/calculators/openvino:openvino_tensors_to_detections_calculator", + ":modelapiovmsadapter", + ":openvinoinferencecalculator_no_tflite", + ":openvinomodelserversessioncalculator", + ], + copts = ["-Iexternal/ovms/src","-Isrc"], + linkopts = ["-Lmediapipe/"], + alwayslink = 1, +) + cc_library( name = "modelapiovmsadapter", srcs = [ @@ -125,6 +145,28 @@ cc_library( alwayslink = 1, ) +cc_library( + name = "openvinoinferencecalculator_no_tflite", + srcs = [ + "openvinoinferencecalculator.cc", + ], + hdrs = [ + "openvinoinferencecalculator.h", + ], + deps = [ + "openvinoinferencecalculatoroptions", + ":modelapiovmsadapter", + ":openvinoinferencecalculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:tensor", + "//third_party:openvino", + "@ovms//src:ovms_header", + ], + copts = ["-Iexternal/ovms/src","-Isrc"], + local_defines = ["OVMS_RUNTIME_DISABLE_TFLITE_TENSORS=1"], + alwayslink = 1, +) + cc_library( name = "openvinomodelserversessioncalculator", srcs = [ diff --git a/mediapipe/calculators/ovms/modelapiovmsadapter.cc b/mediapipe/calculators/ovms/modelapiovmsadapter.cc index 8065ace395d..87680ab7033 100644 --- a/mediapipe/calculators/ovms/modelapiovmsadapter.cc +++ b/mediapipe/calculators/ovms/modelapiovmsadapter.cc @@ -36,7 +36,6 @@ // for the one inside OVMS repo it makes sense to reuse code from ovms lib namespace mediapipe::ovms { -using std::endl; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" using InferenceOutput = std::map; diff --git a/mediapipe/calculators/ovms/modelapiovmsadapter.hpp b/mediapipe/calculators/ovms/modelapiovmsadapter.hpp index 7a315bd9505..9d78c7c8967 100644 --- a/mediapipe/calculators/ovms/modelapiovmsadapter.hpp +++ b/mediapipe/calculators/ovms/modelapiovmsadapter.hpp @@ -17,8 +17,12 @@ #include #include #include +#include +#include #include +#include #include +#include #include #include #include @@ -28,6 +32,10 @@ #include "ovms.h" // NOLINT +#ifdef _WIN32 +#include +#endif + // here we need to decide if we have several calculators (1 for OVMS repository, 1-N inside mediapipe) // for the one inside OVMS repo it makes sense to reuse code from ovms lib @@ -54,6 +62,45 @@ class OVMSInferenceAdapter : public ::InferenceAdapter { std::unordered_map outputDatatypes; ov::AnyMap modelConfig; + // Creates (once per process) and returns a shared OVMS server handle for adapters + // that are constructed without an explicit server pointer. + // This path is used by calculators running in-process when they rely on the default + // OVMS singleton instead of receiving a handle from runtime/shared-library plumbing. + // Keeping one shared handle avoids repeated OVMS_ServerNew calls and keeps all such + // adapters bound to the same server instance. + static OVMS_Server* getSharedServerHandle() { + static std::once_flag once; + static OVMS_Server* sharedServer{nullptr}; + std::fprintf(stderr, "OVMSAdapter shared handle call_once entry thread=%zu\n", std::hash{}(std::this_thread::get_id())); + std::call_once(once, []() { + std::fprintf(stderr, "OVMSAdapter shared handle init start\n"); + auto* serverNewFn = &OVMS_ServerNew; + std::fprintf(stderr, "OVMSAdapter shared handle OVMS_ServerNew fn=%p\n", reinterpret_cast(serverNewFn)); +#ifdef _WIN32 + HMODULE mod = nullptr; + if (GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(serverNewFn), &mod) != 0) { + char modulePath[MAX_PATH] = {0}; + DWORD pathLen = GetModuleFileNameA(mod, modulePath, MAX_PATH); + if (pathLen > 0) { + std::fprintf(stderr, "OVMSAdapter shared handle OVMS_ServerNew module=%s\n", modulePath); + } + } +#endif + OVMS_Status* status = serverNewFn(&sharedServer); + std::fprintf(stderr, "OVMSAdapter shared handle init OVMS_ServerNew returned status=%p server=%p\n", static_cast(status), static_cast(sharedServer)); + if (status != nullptr) { + const char* msg = nullptr; + OVMS_StatusDetails(status, &msg); + std::string details = (msg != nullptr) ? msg : "unknown error"; + OVMS_StatusDelete(status); + throw std::runtime_error("OVMS_ServerNew failed in OVMSInferenceAdapter: " + details); + } + }); + std::fprintf(stderr, "OVMSAdapter shared handle call_once exit server=%p\n", static_cast(sharedServer)); + return sharedServer; + } + public: // TODO Windows: Fix definition in header - does not compile in cpp. OVMSInferenceAdapter(const std::string& servableName, uint32_t servableVersion = 0, OVMS_Server* server = nullptr) : @@ -62,7 +109,7 @@ class OVMSInferenceAdapter : public ::InferenceAdapter { if (nullptr != server) { this->cserver = server; } else { - OVMS_ServerNew(&this->cserver); + this->cserver = getSharedServerHandle(); } } virtual ~OVMSInferenceAdapter(); diff --git a/mediapipe/calculators/ovms/openvinoinferencecalculator.cc b/mediapipe/calculators/ovms/openvinoinferencecalculator.cc index c4c47f14700..98f880b0bf7 100644 --- a/mediapipe/calculators/ovms/openvinoinferencecalculator.cc +++ b/mediapipe/calculators/ovms/openvinoinferencecalculator.cc @@ -27,50 +27,33 @@ #include "ovms.h" // NOLINT #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#if !defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) #include "tensorflow/core/framework/tensor.h" +#endif #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/formats/tensor.h" #include "mediapipe/calculators/ovms/openvinoinferencecalculator.pb.h" #include "mediapipe/calculators/ovms/openvinoinferencecalculatoroptions.h" #include "mediapipe/calculators/ovms/openvinoinferenceutils.h" +#if !defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) #include "tensorflow/lite/c/common.h" +#endif #if (OVMS_DUMP_TO_FILE == 1) #include "openvinoinferencedumputils.h" #endif #pragma GCC diagnostic pop +#if !defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wall" #include "tensorflow/lite/interpreter.h" #pragma GCC diagnostic pop +#endif // here we need to decide if we have several calculators (1 for OVMS repository, 1-N inside mediapipe) // for the one inside OVMS repo it makes sense to reuse code from ovms lib namespace mediapipe { using std::endl; -using TFSDataType = tensorflow::DataType; - -TFSDataType getPrecisionAsDataType(ov::element::Type_t precision) { - static std::unordered_map precisionMap{ - {ov::element::Type_t::f32, TFSDataType::DT_FLOAT}, - {ov::element::Type_t::f64, TFSDataType::DT_DOUBLE}, - {ov::element::Type_t::f16, TFSDataType::DT_HALF}, - {ov::element::Type_t::i64, TFSDataType::DT_INT64}, - {ov::element::Type_t::i32, TFSDataType::DT_INT32}, - {ov::element::Type_t::i16, TFSDataType::DT_INT16}, - {ov::element::Type_t::i8, TFSDataType::DT_INT8}, - {ov::element::Type_t::u64, TFSDataType::DT_UINT64}, - {ov::element::Type_t::u16, TFSDataType::DT_UINT16}, - {ov::element::Type_t::u8, TFSDataType::DT_UINT8}, - {ov::element::Type_t::boolean, TFSDataType::DT_BOOL} - }; - auto it = precisionMap.find(precision); - if (it == precisionMap.end()) { - return TFSDataType::DT_INVALID; - } - return it->second; -} - static Tensor::ElementType OVType2MPType(ov::element::Type_t precision) { static std::unordered_map precisionMap{ // {ov::element::Type_t::f64, Tensor::ElementType::}, @@ -195,6 +178,30 @@ static Tensor convertOVTensor2MPTensor(const ov::Tensor& inputTensor) { return outputTensor; } +#if !defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) +using TFSDataType = tensorflow::DataType; + +TFSDataType getPrecisionAsDataType(ov::element::Type_t precision) { + static std::unordered_map precisionMap{ + {ov::element::Type_t::f32, TFSDataType::DT_FLOAT}, + {ov::element::Type_t::f64, TFSDataType::DT_DOUBLE}, + {ov::element::Type_t::f16, TFSDataType::DT_HALF}, + {ov::element::Type_t::i64, TFSDataType::DT_INT64}, + {ov::element::Type_t::i32, TFSDataType::DT_INT32}, + {ov::element::Type_t::i16, TFSDataType::DT_INT16}, + {ov::element::Type_t::i8, TFSDataType::DT_INT8}, + {ov::element::Type_t::u64, TFSDataType::DT_UINT64}, + {ov::element::Type_t::u16, TFSDataType::DT_UINT16}, + {ov::element::Type_t::u8, TFSDataType::DT_UINT8}, + {ov::element::Type_t::boolean, TFSDataType::DT_BOOL} + }; + auto it = precisionMap.find(precision); + if (it == precisionMap.end()) { + return TFSDataType::DT_INVALID; + } + return it->second; +} + ov::element::Type_t TFSPrecisionToIE2Precision(TFSDataType precision) { static std::unordered_map precisionMap{ {TFSDataType::DT_DOUBLE, ov::element::Type_t::f64}, @@ -278,14 +285,17 @@ static ov::Tensor convertTFLiteTensor2OVTensor(const TfLiteTensor& t) { ov::Tensor result(datatype, shape, data); return result; } +#endif class OpenVINOInferenceCalculator : public CalculatorBase { std::shared_ptr<::InferenceAdapter> session{nullptr}; std::unordered_map outputNameToTag; std::vector input_order_list; std::vector output_order_list; +#if !defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) std::unique_ptr interpreter_ = absl::make_unique(); bool initialized = false; +#endif public: static absl::Status GetContract(CalculatorContract* cc) { @@ -296,6 +306,12 @@ class OpenVINOInferenceCalculator : public CalculatorBase { RET_CHECK(ValidateCalculatorSettings(cc)); for (const std::string& tag : cc->Inputs().GetTags()) { +#if defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) + if (startsWith(tag, TFTENSORS_TAG) || startsWith(tag, TFTENSOR_TAG) || startsWith(tag, TFLITE_TENSORS_TAG) || startsWith(tag, TFLITE_TENSOR_TAG)) { + LOG(INFO) << "TensorFlow/TFLite packet tags are not supported in OVMS runtime-shared mode: " << tag; + RET_CHECK(false); + } +#endif // could be replaced with absl::StartsWith when migrated to MP if (startsWith(tag, OVTENSORS_TAG)) { LOG(INFO) << "setting input tag:" << tag << " to OVTensors"; @@ -309,6 +325,7 @@ class OpenVINOInferenceCalculator : public CalculatorBase { } else if (startsWith(tag, MPTENSOR_TAG)) { LOG(INFO) << "setting input tag:" << tag << " to MPTensor"; cc->Inputs().Tag(tag).Set(); +#if !defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) } else if (startsWith(tag, TFTENSORS_TAG)) { LOG(INFO) << "setting input tag:" << tag << " to TFTensors"; cc->Inputs().Tag(tag).Set>(); @@ -321,12 +338,19 @@ class OpenVINOInferenceCalculator : public CalculatorBase { } else if (startsWith(tag, TFLITE_TENSOR_TAG)) { LOG(INFO) << "setting input tag:" << tag << " to TFLITE_Tensor"; cc->Inputs().Tag(tag).Set(); +#endif } else { LOG(INFO) << "setting input tag:" << tag << " to OVTensor"; cc->Inputs().Tag(tag).Set(); } } for (const std::string& tag : cc->Outputs().GetTags()) { +#if defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) + if (startsWith(tag, TFTENSORS_TAG) || startsWith(tag, TFTENSOR_TAG) || startsWith(tag, TFLITE_TENSORS_TAG) || startsWith(tag, TFLITE_TENSOR_TAG)) { + LOG(INFO) << "TensorFlow/TFLite packet tags are not supported in OVMS runtime-shared mode: " << tag; + RET_CHECK(false); + } +#endif if (startsWith(tag, OVTENSORS_TAG)) { LOG(INFO) << "setting input tag:" << tag << " to std::vector"; cc->Outputs().Tag(tag).Set>(); @@ -339,6 +363,7 @@ class OpenVINOInferenceCalculator : public CalculatorBase { } else if (startsWith(tag, MPTENSOR_TAG)) { LOG(INFO) << "setting input tag:" << tag << " to MPTensor"; cc->Outputs().Tag(tag).Set(); +#if !defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) } else if (startsWith(tag, TFTENSORS_TAG)) { LOG(INFO) << "setting input tag:" << tag << " to TFTensor"; cc->Outputs().Tag(tag).Set>(); @@ -351,6 +376,7 @@ class OpenVINOInferenceCalculator : public CalculatorBase { } else if (startsWith(tag, TFLITE_TENSOR_TAG)) { LOG(INFO) << "setting input tag:" << tag << " to TFLITE_Tensor"; cc->Outputs().Tag(tag).Set(); +#endif } else { LOG(INFO) << "setting output tag:" << tag << " to OVTensor"; cc->Outputs().Tag(tag).Set(); @@ -443,22 +469,28 @@ class OpenVINOInferenceCalculator : public CalculatorBase { try { if (startsWith(tag, OVTENSORS_TAG)) { DESERIALIZE_TENSORS(ov::Tensor,); +#if !defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) } else if (startsWith(tag, TFLITE_TENSORS_TAG)) { DESERIALIZE_TENSORS(TfLiteTensor, convertTFLiteTensor2OVTensor); +#endif } else if (startsWith(tag, MPTENSORS_TAG)) { DESERIALIZE_TENSORS(Tensor, convertMPTensor2OVTensor); } else if (startsWith(tag, OVTENSOR_TAG)) { auto& packet = cc->Inputs().Tag(tag).Get(); input[realInputName] = packet; +#if !defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) } else if (startsWith(tag, TFLITE_TENSOR_TAG)) { auto& packet = cc->Inputs().Tag(tag).Get(); input[realInputName] = convertTFLiteTensor2OVTensor(packet); +#endif } else if (startsWith(tag, MPTENSOR_TAG)) { auto& packet = cc->Inputs().Tag(tag).Get(); input[realInputName] = convertMPTensor2OVTensor(packet); +#if !defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) } else if (startsWith(tag, TFTENSOR_TAG)) { auto& packet = cc->Inputs().Tag(tag).Get(); input[realInputName] = convertTFTensor2OVTensor(packet); +#endif } else { auto& packet = cc->Inputs().Tag(tag).Get(); input[realInputName] = packet; @@ -545,6 +577,7 @@ class OpenVINOInferenceCalculator : public CalculatorBase { SERIALIZE_TENSORS(Tensor, convertOVTensor2MPTensor) // no need to break since we only have one tag // create concatenator calc +#if !defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) } else if (startsWith(tag, TFLITE_TENSORS_TAG)) { LOG(INFO) << "OVMS calculator will process vector"; auto outputStreamTensors = std::vector(); @@ -581,16 +614,19 @@ class OpenVINOInferenceCalculator : public CalculatorBase { } cc->Outputs().Tag(tag).AddPacket(MakePacket>(std::move(outputStreamTensors)).At( cc->InputTimestamp())); break; +#endif } else if (startsWith(tag, OVTENSOR_TAG)) { LOG(INFO) << "OVMS calculator will process ov::Tensor"; cc->Outputs().Tag(tag).Add( new ov::Tensor(tensorIt->second), cc->InputTimestamp()); +#if !defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) } else if (startsWith(tag, TFTENSOR_TAG)) { LOG(INFO) << "OVMS calculator will process tensorflow::Tensor"; cc->Outputs().Tag(tag).Add( new tensorflow::Tensor(convertOVTensor2TFTensor(tensorIt->second)), cc->InputTimestamp()); +#endif } else if (startsWith(tag, MPTENSOR_TAG)) { LOG(INFO) << "OVMS calculator will process mediapipe::Tensor"; cc->Outputs().Tag(tag).Add( diff --git a/mediapipe/calculators/ovms/openvinoinferencecalculator.h b/mediapipe/calculators/ovms/openvinoinferencecalculator.h index 8a92b8120c4..9c9e5cc7f86 100644 --- a/mediapipe/calculators/ovms/openvinoinferencecalculator.h +++ b/mediapipe/calculators/ovms/openvinoinferencecalculator.h @@ -22,10 +22,12 @@ #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/canonical_errors.h" #pragma GCC diagnostic pop +#if !defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wall" #include "tensorflow/lite/interpreter.h" #pragma GCC diagnostic pop +#endif class InferenceAdapter; namespace mediapipe { class OpenVINOInferenceCalculator : public CalculatorBase { @@ -33,8 +35,10 @@ class OpenVINOInferenceCalculator : public CalculatorBase { std::unordered_map outputNameToTag; std::vector input_order_list; std::vector output_order_list; +#if !defined(OVMS_RUNTIME_DISABLE_TFLITE_TENSORS) std::unique_ptr interpreter_ = absl::make_unique(); bool initialized = false; +#endif public: static absl::Status GetContract(CalculatorContract* cc); absl::Status Close(CalculatorContext* cc) override final; diff --git a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.cc b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.cc index e6924801a06..c3da05efebf 100644 --- a/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.cc +++ b/mediapipe/calculators/ovms/openvinomodelserversessioncalculator.cc @@ -22,6 +22,12 @@ #include #include +#ifdef _WIN32 +#include +#else +#include +#endif + #include #include "ovms.h" // NOLINT @@ -36,6 +42,25 @@ namespace mediapipe { using ovms::OVMSInferenceAdapter; +namespace { +OVMS_Server* getInheritedServerHandleFromRuntime() { + using GetterFn = void* (*)(); +#ifdef _WIN32 + HMODULE runtimeHandle = GetModuleHandleA("ovms_mediapipe_runtime_shared.dll"); + if (runtimeHandle == nullptr) { + return nullptr; + } + auto* fn = reinterpret_cast(GetProcAddress(runtimeHandle, "OVMS_MPGetExternalServerHandle")); +#else + auto* fn = reinterpret_cast(dlsym(RTLD_DEFAULT, "OVMS_MPGetExternalServerHandle")); +#endif + if (fn == nullptr) { + return nullptr; + } + return static_cast(fn()); +} +} // namespace + const std::string SESSION_TAG{"SESSION"}; ov::Core UNUSED_OV_CORE; @@ -133,7 +158,15 @@ absl::Status OpenVINOModelServerSessionCalculator::Close(CalculatorContext* cc) LOG(INFO) << "OpenVINOModelServerSessionCalculator Close"; return absl::OkStatus(); } +// Open() prepares an inference adapter and publishes it as a side packet. +// Flow: +// 1) If server_config is provided, it lazily creates/starts OVMS once (guarded by loadingMtx) +// and validates that the server is live/ready. +// 2) If server_config is empty, it reuses a server handle inherited from runtime glue. +// 3) It builds OVMSInferenceAdapter for (servable_name, servable_version), loads model metadata, +// and stores the adapter in SESSION_TAG output side packet. absl::Status OpenVINOModelServerSessionCalculator::Open(CalculatorContext* cc) { + cc->SetOffset(TimestampDiff(0)); LOG(INFO) << "OpenVINOModelServerSessionCalculator Open start"; for (CollectionItemId id = cc->Inputs().BeginId(); id < cc->Inputs().EndId(); ++id) { @@ -147,19 +180,20 @@ absl::Status OpenVINOModelServerSessionCalculator::Open(CalculatorContext* cc) { cc->OutputSidePackets().Get(id).Set(cc->InputSidePackets().Get(id)); } } - cc->SetOffset(TimestampDiff(0)); const auto& options = cc->Options(); // if config is in calc then we start the server LOG(INFO) << "Will check if we want to start server"; + bool isServerLive = false; if (!options.server_config().empty()) { - // Lock access to server from multiple calculator instances during the model loading phase + // Lock access to shared server handle from multiple calculator instances. std::unique_lock lk(OpenVINOModelServerSessionCalculator::loadingMtx); - bool isServerReady = false; - bool isServerLive = false; - OVMS_ServerNew(&cserver); - + if (cserver == nullptr) { + ASSERT_CAPI_STATUS_NULL(OVMS_ServerNew(&cserver)); + } ASSERT_CAPI_STATUS_NULL(OVMS_ServerLive(cserver, &isServerLive)); + // if config is in calc then we start the server + bool isServerReady = false; if (triedToStartOVMS) { RET_CHECK(isServerLive); @@ -180,13 +214,17 @@ absl::Status OpenVINOModelServerSessionCalculator::Open(CalculatorContext* cc) { RET_CHECK(isServerReady); LOG(INFO) << "Server started"; } + } else { + if (cserver == nullptr) { + cserver = getInheritedServerHandleFromRuntime(); + } } const std::string& servableName = options.servable_name(); const std::string& servableVersionStr = options.servable_version(); auto servableVersionOpt = stou32(servableVersionStr); // 0 means default uint32_t servableVersion = servableVersionOpt.value_or(0); - auto session = std::make_shared(servableName, servableVersion); + auto session = std::make_shared(servableName, servableVersion, cserver); try { session->loadModel(nullptr, UNUSED_OV_CORE, "UNUSED", {}); } catch (const std::exception& e) { diff --git a/third_party/BUILD b/third_party/BUILD index 783762f4937..4ac98c2ddd1 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -51,10 +51,10 @@ cc_library( "@com_github_glog_glog//:glog", ], "//mediapipe:windows": [ - "@com_github_glog_glog_windows//:glog", + "@com_github_glog_glog//:glog", ], "//conditions:default": [ - "@com_github_glog_glog//:glog", + "@com_github_glog_glog_no_gflags//:glog", ], }), ) diff --git a/third_party/model_api/model_api.bzl b/third_party/model_api/model_api.bzl index cd4b2c2565d..ab5045fac7e 100644 --- a/third_party/model_api/model_api.bzl +++ b/third_party/model_api/model_api.bzl @@ -51,7 +51,7 @@ cmake( tags = ["requires-network"], deps = [ "@mediapipe//mediapipe/framework/port:opencv_core", - "@mediapipe//third_party:openvino", + "@windows_openvino//:openvino_new_headers", ], ) @@ -59,7 +59,7 @@ cc_library( name = "model_api", deps = [ "@mediapipe//mediapipe/framework/port:opencv_core", - "@mediapipe//third_party:openvino", + "@windows_openvino//:openvino_new_headers", ":model_api_cmake", ], visibility = ["//visibility:public"],