feat: Storagev2 cpp only (#168)

1. move go part to Milvus 2. support azure compile flag --------- Signed-off-by: shaoting-huang <[email protected]>
milvus-io · Jan 23, 2025 · 7475494 · 7475494
1 parent 9e53927
commit 7475494
Show file tree

Hide file tree

Showing 166 changed files with 331 additions and 7,596 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
diff --git a/.gitignore b/.gitignore
@@ -11,3 +11,4 @@ go/.idea/*
 compile_commands.json
 CMakeUserPresets.json
 .vscode/*
+go/internal/core/output/*
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -1,15 +1,15 @@
 cmake_minimum_required(VERSION 3.20.0)
 
 project(milvus-storage VERSION 0.1.0)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-option(WITH_UT "Build the testing tree." ON)
+option(WITH_UT "Build the testing tree." OFF)
 option(WITH_ASAN "Build with address sanitizer." OFF)
 option(WITH_OPENDAL "Build with opendal." OFF)
 option(WITH_BENCHMARK "Build with micro benchmark." OFF)
+option(WITH_AZURE_FS "Build with azure file system." ON)
 
-set(CMAKE_CXX_STANDARD 17)
-set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR})
 include(GNUInstallDirs)
 
 if (WITH_OPENDAL)
@@ -19,57 +19,64 @@ if (WITH_OPENDAL)
 endif()
 
 find_package(Boost REQUIRED)
+
 find_package(Arrow REQUIRED)
+include_directories(${Arrow_INCLUDE_DIRS})
+
 find_package(Protobuf REQUIRED)
 find_package(glog REQUIRED)
 
 file(GLOB_RECURSE SRC_FILES src/*.cpp src/*.cc)
 
 add_library(milvus-storage SHARED ${SRC_FILES})
-target_include_directories(milvus-storage PUBLIC 
-    ${CMAKE_CURRENT_SOURCE_DIR}/include/milvus-storage 
-    ${CMAKE_CURRENT_SOURCE_DIR}/src 
-    ${CMAKE_CURRENT_SOURCE_DIR}/test/include
-)
 
-set(LINK_LIBS 
-    arrow::arrow
-    Boost::boost
-    protobuf::protobuf
-    AWS::aws-sdk-cpp-identity-management
-    glog::glog
-    Azure::azure-core
-    Azure::azure-storage-blobs
-)
+list(APPEND LINK_LIBS arrow::arrow)
+list(APPEND LINK_LIBS Boost::boost)
+list(APPEND LINK_LIBS protobuf::protobuf)
+list(APPEND LINK_LIBS AWS::aws-sdk-cpp-identity-management)
+list(APPEND LINK_LIBS glog::glog)
 
 if (WITH_OPENDAL)
   list(APPEND LINK_LIBS opendal)
 endif()
 
-target_link_libraries(milvus-storage PUBLIC ${LINK_LIBS})
+if (WITH_AZURE_FS)
+  add_compile_definitions(MILVUS_AZURE_FS)
+  list(APPEND LINK_LIBS Azure::azure-core)
+  list(APPEND LINK_LIBS Azure::azure-storage-blobs)
+endif()
 
-set_target_properties(milvus-storage PROPERTIES 
-    INSTALL_RPATH "$ORIGIN/../lib"
-    BUILD_WITH_INSTALL_RPATH TRUE
-)
+target_link_libraries(milvus-storage PUBLIC ${LINK_LIBS})
+target_include_directories(milvus-storage PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/src)
 
 if (WITH_UT)
-  enable_testing()
   add_subdirectory(test)
 endif()
 
 if (WITH_BENCHMARK)
   add_subdirectory(benchmark)
 endif()
 
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/milvus-storage.pc.in "${CMAKE_CURRENT_BINARY_DIR}/milvus-storage.pc" @ONLY)
+function(add_pkg_config module)
+  configure_file(
+    ${CMAKE_CURRENT_SOURCE_DIR}/src/${module}.pc.in
+    ${CMAKE_CURRENT_BINARY_DIR}/${module}.pc
+    @ONLY
+  )
+  install(
+    FILES "${CMAKE_CURRENT_BINARY_DIR}/${module}.pc"
+    DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig/"
+  )
+endfunction()
+
+add_pkg_config(libstorage)
 
 install(TARGETS milvus-storage
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    DESTINATION ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}
 )
-message( "install cmake install libdir: ${CMAKE_CURRENT_SOURCE_DIR}")
-install(FILES "${CMAKE_CURRENT_BINARY_DIR}/milvus-storage.pc" DESTINATION "${CMAKE_CURRENT_SOURCE_DIR}/build/Release/")
+
+install(DIRECTORY "${PROJECT_SOURCE_DIR}/include/milvus-storage"
+        DESTINATION "${CMAKE_INSTALL_PREFIX}/include")
 
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)
 

diff --git a/cpp/Makefile b/cpp/Makefile
@@ -13,7 +13,7 @@ build:
 
 package: build
 	mkdir -p build && cd build && \
-	conan export .. milvus/testing
+	conan export .. milvus-storage/0.1.0@milvus/dev
 
 debug:
 	mkdir -p build && cd build && \

diff --git a/cpp/conanfile.py b/cpp/conanfile.py
@@ -48,6 +48,7 @@ class StorageConan(ConanFile):
         "arrow:with_thrift": True,
         "arrow:with_jemalloc": True,
         "boost:without_test": True,
+        "boost:without_stacktrace": True,
     }
     exports_sources = (
         "src/*",
@@ -82,19 +83,20 @@ def configure(self):
             self.options.rm_safe("fPIC")
 
     def requirements(self):
-        self.requires("boost/1.81.0")
+        self.requires("boost/1.82.0")
         # self.requires("azure-sdk-for-cpp/1.11.3")
         self.requires("arrow/17.0.0")
         self.requires("openssl/3.1.2")
         self.requires("protobuf/3.21.4")
         self.requires("glog/0.6.0")
         self.requires("zlib/1.2.13")
-        self.requires("libcurl/8.2.1")
-        self.requires("benchmark/1.9.0")
+        self.requires("libcurl/7.86.0")
+        self.requires("benchmark/1.7.0")
         if self.options.with_ut:
             self.requires("gtest/1.13.0")
         if self.settings.os == "Macos":
-            # Macos M1 cannot use jemalloc
+            # Macos M1 cannot use jemalloc and arrow azure fs
+            self.options["arrow"].with_azure = False
             self.options["arrow"].with_jemalloc = False
 
     def validate(self):
@@ -145,6 +147,7 @@ def generate(self):
         tc.variables["WITH_ASAN"] = self.options.with_asan
         tc.variables["WITH_PROFILER"] = self.options.with_profiler
         tc.variables["WITH_UT"] = self.options.with_ut
+        tc.variables["WITH_AZURE_FS"] = self.options["arrow"].with_azure
         tc.generate()
 
         deps = CMakeDeps(self)
@@ -189,4 +192,4 @@ def package_info(self):
         )
         self.cpp_info.components["libstorage"].set_property(
             "pkg_config_name", "libstorage"
-        )
+        )
diff --git a/cpp/include/milvus-storage/common/arrow_util.h b/cpp/include/milvus-storage/common/arrow_util.h
@@ -16,8 +16,8 @@
 #include <memory>
 #include "parquet/arrow/reader.h"
 #include "arrow/filesystem/filesystem.h"
-#include "common/result.h"
-#include "storage/options.h"
+#include "milvus-storage/common/result.h"
+#include "milvus-storage/storage/options.h"
 
 namespace milvus_storage {
 Result<std::unique_ptr<parquet::arrow::FileReader>> MakeArrowFileReader(arrow::fs::FileSystem& fs,

diff --git a/cpp/include/milvus-storage/common/utils.h b/cpp/include/milvus-storage/common/utils.h
@@ -18,8 +18,8 @@
 #include <string>
 #include <unordered_map>
 #include "proto/schema_arrow.pb.h"
-#include "result.h"
-#include "storage/options.h"
+#include "milvus-storage/common/result.h"
+#include "milvus-storage/storage/options.h"
 
 namespace milvus_storage {
 

diff --git a/cpp/include/milvus-storage/file/blob.h b/cpp/include/milvus-storage/file/blob.h
@@ -17,7 +17,7 @@
 #include <string>
 #include <vector>
 #include "proto/manifest.pb.h"
-#include "common/result.h"
+#include "milvus-storage/common/result.h"
 
 namespace milvus_storage {
 struct Blob {

diff --git a/cpp/include/milvus-storage/file/delete_fragment.h b/cpp/include/milvus-storage/file/delete_fragment.h
@@ -17,9 +17,9 @@
 #include <memory>
 #include <unordered_map>
 #include <variant>
-#include "file/fragment.h"
-#include "common/result.h"
-#include "storage/schema.h"
+#include "milvus-storage/file/fragment.h"
+#include "milvus-storage/common/result.h"
+#include "milvus-storage/storage/schema.h"
 #include "arrow/filesystem/filesystem.h"
 #include "arrow/array.h"
 

diff --git a/cpp/include/milvus-storage/file/fragment.h b/cpp/include/milvus-storage/file/fragment.h
@@ -17,7 +17,7 @@
 #include <cstdint>
 #include <memory>
 #include <vector>
-#include <file/file.h>
+#include <milvus-storage/file/file.h>
 #include "proto/manifest.pb.h"
 
 namespace milvus_storage {

diff --git a/cpp/include/milvus-storage/filesystem/azure/azure_fs.h b/cpp/include/milvus-storage/filesystem/azure/azure_fs.h
@@ -13,12 +13,13 @@
 // limitations under the License.
 
 #pragma once
+#ifdef MILVUS_AZURE_FS
 
 #include "arrow/filesystem/azurefs.h"
 #include <cstdlib>
-#include "common/log.h"
-#include "common/macro.h"
-#include "filesystem/fs.h"
+#include "milvus-storage/common/log.h"
+#include "milvus-storage/common/macro.h"
+#include "milvus-storage/filesystem/fs.h"
 
 namespace milvus_storage {
 
@@ -47,3 +48,4 @@ class AzureFileSystemProducer : public FileSystemProducer {
 };
 
 }  // namespace milvus_storage
+#endif
diff --git a/cpp/include/milvus-storage/filesystem/fs.h b/cpp/include/milvus-storage/filesystem/fs.h
@@ -18,8 +18,8 @@
 #include <arrow/util/uri.h>
 #include <memory>
 #include <string>
-#include "common/result.h"
-#include "common/config.h"
+#include "milvus-storage/common/result.h"
+#include "milvus-storage/common/config.h"
 
 namespace milvus_storage {
 

diff --git a/cpp/include/milvus-storage/filesystem/io/io_util.h b/cpp/include/milvus-storage/filesystem/io/io_util.h
@@ -1,7 +1,7 @@
 #include <arrow/io/interfaces.h>
 #include <arrow/status.h>
 #include <arrow/util/thread_pool.h>
-#include "common/log.h"
+#include "milvus-storage/common/log.h"
 
 namespace milvus_storage {
 

diff --git a/cpp/include/milvus-storage/filesystem/s3/multi_part_upload_s3_fs.h b/cpp/include/milvus-storage/filesystem/s3/multi_part_upload_s3_fs.h
@@ -18,8 +18,8 @@
 #include <string>
 #include <vector>
 #include <cstdlib>
-#include "common/log.h"
-#include "common/macro.h"
+#include "milvus-storage/common/log.h"
+#include "milvus-storage/common/macro.h"
 
 #include <arrow/util/key_value_metadata.h>
 #include <arrow/filesystem/s3fs.h>

diff --git a/cpp/include/milvus-storage/filesystem/s3/s3_fs.h b/cpp/include/milvus-storage/filesystem/s3/s3_fs.h
@@ -17,10 +17,10 @@
 #include <arrow/filesystem/s3fs.h>
 #include <arrow/util/uri.h>
 #include <cstdlib>
-#include "common/log.h"
-#include "common/macro.h"
-#include "filesystem/fs.h"
-#include "filesystem/s3/multi_part_upload_s3_fs.h"
+#include "milvus-storage/common/log.h"
+#include "milvus-storage/common/macro.h"
+#include "milvus-storage/filesystem/fs.h"
+#include "milvus-storage/filesystem/s3/multi_part_upload_s3_fs.h"
 
 namespace milvus_storage {
 

diff --git a/cpp/include/milvus-storage/filter/conjunction_filter.h b/cpp/include/milvus-storage/filter/conjunction_filter.h
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 #pragma once
-#include "filter/filter.h"
+#include "milvus-storage/filter/filter.h"
 
 #include <utility>
 

diff --git a/cpp/include/milvus-storage/filter/filter.h b/cpp/include/milvus-storage/filter/filter.h
@@ -17,10 +17,10 @@
 #include <bitset>
 #include <memory>
 #include "arrow/record_batch.h"
-#include "common/constants.h"
+#include "milvus-storage/common/constants.h"
 #include "parquet/statistics.h"
-#include "common/status.h"
-#include "common/macro.h"
+#include "milvus-storage/common/status.h"
+#include "milvus-storage/common/macro.h"
 
 namespace milvus_storage {
 

diff --git a/cpp/include/milvus-storage/format/parquet/file_reader.h b/cpp/include/milvus-storage/format/parquet/file_reader.h
@@ -14,10 +14,10 @@
 
 #pragma once
 #include "arrow/filesystem/filesystem.h"
-#include "format/reader.h"
+#include "milvus-storage/format/reader.h"
 #include "parquet/arrow/reader.h"
-#include "storage/options.h"
-#include "common/config.h"
+#include "milvus-storage/storage/options.h"
+#include "milvus-storage/common/config.h"
 namespace milvus_storage {
 
 class FileRecordBatchReader : public arrow::RecordBatchReader {