diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index d8449797f..1f3d05e5a 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -19,13 +19,12 @@ under the License. cmake_minimum_required(VERSION 3.11) project(TsFile_CPP) -cmake_policy(SET CMP0079 NEW) set(TsFile_CPP_VERSION 2.0.0.dev) set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall -Werror") message("cmake using: USE_CPP11=${USE_CPP11}") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D__STDC_FORMAT_MACROS") if(DEFINED ENV{CXX}) set(CMAKE_CXX_COMPILER $ENV{CXX}) @@ -65,24 +64,18 @@ elseif (CMAKE_BUILD_TYPE STREQUAL "MinSizeRel") endif() message("CMAKE DEBUG: CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}") + +# All libs will be stored here, including libtsfile, compress-encoding lib. set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib) -set(PROJECT_INCLUDE_DIR ${PROJECT_INCLUDE_DIR} - ${PROJECT_SOURCE_DIR}/src - ${PROJECT_SOURCE_DIR}/third_party/lz4 - ${PROJECT_SOURCE_DIR}/third_party/lzokay - ${PROJECT_SOURCE_DIR}/third_party/zlib-1.2.13 - ${PROJECT_BINARY_DIR}/third_party/zlib-1.2.13 -) -set(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/bin) -include_directories(${PROJECT_INCLUDE_DIR}) +# TsFile code will be stored here. +set(PROJECT_SRC_DIR ${PROJECT_SOURCE_DIR}/src) -include_directories(${PROJECT_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src) +# All include files will be installed here. +set(LIBRARY_INCLUDE_DIR ${PROJECT_BINARY_DIR}/include) add_subdirectory(third_party) - add_subdirectory(src) - add_subdirectory(test) if(TESTS_ENABLED) add_dependencies(TsFile_Test tsfile) diff --git a/cpp/cmake/CopyToDir.cmake b/cpp/cmake/CopyToDir.cmake new file mode 100644 index 000000000..57afa0d9b --- /dev/null +++ b/cpp/cmake/CopyToDir.cmake @@ -0,0 +1,38 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +# CopyToDir.cmake + +# This function is used to copy files to a directory and it will handle relative paths automatically. +function(copy_to_dir) + set(INCLUDE_EXPORT_DR ${LIBRARY_INCLUDE_DIR} CACHE INTERNAL "Include export directory") + foreach(file ${ARGN}) + get_filename_component(file_name ${file} NAME) + get_filename_component(file_path ${file} PATH) + string(REPLACE "${CMAKE_SOURCE_DIR}/src" "" relative_path "${file_path}") + add_custom_target( + copy_${file_name} ALL + COMMAND ${CMAKE_COMMAND} -E make_directory ${INCLUDE_EXPORT_DR}/${relative_path} + COMMAND ${CMAKE_COMMAND} -E copy_if_different ${file} ${INCLUDE_EXPORT_DR}/${relative_path}/${file_name} + COMMENT "Copying ${file_name} to ${INCLUDE_EXPORT_DR}/${relative_path}" + ) + endforeach() +endfunction() + + diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index ecacad70a..715bfa8cf 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -16,12 +16,33 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ]] -message("Running in src diectory") +cmake_minimum_required(VERSION 3.11) +project(TsFile_CPP_SDK) + +include (${CMAKE_SOURCE_DIR}/cmake/CopyToDir.cmake) + +if(POLICY CMP0079) + cmake_policy(SET CMP0079 NEW) +endif() + +message("Running in src directory") if (${COV_ENABLED}) add_compile_options(-fprofile-arcs -ftest-coverage) endif () add_definitions(-DANTLR4CPP_STATIC) set(ANTLR4_WITH_STATIC_CRT OFF) + + +set(PROJECT_INCLUDE_DIR + ${CMAKE_SOURCE_DIR}/src + ${CMAKE_SOURCE_DIR}/third_party/lz4 + ${CMAKE_SOURCE_DIR}/third_party/lzokay + ${CMAKE_SOURCE_DIR}/third_party/zlib-1.2.13 + ${CMAKE_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src +) + +include_directories(${PROJECT_INCLUDE_DIR}) + add_subdirectory(parser) add_subdirectory(common) add_subdirectory(compress) @@ -32,6 +53,7 @@ add_subdirectory(reader) add_subdirectory(utils) add_subdirectory(writer) + set(COMPRESSION_LIBS snappy LZ4 lzokay zlibstatic) target_link_libraries(parser_obj antlr4_static) target_link_libraries(compress_obj ${COMPRESSION_LIBS}) @@ -40,29 +62,26 @@ target_link_libraries(read_obj ${COMPRESSION_LIBS}) target_link_libraries(write_obj ${COMPRESSION_LIBS}) add_library(tsfile SHARED) +add_library(tsfile_static STATIC) if (${COV_ENABLED}) message("Enable code cov...") target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj -lgcov) + target_link_libraries(tsfile_static common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj -lgcov) else() message("Disable code cov...") target_link_libraries(tsfile common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj) + target_link_libraries(tsfile_static common_obj compress_obj cwrapper_obj file_obj read_obj write_obj parser_obj) endif() set(LIBTSFILE_PROJECT_VERSION ${TsFile_CPP_VERSION}) set(LIBTSFILE_SO_VERSION ${TsFile_CPP_VERSION}) set_target_properties(tsfile PROPERTIES VERSION ${LIBTSFILE_PROJECT_VERSION}) set_target_properties(tsfile PROPERTIES SOVERSION ${LIBTSFILE_SO_VERSION}) +set_target_properties(tsfile_static PROPERTIES VERSION ${LIBTSFILE_PROJECT_VERSION}) +set_target_properties(tsfile_static PROPERTIES SOVERSION ${LIBTSFILE_SO_VERSION}) + +install(TARGETS tsfile LIBRARY DESTINATION ${LIBRARY_OUTPUT_PATH}) +install(TARGETS tsfile_static LIBRARY DESTINATION ${LIBRARY_OUTPUT_PATH}) -set(LIBTSFILE_SDK_DIR ${LIBRARY_OUTPUT_PATH}) -install(TARGETS tsfile LIBRARY DESTINATION ${LIBTSFILE_SDK_DIR}) -# set(CMAKE_PREFIX_PATH ../../third-party/lz4-dev/lib) -# set(LZ4_LIB_DIR ../../third-party/lz4-dev/lib) -# find_library(my_lz4_lib NAMES lz4 PATHS ${LZ4_LIB_DIR} NO_DEFAULT_PATH REQUIRED) -# link_directories(${LZ4_LIB_DIR}) -# target_link_libraries(tsfile ${my_lz4_lib}) -# if(CMAKE_SYSTEM_NAME MATCHES "Darwin") -# add_custom_command(TARGET tsfile POST_BUILD COMMAND ${CMAKE_INSTALL_NAME_TOOL} -change `otool -L ${LIBRARY_OUTPUT_PATH}/libtsfile.dylib | grep liblz4 | sed 's/dylib.*/dylib/g'` ${my_lz4_lib} ${LIBRARY_OUTPUT_PATH}/libtsfile.dylib) -# add_custom_command(TARGET tsfile POST_BUILD COMMAND ${CMAKE_INSTALL_NAME_TOOL} -change `otool -L ${LIBRARY_OUTPUT_PATH}/libtsfile.dylib | grep libz | sed 's/dylib.*/dylib/g'` ${my_z_lib} ${LIBRARY_OUTPUT_PATH}/libtsfile.dylib) -# endif() diff --git a/cpp/src/common/CMakeLists.txt b/cpp/src/common/CMakeLists.txt index 7087d03dc..ec285c1d1 100644 --- a/cpp/src/common/CMakeLists.txt +++ b/cpp/src/common/CMakeLists.txt @@ -33,3 +33,6 @@ add_library(common_obj OBJECT ${common_SRC_LIST} ${common_mutex_SRC_LIST} ${common_datatype_SRC_LIST}) +# install header files recursively +file(GLOB_RECURSE HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/*.h") +copy_to_dir(${HEADERS}) \ No newline at end of file diff --git a/cpp/src/common/allocator/object_pool.h b/cpp/src/common/allocator/object_pool.h deleted file mode 100644 index 721823a21..000000000 --- a/cpp/src/common/allocator/object_pool.h +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#ifndef COMMON_ALLOCTOR_OBJECT_POOL_H -#define COMMON_ALLOCTOR_OBJECT_POOL_H - -#include "common/allocator/alloc_base.h" -#include "common/mutex/mutex.h" - -namespace common { - -template -class ObjectPool { - private: - struct ObjectPoolNode { - T data_; - ObjectPoolNode *next_; - - ObjectPoolNode() : data_(), next_(nullptr) {} - }; - - public: - /* - * max_cache_count is a soft limitation: - */ - ObjectPool(const uint32_t max_cache_count, const AllocModID mid, - BaseAllocator &allocator = g_base_allocator) - : max_cache_count_(max_cache_count), - cur_alloc_count_(0), - mid_(mid), - allocator_(allocator), - mutex_(), - head_(nullptr) { - assert(max_cache_count > 1); - } - - ~ObjectPool() { destroy(); } - - void destroy() { - ObjectPoolNode *cur = head_; - while (cur) { - head_ = cur->next_; - allocator_.free(cur); - cur = head_; - cur_alloc_count_--; - } - ASSERT(cur_alloc_count_ == 0); - } - - T *alloc() { - T *ret_obj = nullptr; - common::MutexGuard g(mutex_); - if (head_) { - ret_obj = &(head_->data_); - head_ = head_->next_; - return ret_obj; - } else { - void *buf = allocator_.alloc(sizeof(ObjectPoolNode), mid_); - if (UNLIKELY(buf == nullptr)) { - return nullptr; - } - cur_alloc_count_++; - ret_obj = &(new (buf) ObjectPoolNode)->data_; - return ret_obj; - } - } - - void free(T *obj) { - ASSERT(obj != nullptr); - common::MutexGuard g(mutex_); - if (cur_alloc_count_ > max_cache_count_) { - allocator_.free(obj); - cur_alloc_count_--; - ASSERT(cur_alloc_count_ >= 0); - } else { - ObjectPoolNode *n = (ObjectPoolNode *)obj; - n->next_ = head_; - head_ = n; - } - } - - uint32_t get_cur_alloc_count() const { return cur_alloc_count_; } - - private: - uint32_t max_cache_count_; - uint32_t cur_alloc_count_; - AllocModID mid_; - BaseAllocator allocator_; - common::Mutex mutex_; - ObjectPoolNode *head_; // freelist head -}; - -} // namespace common -#endif // COMMON_ALLOCTOR_OBJECT_POOL_H diff --git a/cpp/src/common/allocator/page_arena.cc b/cpp/src/common/allocator/page_arena.cc index 731b0b0ba..9cc9cc009 100644 --- a/cpp/src/common/allocator/page_arena.cc +++ b/cpp/src/common/allocator/page_arena.cc @@ -18,8 +18,6 @@ */ #include "page_arena.h" -#include - #include namespace common { diff --git a/cpp/src/common/allocator/stl_allocator.h b/cpp/src/common/allocator/stl_allocator.h deleted file mode 100644 index e85839850..000000000 --- a/cpp/src/common/allocator/stl_allocator.h +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#ifndef COMMON_ALLOCATOR_STL_ALLOCATOR_H -#define COMMON_ALLOCATOR_STL_ALLOCATOR_H - -#include "alloc_base.h" - -namespace common { - -template -class StlAllocator { - public: - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef T *pointer; - typedef const T *const_pointer; - typedef T &reference; - typedef const T &const_reference; - typedef T value_type; - - /* - * rebind provides a way to obtain an allocator for a different type. - * For example, std::list will alloc object Node beside alloc object T. - */ - template - struct rebind { - typedef StlAllocator other; - }; - - StlAllocator() {} - StlAllocator(const StlAllocator &) {} - - template - StlAllocator(const StlAllocator &) {} - - StlAllocator(TAllocator base_allocator) : base_allocator_(base_allocator) {} - - pointer address(reference x) const { return &x; } - const_pointer address(const_reference x) { return &x; } - - pointer allocate(size_type n, const void *hint = 0) { - return (pointer)base_allocator_.alloc(n * sizeof(T), Mid); - }; - void deallocate(void *p, size_type) { base_allocator_.free(p); } - size_type max_size() const { return uint32_t(-1); } - - void construct(pointer p, const T &val) { new ((T *)p) T(val); } - void destroy(pointer p) { p->~T(); } - - private: - TAllocator base_allocator_; -}; - -/* - * According to the manual, allocator is stateless. - * Although we define a base_allocator_ here, but base_allocator_ is also - * stateless. so '==' is always true and '!=' is always false. refer to - * https://en.cppreference.com/w/cpp/memory/allocator/operator_cmp. - */ -template -bool operator==(const StlAllocator &a1, - const StlAllocator &a2) { - return true; -} - -template -bool operator!=(const StlAllocator &a1, - const StlAllocator &a2) { - return false; -} - -} // end namespace common -#endif // COMMON_ALLOCATOR_STL_ALLOCATOR_H diff --git a/cpp/src/common/allocator/util_define.h b/cpp/src/common/allocator/util_define.h deleted file mode 100644 index 0526c08b0..000000000 --- a/cpp/src/common/allocator/util_define.h +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * This file defines some basic macros - */ - -#ifndef COMMON_UTIL_DEFINE_H -#define COMMON_UTIL_DEFINE_H - -#include -#include -#include - -/* ======== cmake config define ======== */ - -/* ======== unsued ======== */ -#define UNUSED(v) ((void)(v)) - -/* ======== inline ======== */ -#ifdef __GNUC__ -#define FORCE_INLINE inline __attribute__((always_inline)) -#else -#define FORCE_INLINE inline -#endif // __GNUC__ - -#ifdef BUILD_FOR_SMALL_BINARY -#define INLINE FORCE_INLINE -#else -#define INLINE -#endif // BUILD_FOR_SMALL_BINARY - -/* ======== likely ======== */ -#if defined(__GNUC__) && __GNUC__ >= 4 -#define LIKELY(x) (__builtin_expect((x), 1)) -#define UNLIKELY(x) (__builtin_expect((x), 0)) -#else -#define LIKELY(x) (x) -#define UNLIKELY(x) (x) -#endif // __GNUC__ >= 4 - -/* ======== nullptr ======== */ -#if __cplusplus < 201103L -#ifndef nullptr -#define nullptr NULL -#endif -#define OVERRIDE -#else -#define OVERRIDE override -#endif // __cplusplus < 201103L - -/* ======== cache line ======== */ -#ifndef CACHE_LINE_SIZE -#define CACHE_LINE_SIZE 64 -#endif // CACHE_LINE_SIZE - -/* ======== assert ======== */ -#ifdef NDEBUG -#define ASSERT(condition) ((void)0) -#else -#define ASSERT(condition) assert((condition)) -#endif // NDEBUG - -/* ======== statis assert ======== */ -/* - * To be compatible with C++ before C++11, - * @msg should be a single word (use -/_ to concat) - * such as This_should_be_TRUE - */ -#if __cplusplus < 201103L -// TODO only define this when DEBUG -#define STATIC_ASSERT(cond, msg) \ - typedef char static_assertion_##msg[(cond) ? 1 : -1] __attribute__((unused)) -#else -#define STATIC_ASSERT(cond, msg) static_assert((cond), #msg) -#endif // __cplusplus < 201103L - -/* ======== atomic operation ======== */ -#define ATOMIC_FAA(val_addr, addv) \ - __atomic_fetch_add((val_addr), (addv), __ATOMIC_SEQ_CST) -#define ATOMIC_AAF(val_addr, addv) \ - __atomic_add_fetch((val_addr), (addv), __ATOMIC_SEQ_CST) -/* - * It implements an atomic compare and exchange operation. - * This compares the contents of *ptr with the contents of *expected. - * - If equal, the operation is a reader-modify-writer operation that writes - * desired into *ptr. - * - If they are not equal, the operation is a reader and the current contents - * of *ptr are written into *expected - */ -#define ATOMIC_CAS(val_addr, expected, desired) \ - __atomic_compare_exchange_n((val_addr), (expected), (desired), \ - /* weak = */ false, \ - /* success_memorder = */ __ATOMIC_SEQ_CST, \ - /* failure_memorder = */ __ATOMIC_SEQ_CST) -#define ATOMIC_LOAD(val_addr) __atomic_load_n((val_addr), __ATOMIC_SEQ_CST) -#define ATOMIC_STORE(val_addr, val) \ - __atomic_store_n((val_addr), (val), __ATOMIC_SEQ_CST) - -/* ======== align ======== */ -#define ALIGNED(a) __attribute__((aligned(a))) -#define ALIGNED_4 ALIGNED(4) -#define ALIGNED_8 ALIGNED(8) - -/* ======== disallow copy and assign ======== */ -#if __cplusplus < 201103L -#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ - TypeName(const TypeName&); \ - void operator=(const TypeName&) -#else -#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ - TypeName(const TypeName&) = delete; \ - TypeName& operator=(const TypeName&) = delete; -#endif - -/* ======== return value check ======== */ -#define RET_FAIL(expr) UNLIKELY(common::E_OK != (ret = (expr))) -#define RFAIL(expr) UNLIKELY(common::E_OK != (ret = (expr))) -#define RET_SUCC(expr) LIKELY(common::E_OK != (ret = (exprt))) -#define RSUCC(expr) LIKELY(common::E_OK != (ret = (exprt))) -#define IS_SUCC(ret) LIKELY(common::E_OK == (ret)) -#define IS_FAIL(ret) UNLIKELY(common::E_OK != (ret)) - -#define IS_NULL(ptr) UNLIKELY((ptr) == nullptr) - -/* ======== min/max ======== */ -#define UTIL_MAX(a, b) ((a) > (b) ? (a) : (b)) -#define UTIL_MIN(a, b) ((a) > (b) ? (b) : (a)) - -/* - * int64_max < 10^20 - * consider +/- and the '\0' tail. 24 is enough - */ -#define INT64_TO_BASE10_MAX_LEN 24 - -#endif // COMMON_UTIL_DEFINE_H diff --git a/cpp/src/common/config/config.h b/cpp/src/common/config/config.h index 9eea1086c..31f601d3d 100644 --- a/cpp/src/common/config/config.h +++ b/cpp/src/common/config/config.h @@ -21,9 +21,7 @@ #include -#include "common/mutex/mutex.h" #include "utils/db_utils.h" -#include "utils/util_define.h" namespace common { enum ConfigLevel { @@ -50,18 +48,11 @@ typedef struct ConfigValue { extern void init_config_value(); -// In the future, configuration items need to be dynamically adjusted according -// to the level extern void set_config_value(); -extern void config_set_page_max_point_count(uint32_t page_max_ponint_count); +extern void config_set_page_max_point_count(uint32_t page_max_point_count); extern void config_set_max_degree_of_index_node( uint32_t max_degree_of_index_node); -// FORCE_INLINE bool wal_cfg_enabled() { return -// g_config_value_.wal_flush_policy_ != WAL_DISABLED; } FORCE_INLINE bool -// wal_cfg_should_wait_persisted() { return g_config_value_.wal_flush_policy_ >= -// WAL_FLUSH; } - } // namespace common #endif // COMMON_CONFIG_CONFIG_H diff --git a/cpp/src/common/global.cc b/cpp/src/common/global.cc index 973c6cbba..dd3b9406b 100644 --- a/cpp/src/common/global.cc +++ b/cpp/src/common/global.cc @@ -24,6 +24,7 @@ #endif #include +#include "mutex/mutex.h" #include "utils/injection.h" namespace common { @@ -46,8 +47,8 @@ void init_config_value() { g_config_value_.time_compress_type_ = LZ4; } -void config_set_page_max_point_count(uint32_t page_max_ponint_count) { - g_config_value_.page_writer_max_point_num_ = page_max_ponint_count; +void config_set_page_max_point_count(uint32_t page_max_point_count) { + g_config_value_.page_writer_max_point_num_ = page_max_point_count; } void config_set_max_degree_of_index_node(uint32_t max_degree_of_index_node) { @@ -174,7 +175,6 @@ void print_backtrace() { } #endif -Mutex g_all_inject_points_mutex; std::map g_all_inject_points; } // namespace common diff --git a/cpp/src/common/statistic.h b/cpp/src/common/statistic.h index 37679ba17..907927721 100644 --- a/cpp/src/common/statistic.h +++ b/cpp/src/common/statistic.h @@ -465,12 +465,12 @@ class Int32Statistic : public Statistic { std::string to_string() const { const int buf_len = 256; char buf[buf_len]; - snprintf(buf, buf_len, - "{count=%d, start_time=%" PRId64 ", end_time=%" PRId64 - ", first_val=%d, last_val=%d, sum_value=%" PRId64 - ", min_value=%d, max_value=%d}", - count_, start_time_, end_time_, first_value_, last_value_, - sum_value_, min_value_, max_value_); + // snprintf(buf, buf_len, + // "{count=%d, start_time=%" PRId64 ", end_time=%" PRId64 + // ", first_val=%d, last_val=%d, sum_value=%" PRId64 + // ", min_value=%d, max_value=%d}", + // count_, start_time_, end_time_, first_value_, last_value_, + // sum_value_, min_value_, max_value_); return std::string(buf); } }; @@ -548,13 +548,13 @@ class Int64Statistic : public Statistic { std::string to_string() const { const int buf_len = 256; char buf[buf_len]; - snprintf(buf, buf_len, - "{count=%d, start_time=%" PRId64 ", end_time=%" PRId64 - ", first_val=%" PRId64 ", last_val=%" PRId64 - ", sum_value=%lf, min_value=%" PRId64 ", max_value=%" PRId64 - "}", - count_, start_time_, end_time_, first_value_, last_value_, - sum_value_, min_value_, max_value_); + // snprintf(buf, buf_len, + // "{count=%d, start_time=%" PRId64 ", end_time=%" PRId64 + // ", first_val=%" PRId64 ", last_val=%" PRId64 + // ", sum_value=%lf, min_value=%" PRId64 ", max_value=%" PRId64 + // "}", + // count_, start_time_, end_time_, first_value_, last_value_, + // sum_value_, min_value_, max_value_); return std::string(buf); } }; @@ -733,9 +733,9 @@ class TimeStatistic : public Statistic { std::string to_string() const { const int buf_len = 256; char buf[buf_len]; - snprintf(buf, buf_len, - "{count=%d, start_time=%" PRId64 ", end_time=%" PRId64 "}", - count_, start_time_, end_time_); + // snprintf(buf, buf_len, + // "{count=%d, start_time=%" PRId64 ", end_time=%" PRId64 "}", + // count_, start_time_, end_time_); return std::string(buf); } }; diff --git a/cpp/src/compress/CMakeLists.txt b/cpp/src/compress/CMakeLists.txt index dbc8e5c3c..99bc0cc27 100644 --- a/cpp/src/compress/CMakeLists.txt +++ b/cpp/src/compress/CMakeLists.txt @@ -21,3 +21,5 @@ message("Running in src/storage/tsfile/compress directory") set(CMAKE_POSITION_INDEPENDENT_CODE ON) aux_source_directory(. compress_SRC_LIST) add_library(compress_obj OBJECT ${compress_SRC_LIST}) +file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/*.h") +copy_to_dir(${HEADERS}) diff --git a/cpp/src/cwrapper/CMakeLists.txt b/cpp/src/cwrapper/CMakeLists.txt index 345c66d3c..877902c2d 100644 --- a/cpp/src/cwrapper/CMakeLists.txt +++ b/cpp/src/cwrapper/CMakeLists.txt @@ -19,4 +19,8 @@ under the License. message("Running in cwrapper directory") set(CMAKE_POSITION_INDEPENDENT_CODE ON) aux_source_directory(. CWRAPPER_SRC_LIST) -add_library(cwrapper_obj OBJECT ${CWRAPPER_SRC_LIST}) \ No newline at end of file +add_library(cwrapper_obj OBJECT ${CWRAPPER_SRC_LIST}) + +# install header files +file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/*.h") +copy_to_dir(${HEADERS}) \ No newline at end of file diff --git a/cpp/src/encoding/CMakeLists.txt b/cpp/src/encoding/CMakeLists.txt index d10919691..4611b7472 100644 --- a/cpp/src/encoding/CMakeLists.txt +++ b/cpp/src/encoding/CMakeLists.txt @@ -19,14 +19,5 @@ under the License. message("Running in src/encoding directory") -# aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} storage_tsfile_encode_SRC_LIST) -# set(storage_tsfile_encode_INC_LIST ) -# set(storage_tsfile_encode_DEFINE_OPTIONS ) -# set(storage_tsfile_encode_COMPILE_OPTIONS ) -# set(storage_tsfile_encode_LINK_OPTIONS ) -# -# add_statis_objtarget(storage_tsfile_encoding storage_tsfile_encode_SRC_LIST -# storage_tsfile_encode_INC_LIST -# "${storage_tsfile_encode_DEFINE_OPTIONS}" -# "${storage_tsfile_encode_COMPILE_OPTIONS}" -# "${storage_tsfile_encode_LINK_OPTIONS}") +file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/*.h") +copy_to_dir(${HEADERS}) \ No newline at end of file diff --git a/cpp/src/file/CMakeLists.txt b/cpp/src/file/CMakeLists.txt index e0d863112..5c5296b99 100644 --- a/cpp/src/file/CMakeLists.txt +++ b/cpp/src/file/CMakeLists.txt @@ -21,4 +21,8 @@ message("running in src/file diectory") message("CMAKE_CURRENT_SOURCE_DIR: ${CMAKE_CURRENT_SOURCE_DIR}") set(CMAKE_POSITION_INDEPENDENT_CODE ON) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} file_SRC_LIST) -add_library(file_obj OBJECT ${file_SRC_LIST}) \ No newline at end of file +add_library(file_obj OBJECT ${file_SRC_LIST}) + +# install header files +file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/*.h") +copy_to_dir(${HEADERS}) \ No newline at end of file diff --git a/cpp/src/parser/CMakeLists.txt b/cpp/src/parser/CMakeLists.txt index 92694434e..56cf40aea 100644 --- a/cpp/src/parser/CMakeLists.txt +++ b/cpp/src/parser/CMakeLists.txt @@ -20,3 +20,6 @@ message("Running in src/parser directory") set(CMAKE_POSITION_INDEPENDENT_CODE ON) file(GLOB_RECURSE PARSER_SRC_LIST "*.cpp") add_library(parser_obj OBJECT ${PARSER_SRC_LIST}) + +file(GLOB_RECURSE HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/*.h") +copy_to_dir(${HEADERS}) \ No newline at end of file diff --git a/cpp/src/reader/CMakeLists.txt b/cpp/src/reader/CMakeLists.txt index 1bbc8af06..a3e3b97e3 100644 --- a/cpp/src/reader/CMakeLists.txt +++ b/cpp/src/reader/CMakeLists.txt @@ -20,4 +20,8 @@ message("Running in src/read directory") message("CMAKE_CURRENT_SOURCE_DIR: ${CMAKE_CURRENT_SOURCE_DIR}") set(CMAKE_POSITION_INDEPENDENT_CODE ON) file(GLOB_RECURSE read_SRC_LIST "*.cc") -add_library(read_obj OBJECT ${read_SRC_LIST}) \ No newline at end of file +add_library(read_obj OBJECT ${read_SRC_LIST}) + +# install header files +file(GLOB_RECURSE HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/*.h") +copy_to_dir(${HEADERS}) \ No newline at end of file diff --git a/cpp/src/common/datatype/CMakeLists.txt b/cpp/src/reader/filter/CMakeLists.txt similarity index 99% rename from cpp/src/common/datatype/CMakeLists.txt rename to cpp/src/reader/filter/CMakeLists.txt index f7db3afcc..5860be1f2 100644 --- a/cpp/src/common/datatype/CMakeLists.txt +++ b/cpp/src/reader/filter/CMakeLists.txt @@ -15,4 +15,4 @@ software distributed under the License is distributed on an KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -]] +]] \ No newline at end of file diff --git a/cpp/src/utils/CMakeLists.txt b/cpp/src/utils/CMakeLists.txt index 5f8e83722..0a3594c87 100644 --- a/cpp/src/utils/CMakeLists.txt +++ b/cpp/src/utils/CMakeLists.txt @@ -16,4 +16,8 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ]] -message("Running in src/utils directory") \ No newline at end of file +message("Running in src/utils directory") + +# install header files +file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/*.h") +copy_to_dir(${HEADERS}) \ No newline at end of file diff --git a/cpp/src/utils/injection.h b/cpp/src/utils/injection.h index 76c5aa661..3175a2ab4 100644 --- a/cpp/src/utils/injection.h +++ b/cpp/src/utils/injection.h @@ -33,7 +33,6 @@ struct InjectPoint { // define DBUG_EXECUTE_IF #define DBUG_EXECUTE_IF(inject_point_name, code) \ do { \ - common::MutexGuard mg(g_all_inject_points_mutex); \ if (g_all_inject_points.find(inject_point_name) != \ g_all_inject_points.end()) { \ InjectPoint& inject_point = \ @@ -48,19 +47,16 @@ struct InjectPoint { // open injection #define ENABLE_INJECTION(inject_point_name, count) \ do { \ - common::MutexGuard mg(g_all_inject_points_mutex); \ g_all_inject_points[inject_point_name] = {count}; \ } while (0) // close injection #define DISABLE_INJECTION(inject_point_name) \ do { \ - common::MutexGuard mg(g_all_inject_points_mutex); \ g_all_inject_points.erase(inject_point_name); \ } while (0) // the map save all inject points -extern Mutex g_all_inject_points_mutex; extern std::map g_all_inject_points; } // end namespace common diff --git a/cpp/src/utils/storage_utils.h b/cpp/src/utils/storage_utils.h index ea3698476..faec4a756 100644 --- a/cpp/src/utils/storage_utils.h +++ b/cpp/src/utils/storage_utils.h @@ -76,8 +76,8 @@ FORCE_INLINE std::string get_file_path_from_file_id( char path_buf[len]; memset(path_buf, 0, len); // TODO config - snprintf(path_buf, len, "./%" PRId64 "-%d-%d.tsfile", file_id.seq_, - file_id.version_, file_id.merge_); + // snprintf(path_buf, len, "./%" PRId64 "-%d-%d.tsfile", file_id.seq_, + // file_id.version_, file_id.merge_); return std::string(path_buf); } diff --git a/cpp/src/writer/CMakeLists.txt b/cpp/src/writer/CMakeLists.txt index 30b031dea..aee6ad5a9 100644 --- a/cpp/src/writer/CMakeLists.txt +++ b/cpp/src/writer/CMakeLists.txt @@ -21,4 +21,8 @@ message("running in src/write diectory") message("CMAKE_CURRENT_SOURCE_DIR: ${CMAKE_CURRENT_SOURCE_DIR}") set(CMAKE_POSITION_INDEPENDENT_CODE ON) aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR} write_SRC_LIST) -add_library(write_obj OBJECT ${write_SRC_LIST}) \ No newline at end of file +add_library(write_obj OBJECT ${write_SRC_LIST}) + +# install header files +file(GLOB HEADERS "${CMAKE_CURRENT_SOURCE_DIR}/*.h") +copy_to_dir(${HEADERS}) \ No newline at end of file diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index 77e31d6c0..f8facb496 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -16,32 +16,47 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ]] +cmake_minimum_required(VERSION 3.11) +project(TsFile_CPP_TEST) include(FetchContent) -set(URL_LIST +set(CMAKE_VERBOSE_MAKEFILE ON) + +set(GTEST_URL_LIST "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.zip" "https://hub.nuaa.cf/google/googletest/archive/refs/tags/release-1.12.1.zip" "https://hub.yzuu.cf/google/googletest/archive/refs/tags/release-1.12.1.zip" ) +set(GTEST_ZIP_PATH "${CMAKE_SOURCE_DIR}/third_party/googletest-release-1.12.1.zip") set(DOWNLOADED 0) set(GTEST_URL "") set(TIMEOUT 30) -foreach(URL ${URL_LIST}) - message(STATUS "Trying to download from ${URL}") - file(DOWNLOAD ${URL} "${CMAKE_BINARY_DIR}/googletest-release-1.12.1.zip" STATUS DOWNLOAD_STATUS TIMEOUT ${TIMEOUT}) +if (EXISTS ${GTEST_ZIP_PATH}) + message(STATUS "Using local gtest zip file: ${GTEST_ZIP_PATH}") + set(DOWNLOADED 1) + set(GTEST_URL ${GTEST_ZIP_PATH}) +else() + message(STATUS "Local gtest zip file not found, trying to download from network...") +endif() - list(GET DOWNLOAD_STATUS 0 DOWNLOAD_RESULT) - if(${DOWNLOAD_RESULT} EQUAL 0) - set(DOWNLOADED 1) - set(GTEST_URL ${URL}) - break() - endif() -endforeach() +if (NOT DOWNLOADED) + foreach (URL ${GTEST_URL_LIST}) + message(STATUS "Trying to download from ${URL}") + file(DOWNLOAD ${URL} "${CMAKE_SOURCE_DIR}/third_party/googletest-release-1.12.1.zip" STATUS DOWNLOAD_STATUS TIMEOUT ${TIMEOUT}) + + list(GET DOWNLOAD_STATUS 0 DOWNLOAD_RESULT) + if (${DOWNLOAD_RESULT} EQUAL 0) + set(DOWNLOADED 1) + set(GTEST_URL ${GTEST_ZIP_PATH}) + break() + endif() + endforeach() +endif() if(${DOWNLOADED}) - message(STATUS "Successfully downloaded googletest from ${GTEST_URL}") + message(STATUS "Successfully get googletest from ${GTEST_URL}") FetchContent_Declare( googletest URL ${GTEST_URL} @@ -55,14 +70,19 @@ else() return() endif() + message(STATUS "Adding test configurations...") -set(SDK_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/../src) -message("SDK_INCLUDE_DIR: ${SDK_INCLUDE_DIR}") -set(LIBTSFILE_SDK_DIR ${PROJECT_BINARY_DIR}/lib) -message("LIBTSFILE_SDK_DIR: ${LIBTSFILE_SDK_DIR}") +set(LIB_TSFILE_SDK_DIR ${PROJECT_BINARY_DIR}/lib) +message("LIB_TSFILE_SDK_DIR: ${LIB_TSFILE_SDK_DIR}") -include_directories(${SDK_INCLUDE_DIR}) +include_directories( + ${LIBRARY_INCLUDE_DIR} + ${CMAKE_SOURCE_DIR}/third_party/lz4 + ${CMAKE_SOURCE_DIR}/third_party/lzokay + ${CMAKE_SOURCE_DIR}/third_party/zlib-1.2.13 + ${CMAKE_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src +) enable_testing() @@ -76,6 +96,7 @@ file(GLOB_RECURSE TEST_SRCS "reader/*_test.cc" "writer/*_test.cc" ) + if (${COV_ENABLED}) message("Enable code cov...") add_compile_options(-fprofile-arcs -ftest-coverage) @@ -91,9 +112,9 @@ target_link_libraries( TsFile_Test GTest::gtest_main GTest::gmock - tsfile + tsfile_static ) -set_target_properties(TsFile_Test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${LIBTSFILE_SDK_DIR}) +set_target_properties(TsFile_Test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${LIB_TSFILE_SDK_DIR}) include(GoogleTest) diff --git a/cpp/test/reader/tsfile_reader_test.cc b/cpp/test/reader/tsfile_reader_test.cc index 13ebe6947..b10b419f2 100644 --- a/cpp/test/reader/tsfile_reader_test.cc +++ b/cpp/test/reader/tsfile_reader_test.cc @@ -23,7 +23,6 @@ #include #include -#include "common/path.h" #include "common/record.h" #include "common/schema.h" #include "common/tablet.h" @@ -60,8 +59,8 @@ class TsFileReaderTest : public ::testing::Test { public: static std::string generate_random_string(int length) { - std::random_device rd; - std::mt19937 gen(rd()); + std::mt19937 gen(static_cast( + std::chrono::system_clock::now().time_since_epoch().count())); std::uniform_int_distribution<> dis(0, 61); const std::string chars = @@ -160,7 +159,7 @@ TEST_F(TsFileReaderTest, GetAllDevice) { tsfile_writer_->register_timeseries( "device.ln" + to_string(i), storage::MeasurementSchema(measurement_name, data_type, encoding, - compression_type)); + compression_type)); } for (size_t i = 0; i < 1024; i++) { @@ -180,9 +179,10 @@ TEST_F(TsFileReaderTest, GetAllDevice) { for (size_t i = 0; i < 1024; i++) { devices_name_expected.push_back("device.ln" + std::to_string(i)); } - std::sort(devices_name_expected.begin(), devices_name_expected.end(), [](const std::string& left_str, const std::string& right_str) { - return left_str < right_str; - }); + std::sort(devices_name_expected.begin(), devices_name_expected.end(), + [](const std::string &left_str, const std::string &right_str) { + return left_str < right_str; + }); for (size_t i = 0; i < devices.size(); i++) { ASSERT_EQ(devices[i], devices_name_expected[i]); diff --git a/cpp/test/writer/tsfile_writer_test.cc b/cpp/test/writer/tsfile_writer_test.cc index 9d51078f5..d156db09c 100644 --- a/cpp/test/writer/tsfile_writer_test.cc +++ b/cpp/test/writer/tsfile_writer_test.cc @@ -47,11 +47,13 @@ class TsFileWriterTest : public ::testing::Test { flags |= O_BINARY; #endif mode_t mode = 0666; - EXPECT_EQ(tsfile_writer_->open(file_name_, flags, mode), common::E_OK); + ASSERT_EQ(tsfile_writer_->open(file_name_, flags, mode), common::E_OK); } void TearDown() override { + tsfile_writer_->close(); delete tsfile_writer_; - remove(file_name_.c_str()); + int ret = remove(file_name_.c_str()); + ASSERT_EQ(0, ret); } std::string file_name_; @@ -59,8 +61,8 @@ class TsFileWriterTest : public ::testing::Test { public: static std::string generate_random_string(int length) { - std::random_device rd; - std::mt19937 gen(rd()); + std::mt19937 gen(static_cast( + std::chrono::system_clock::now().time_since_epoch().count())); std::uniform_int_distribution<> dis(0, 61); const std::string chars = @@ -73,7 +75,6 @@ class TsFileWriterTest : public ::testing::Test { for (int i = 0; i < length; ++i) { random_string += chars[dis(gen)]; } - return random_string; } @@ -399,7 +400,6 @@ TEST_F(TsFileWriterTest, WriteMultipleTabletsDouble) { ASSERT_EQ(tsfile_writer_->close(), E_OK); } - TEST_F(TsFileWriterTest, FlushMultipleDevice) { const int device_num = 50; const int measurement_num = 50; @@ -415,15 +415,19 @@ TEST_F(TsFileWriterTest, FlushMultipleDevice) { common::TSEncoding::PLAIN, common::CompressionType::UNCOMPRESSED)); tsfile_writer_->register_timeseries( - device_name, MeasurementSchema(measure_name, common::TSDataType::INT64, - common::TSEncoding::PLAIN, - common::CompressionType::UNCOMPRESSED)); + device_name, + MeasurementSchema(measure_name, common::TSDataType::INT64, + common::TSEncoding::PLAIN, + common::CompressionType::UNCOMPRESSED)); } } for (int i = 0; i < device_num; i++) { std::string device_name = "test_device" + std::to_string(i); - Tablet tablet(device_name, std::make_shared>(schema_vec[i]), max_rows); + Tablet tablet( + device_name, + std::make_shared>(schema_vec[i]), + max_rows); tablet.init(); for (int j = 0; j < measurement_num; j++) { for (int row = 0; row < max_rows; row++) { @@ -438,9 +442,9 @@ TEST_F(TsFileWriterTest, FlushMultipleDevice) { ASSERT_EQ(tsfile_writer_->flush(), E_OK); } ASSERT_EQ(tsfile_writer_->close(), E_OK); - + std::vector select_list; - for (int i = 0; i < device_num; i++) { + for (int i = 0; i < device_num; i++) { std::string device_name = "test_device" + std::to_string(i); for (int j = 0; j < measurement_num; j++) { std::string measurement_name = "measurement" + std::to_string(j); @@ -492,15 +496,19 @@ TEST_F(TsFileWriterTest, AnalyzeTsfileForload) { common::TSEncoding::PLAIN, common::CompressionType::UNCOMPRESSED)); tsfile_writer_->register_timeseries( - device_name, MeasurementSchema(measure_name, common::TSDataType::INT64, - common::TSEncoding::PLAIN, - common::CompressionType::UNCOMPRESSED)); + device_name, + MeasurementSchema(measure_name, common::TSDataType::INT64, + common::TSEncoding::PLAIN, + common::CompressionType::UNCOMPRESSED)); } } for (int i = 0; i < device_num; i++) { std::string device_name = "test_device" + std::to_string(i); - Tablet tablet(device_name, std::make_shared>(schema_vec[i]), max_rows); + Tablet tablet( + device_name, + std::make_shared>(schema_vec[i]), + max_rows); tablet.init(); for (int j = 0; j < measurement_num; j++) { for (int row = 0; row < max_rows; row++) { @@ -514,14 +522,15 @@ TEST_F(TsFileWriterTest, AnalyzeTsfileForload) { } auto schemas = tsfile_writer_->get_schema_group_map(); ASSERT_EQ(schemas->size(), 50); - for (const auto& device_iter : *schemas) { - for (const auto& chunk_iter : device_iter.second->measurement_schema_map_) { + for (const auto &device_iter : *schemas) { + for (const auto &chunk_iter : + device_iter.second->measurement_schema_map_) { ASSERT_NE(chunk_iter.second->chunk_writer_, nullptr); ASSERT_TRUE(chunk_iter.second->chunk_writer_->hasData()); } } ASSERT_EQ(tsfile_writer_->flush(), E_OK); - ASSERT_EQ(tsfile_writer_->close(), E_OK); + ASSERT_EQ(tsfile_writer_->close(), E_OK); } TEST_F(TsFileWriterTest, FlushWithoutWriteAfterRegisterTS) { std::string device_path = "device1";